From de347884acbf2e982265e2a50034e0dd6d624213 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 24 Mar 2026 19:03:05 +0200 Subject: [PATCH 001/201] Release 4.2.0 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- jetcd-core-shaded/pom.xml | 2 +- jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml | 2 +- jetty-upgrade/pom.xml | 2 +- jetty-upgrade/zookeeper-prometheus-metrics/pom.xml | 2 +- jetty-upgrade/zookeeper-with-patched-admin/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- microbench/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-bom/pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-cli-utils/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-dependencies-minimized/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-docs-tools/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/azure-data-explorer/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis-kpl-shaded/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-opentelemetry/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 135 files changed, 138 insertions(+), 138 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 7804b236c0f0d..3592df42f4f5c 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.0-SNAPSHOT + 4.2.0 bouncy-castle-bc diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index ec888bee54e6c..4088d7fbf760f 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.0-SNAPSHOT + 4.2.0 bcfips-include-test diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index f3859a833f95b..5511d63822e75 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.0-SNAPSHOT + 4.2.0 bouncy-castle-bcfips diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 8c027ad382afa..95c20f2edd6c1 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.0-SNAPSHOT + 4.2.0 diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 0dc58e70729be..473159b5b1427 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 4.2.0-SNAPSHOT + 4.2.0 jar Pulsar Build Tools @@ -43,7 +43,7 @@ - 2025-09-01T03:46:57Z + 2026-03-24T17:03:02Z 1.8 1.8 8 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index ac2dec0201fa9..6ffa58ce8eb49 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.0-SNAPSHOT + 4.2.0 pulsar-io-distribution diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index e94893d192366..26b377bdc67e2 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.0-SNAPSHOT + 4.2.0 pulsar-offloader-distribution diff --git a/distribution/pom.xml b/distribution/pom.xml index 52724020580b9..7c602833fb7ba 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.0-SNAPSHOT + 4.2.0 distribution diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 339b402c9808e..cac6b8ff2ed4d 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.0-SNAPSHOT + 4.2.0 pulsar-server-distribution diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 49346616b0338..9c2c22dab301f 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.0-SNAPSHOT + 4.2.0 pulsar-shell-distribution diff --git a/docker/pom.xml b/docker/pom.xml index 15e740571f0fd..5889db2968171 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.0-SNAPSHOT + 4.2.0 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index bd59bbeac9abf..84fc39a77d973 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.0-SNAPSHOT + 4.2.0 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 72ab40852fb0f..0b4a345771901 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.0-SNAPSHOT + 4.2.0 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index fbe6be31b1945..5b96d9e6d813d 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.0-SNAPSHOT + 4.2.0 jclouds-shaded diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml index 47f4df08f9dd9..6651614e0d060 100644 --- a/jetcd-core-shaded/pom.xml +++ b/jetcd-core-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.0-SNAPSHOT + 4.2.0 jetcd-core-shaded diff --git a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml index b181cc70c3ee9..5a44e6563e0fa 100644 --- a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml +++ b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.0-SNAPSHOT + 4.2.0 pulsar-bookkeeper-prometheus-metrics-provider Apache Pulsar :: BookKeeper Stats Providers :: Prometheus diff --git a/jetty-upgrade/pom.xml b/jetty-upgrade/pom.xml index 6c35a3b41e22c..8889ac3625237 100644 --- a/jetty-upgrade/pom.xml +++ b/jetty-upgrade/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.0-SNAPSHOT + 4.2.0 jetty-upgrade diff --git a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml index 8b12e97181158..32dadb8d96124 100755 --- a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml +++ b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.0-SNAPSHOT + 4.2.0 pulsar-zookeeper-prometheus-metrics diff --git a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml index 963ce462ca7cb..6061e690330be 100644 --- a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml +++ b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar jetty-upgrade - 4.2.0-SNAPSHOT + 4.2.0 zookeeper-with-patched-admin diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 87ebff5655b02..b80620b706123 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.0-SNAPSHOT + 4.2.0 managed-ledger diff --git a/microbench/pom.xml b/microbench/pom.xml index a3db21b4927ee..4ed1580e9acb9 100644 --- a/microbench/pom.xml +++ b/microbench/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.0-SNAPSHOT + 4.2.0 microbench diff --git a/pom.xml b/pom.xml index 9d9c8ad5912c8..3f46d87a0ce63 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 4.2.0-SNAPSHOT + 4.2.0 Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -94,7 +94,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - 2025-09-01T03:46:57Z + 2026-03-24T17:02:57Z true 4.1.12.1 5.7.1 - 4.1.131.Final + 4.1.132.Final 0.0.26.Final 12.1.5 From a0051e52a248cc17ed96eb67676298bd6ed7e7eb Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 26 Mar 2026 14:22:48 +0200 Subject: [PATCH 003/201] [fix][client] Fix thread-safety and refactor MessageCryptoBc key management (#25400) (cherry picked from commit f70745c68a2a1d8ae08137edf0363e9a7595d1d2) --- .../shell/src/assemble/LICENSE.bin.txt | 1 + .../anonymizer/DefaultRoleAnonymizerType.java | 50 +-- .../proxy/WssClientSideEncryptUtils.java | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 5 + .../client/impl/crypto/MessageCryptoBc.java | 321 +++++++++--------- .../pulsar/common/util/SecurityUtility.java | 23 +- 6 files changed, 205 insertions(+), 197 deletions(-) diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 48f7f1ddd54ce..ebeee2013ce6b 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -324,6 +324,7 @@ The Apache Software License, Version 2.0 - jackson-datatype-jdk8-2.18.6.jar - jackson-datatype-jsr310-2.18.6.jar - jackson-module-parameter-names-2.18.6.jar + * Caffeine -- caffeine-3.2.3.jar * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar * Gson - gson-2.13.2.jar diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/common/configuration/anonymizer/DefaultRoleAnonymizerType.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/common/configuration/anonymizer/DefaultRoleAnonymizerType.java index 3333b69cf22ab..d30769aa2f126 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/common/configuration/anonymizer/DefaultRoleAnonymizerType.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/common/configuration/anonymizer/DefaultRoleAnonymizerType.java @@ -18,8 +18,8 @@ */ package org.apache.pulsar.common.configuration.anonymizer; +import io.netty.util.concurrent.FastThreadLocal; import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.Base64; public enum DefaultRoleAnonymizerType { @@ -37,44 +37,44 @@ public String anonymize(String role) { }, SHA256 { private static final String PREFIX = "SHA-256:"; - private final MessageDigest digest; - - { - // Initializing the MessageDigest once for SHA-256 - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("SHA-256 algorithm not found", e); + private static final FastThreadLocal DIGEST = new FastThreadLocal() { + @Override + protected MessageDigest initialValue() throws Exception { + return MessageDigest.getInstance("SHA-256"); } - } + }; @Override public String anonymize(String role) { - byte[] hash = digest.digest(role.getBytes()); - return PREFIX + Base64.getEncoder().encodeToString(hash); + try { + byte[] hash = DIGEST.get().digest(role.getBytes()); + return PREFIX + Base64.getEncoder().encodeToString(hash); + } catch (Exception e) { + throw new RuntimeException("SHA-256 algorithm not found", e); + } } }, MD5 { private static final String PREFIX = "MD5:"; - private final MessageDigest digest; - - { - // Initializing the MessageDigest once for MD5 - try { - // codeql[java/weak-cryptographic-algorithm] - md5 is sufficient for this use case& - digest = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("MD5 algorithm not found", e); + private static final FastThreadLocal DIGEST = new FastThreadLocal() { + @Override + protected MessageDigest initialValue() throws Exception { + // codeql[java/weak-cryptographic-algorithm] - md5 is sufficient for this use case + return MessageDigest.getInstance("MD5"); } - } + }; @Override public String anonymize(String role) { - byte[] hash = digest.digest(role.getBytes()); - return PREFIX + Base64.getEncoder().encodeToString(hash); + try { + byte[] hash = DIGEST.get().digest(role.getBytes()); + return PREFIX + Base64.getEncoder().encodeToString(hash); + } catch (Exception e) { + throw new RuntimeException("MD5 algorithm not found", e); + } } }; private static final String REDACTED_VALUE = "[REDACTED]"; public abstract String anonymize(String role); -} \ No newline at end of file +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/WssClientSideEncryptUtils.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/WssClientSideEncryptUtils.java index 1fb0fd2fd5089..7e05eeb7d660e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/WssClientSideEncryptUtils.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/WssClientSideEncryptUtils.java @@ -131,7 +131,7 @@ public static byte[] calculateEncryptedKeyValue(MessageCryptoBc msgCrypto, byte[ try { PublicKey pubKey = MessageCryptoBc.loadPublicKey(publicKeyData); Cipher dataKeyCipher = loadAndInitCipher(pubKey); - return dataKeyCipher.doFinal(msgCrypto.getDataKey().getEncoded()); + return dataKeyCipher.doFinal(msgCrypto.getEncryptionKey().getEncoded()); } catch (Exception e) { log.error("Failed to encrypt data key. {}", e.getMessage()); throw new PulsarClientException.CryptoException(e.getMessage()); diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index e296721d34676..e9efaa393f8d0 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -47,6 +47,11 @@ + + com.github.ben-manes.caffeine + caffeine + + ${project.groupId} bouncy-castle-bc diff --git a/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java b/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java index 62cc00cd61ec3..ab071ae65584f 100644 --- a/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java +++ b/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java @@ -18,9 +18,9 @@ */ package org.apache.pulsar.client.impl.crypto; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import io.netty.util.concurrent.FastThreadLocal; import java.io.IOException; import java.io.Reader; import java.io.StringReader; @@ -28,15 +28,16 @@ import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.KeyFactory; -import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.NoSuchProviderException; import java.security.PrivateKey; import java.security.PublicKey; import java.security.SecureRandom; import java.security.Security; +import java.security.interfaces.ECPrivateKey; import java.security.spec.AlgorithmParameterSpec; import java.security.spec.InvalidKeySpecException; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -69,8 +70,7 @@ import org.bouncycastle.asn1.x9.ECNamedCurveTable; import org.bouncycastle.asn1.x9.X9ECParameters; import org.bouncycastle.cert.X509CertificateHolder; -import org.bouncycastle.jcajce.provider.asymmetric.ec.BCECPrivateKey; -import org.bouncycastle.jcajce.provider.asymmetric.ec.BCECPublicKey; +import org.bouncycastle.jce.interfaces.ECPublicKey; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.jce.spec.ECParameterSpec; import org.bouncycastle.jce.spec.ECPrivateKeySpec; @@ -83,7 +83,6 @@ @Slf4j public class MessageCryptoBc implements MessageCrypto { - public static final String ECDSA = "ECDSA"; public static final String RSA = "RSA"; public static final String ECIES = "ECIES"; @@ -94,31 +93,26 @@ public class MessageCryptoBc implements MessageCrypto dataKeyCache; + private volatile SecretKey encryptionKey; + private final Cache decryptionKeyCache; + private final Cache lastDecryptionKeyCache; // Map of key name and encrypted gcm key, metadata pair which is sent with encrypted message - private ConcurrentHashMap encryptedDataKeyMap; - + private final ConcurrentHashMap encryptedDataKeyMap; private static final SecureRandom secureRandom; static { - SecureRandom rand = null; + SecureRandom rand; try { rand = SecureRandom.getInstance("NativePRNGNonBlocking"); } catch (NoSuchAlgorithmException nsa) { rand = new SecureRandom(); } - secureRandom = rand; // Initial seed @@ -139,79 +133,101 @@ public class MessageCryptoBc implements MessageCrypto(); - dataKeyCache = CacheBuilder.newBuilder().expireAfterAccess(4, TimeUnit.HOURS) - .build(new CacheLoader() { - - @Override - public SecretKey load(ByteBuffer key) { - return null; - } - - }); - - try { - - cipher = Cipher.getInstance(AESGCM, AESGCM_PROVIDER_NAME); - // If keygen is not needed(e.g: consumer), data key will be decrypted from the message - if (!keyGenNeeded) { - // codeql[java/weak-cryptographic-algorithm] - md5 is sufficient for this use case - digest = MessageDigest.getInstance("MD5"); + // Thread-local instances for non-thread-safe JCA classes + private static final FastThreadLocal THREAD_LOCAL_CIPHER = new FastThreadLocal() { + @Override + protected Cipher initialValue() throws Exception { + return Cipher.getInstance(AESGCM, AESGCM_PROVIDER_NAME); + } + }; - dataKey = null; - return; - } - keyGenerator = KeyGenerator.getInstance("AES"); + private static final FastThreadLocal THREAD_LOCAL_KEY_GENERATOR = + new FastThreadLocal() { + @Override + protected KeyGenerator initialValue() throws Exception { + KeyGenerator kg = KeyGenerator.getInstance("AES"); int aesKeyLength = Cipher.getMaxAllowedKeyLength("AES"); if (aesKeyLength <= 128) { - log.warn("{} AES Cryptographic strength is limited to {} bits. " + log.warn("AES Cryptographic strength is limited to {} bits. " + "Consider installing JCE Unlimited Strength Jurisdiction Policy Files.", - logCtx, aesKeyLength); - keyGenerator.init(aesKeyLength, secureRandom); + aesKeyLength); + kg.init(aesKeyLength, secureRandom); } else { - keyGenerator.init(256, secureRandom); + kg.init(256, secureRandom); } + return kg; + } + }; - } catch (NoSuchAlgorithmException | NoSuchProviderException | NoSuchPaddingException e) { - - cipher = null; - log.error("{} MessageCrypto initialization Failed {}", logCtx, e.getMessage()); - + private static Cipher getAesGcmCipher() throws CryptoException { + try { + return THREAD_LOCAL_CIPHER.get(); + } catch (Exception e) { + log.error("Failed to get AES-GCM cipher instance. {}", e.getMessage()); + throw new PulsarClientException.CryptoException(e.getMessage()); } + } - // Generate data key to encrypt messages - dataKey = keyGenerator.generateKey(); + private static KeyGenerator getKeyGenerator() throws CryptoException { + try { + return THREAD_LOCAL_KEY_GENERATOR.get(); + } catch (Exception e) { + log.error("Failed to get AES key generator instance. {}", e.getMessage()); + throw new PulsarClientException.CryptoException(e.getMessage()); + } + } - iv = new byte[IV_LEN]; + private static SecretKey generateEncryptionKey() throws CryptoException { + return getKeyGenerator().generateKey(); + } + public MessageCryptoBc(String logCtx, boolean keyGenNeeded) { + this.logCtx = logCtx; + encryptedDataKeyMap = new ConcurrentHashMap(); + decryptionKeyCache = Caffeine.newBuilder() + .expireAfterAccess(4, TimeUnit.HOURS) + .weigher((SecretKeyCacheKey key, SecretKeySpec value) -> key.encryptedKeyBytes.length + + value.getEncoded().length) + .maximumWeight(10 * 1024 * 1024) // 10MB upperbound + .build(); + lastDecryptionKeyCache = Caffeine.newBuilder() + .expireAfterAccess(4, TimeUnit.HOURS) + .maximumWeight(10 * 1024 * 1024) // 10MB upperbound + .weigher((String key, SecretKey value) -> key.length() + value.getEncoded().length) + .build(); + if (keyGenNeeded) { + // Generate data key to encrypt messages + try { + encryptionKey = generateEncryptionKey(); + } catch (CryptoException e) { + // retain same contract as before + if (e.getCause() instanceof RuntimeException) { + throw (RuntimeException) e.getCause(); + } else { + throw new RuntimeException(e.getCause()); + } + } + } } public static PublicKey loadPublicKey(byte[] keyBytes) throws Exception { - Reader keyReader = new StringReader(new String(keyBytes)); - PublicKey publicKey = null; + PublicKey publicKey; try (PEMParser pemReader = new PEMParser(keyReader)) { Object pemObj = pemReader.readObject(); JcaPEMKeyConverter pemConverter = new JcaPEMKeyConverter(); - SubjectPublicKeyInfo keyInfo = null; + SubjectPublicKeyInfo keyInfo; X9ECParameters ecParam = null; if (pemObj instanceof ASN1ObjectIdentifier) { - // make sure this is EC Parameter we're handling. In which case // we'll store it and read the next object which should be our // EC Public Key - ASN1ObjectIdentifier ecOID = (ASN1ObjectIdentifier) pemObj; ecParam = ECNamedCurveTable.getByOID(ecOID); if (ecParam == null) { - throw new PEMException("Unable to find EC Parameter for the given curve oid: " - + ((ASN1ObjectIdentifier) pemObj).getId()); + throw new PEMException("Unable to find EC Parameter for the given curve oid: " + ecOID.getId()); } - pemObj = pemReader.readObject(); } else if (pemObj instanceof X9ECParameters) { ecParam = (X9ECParameters) pemObj; @@ -229,7 +245,7 @@ public static PublicKey loadPublicKey(byte[] keyBytes) throws Exception { ECParameterSpec ecSpec = new ECParameterSpec(ecParam.getCurve(), ecParam.getG(), ecParam.getN(), ecParam.getH(), ecParam.getSeed()); KeyFactory keyFactory = KeyFactory.getInstance(ECDSA, BouncyCastleProvider.PROVIDER_NAME); - ECPublicKeySpec keySpec = new ECPublicKeySpec(((BCECPublicKey) publicKey).getQ(), ecSpec); + ECPublicKeySpec keySpec = new ECPublicKeySpec(((ECPublicKey) publicKey).getQ(), ecSpec); publicKey = keyFactory.generatePublic(keySpec); } } catch (IOException | NoSuchAlgorithmException | NoSuchProviderException | InvalidKeySpecException e) { @@ -238,8 +254,7 @@ public static PublicKey loadPublicKey(byte[] keyBytes) throws Exception { return publicKey; } - private PrivateKey loadPrivateKey(byte[] keyBytes) throws Exception { - + private static PrivateKey loadPrivateKey(byte[] keyBytes) throws Exception { Reader keyReader = new StringReader(new String(keyBytes)); PrivateKey privateKey = null; try (PEMParser pemReader = new PEMParser(keyReader)) { @@ -248,31 +263,24 @@ private PrivateKey loadPrivateKey(byte[] keyBytes) throws Exception { Object pemObj = pemReader.readObject(); if (pemObj instanceof ASN1ObjectIdentifier) { - // make sure this is EC Parameter we're handling. In which case // we'll store it and read the next object which should be our // EC Private Key - ASN1ObjectIdentifier ecOID = (ASN1ObjectIdentifier) pemObj; ecParam = ECNamedCurveTable.getByOID(ecOID); if (ecParam == null) { throw new PEMException("Unable to find EC Parameter for the given curve oid: " + ecOID.getId()); } - pemObj = pemReader.readObject(); - } else if (pemObj instanceof X9ECParameters) { - ecParam = (X9ECParameters) pemObj; pemObj = pemReader.readObject(); } if (pemObj instanceof PEMKeyPair) { - PrivateKeyInfo pKeyInfo = ((PEMKeyPair) pemObj).getPrivateKeyInfo(); JcaPEMKeyConverter pemConverter = new JcaPEMKeyConverter(); privateKey = pemConverter.getPrivateKey(pKeyInfo); - } else if (pemObj instanceof PrivateKeyInfo) { JcaPEMKeyConverter pemConverter = new JcaPEMKeyConverter(); privateKey = pemConverter.getPrivateKey((PrivateKeyInfo) pemObj); @@ -280,12 +288,11 @@ private PrivateKey loadPrivateKey(byte[] keyBytes) throws Exception { // if our private key is EC type and we have parameters specified // then we need to set it accordingly - if (ecParam != null && ECDSA.equals(privateKey.getAlgorithm())) { ECParameterSpec ecSpec = new ECParameterSpec(ecParam.getCurve(), ecParam.getG(), ecParam.getN(), ecParam.getH(), ecParam.getSeed()); KeyFactory keyFactory = KeyFactory.getInstance(ECDSA, BouncyCastleProvider.PROVIDER_NAME); - ECPrivateKeySpec keySpec = new ECPrivateKeySpec(((BCECPrivateKey) privateKey).getS(), ecSpec); + ECPrivateKeySpec keySpec = new ECPrivateKeySpec(((ECPrivateKey) privateKey).getS(), ecSpec); privateKey = keyFactory.generatePrivate(keySpec); } @@ -306,11 +313,9 @@ private PrivateKey loadPrivateKey(byte[] keyBytes) throws Exception { * */ @Override - public synchronized void addPublicKeyCipher(Set keyNames, CryptoKeyReader keyReader) - throws CryptoException { - - // Generate data key - dataKey = keyGenerator.generateKey(); + public void addPublicKeyCipher(Set keyNames, CryptoKeyReader keyReader) throws CryptoException { + // Rotate the encryption key each time this method is called + encryptionKey = generateEncryptionKey(); for (String key : keyNames) { addPublicKeyCipher(key, keyReader); @@ -318,7 +323,6 @@ public synchronized void addPublicKeyCipher(Set keyNames, CryptoKeyReade } private void addPublicKeyCipher(String keyName, CryptoKeyReader keyReader) throws CryptoException { - if (keyName == null || keyReader == null) { throw new PulsarClientException.CryptoException("Keyname or KeyReader is null"); } @@ -336,9 +340,8 @@ private void addPublicKeyCipher(String keyName, CryptoKeyReader keyReader) throw throw new PulsarClientException.CryptoException(msg); } - Cipher dataKeyCipher = null; + Cipher dataKeyCipher; byte[] encryptedKey; - try { AlgorithmParameterSpec params = null; // Encrypt data key using public key @@ -357,8 +360,7 @@ private void addPublicKeyCipher(String keyName, CryptoKeyReader keyReader) throw } else { dataKeyCipher.init(Cipher.ENCRYPT_MODE, pubKey); } - encryptedKey = dataKeyCipher.doFinal(dataKey.getEncoded()); - + encryptedKey = dataKeyCipher.doFinal(encryptionKey.getEncoded()); } catch (IllegalBlockSizeException | BadPaddingException | NoSuchAlgorithmException | NoSuchProviderException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { log.error("{} Failed to encrypt data key {}. {}", logCtx, keyName, e.getMessage()); @@ -384,7 +386,6 @@ public static IESParameterSpec createIESParameterSpec() { */ @Override public boolean removeKeyCipher(String keyName) { - if (keyName == null) { return false; } @@ -404,10 +405,9 @@ public boolean removeKeyCipher(String keyName) { * @return encryptedData if success */ @Override - public synchronized void encrypt(Set encKeys, CryptoKeyReader keyReader, - Supplier messageMetadataBuilderSupplier, - ByteBuffer payload, ByteBuffer outBuffer) throws PulsarClientException { - + public void encrypt(Set encKeys, CryptoKeyReader keyReader, + Supplier messageMetadataBuilderSupplier, + ByteBuffer payload, ByteBuffer outBuffer) throws PulsarClientException { MessageMetadata msgMetadata = messageMetadataBuilderSupplier.get(); if (encKeys.isEmpty()) { @@ -444,11 +444,11 @@ public synchronized void encrypt(Set encKeys, CryptoKeyReader keyReader, // We should never reach here. log.error("{} Failed to find encrypted Data key for key {}.", logCtx, keyName); } - } // Create gcm param // TODO: Replace random with counter and periodic refreshing based on timer/counter value + byte[] iv = new byte[IV_LEN]; secureRandom.nextBytes(iv); GCMParameterSpec gcmParam = new GCMParameterSpec(tagLen, iv); @@ -457,7 +457,8 @@ public synchronized void encrypt(Set encKeys, CryptoKeyReader keyReader, try { // Encrypt the data - cipher.init(Cipher.ENCRYPT_MODE, dataKey, gcmParam); + Cipher cipher = getAesGcmCipher(); + cipher.init(Cipher.ENCRYPT_MODE, encryptionKey, gcmParam); int maxLength = cipher.getOutputSize(payload.remaining()); if (outBuffer.remaining() < maxLength) { @@ -474,9 +475,8 @@ public synchronized void encrypt(Set encKeys, CryptoKeyReader keyReader, } } - private boolean decryptDataKey(String keyName, byte[] encryptedDataKey, List encKeyMeta, + private SecretKeySpec tryDecryptDataKey(String keyName, byte[] encryptedDataKey, List encKeyMeta, CryptoKeyReader keyReader) { - Map keyMeta = new HashMap(); encKeyMeta.forEach(kv -> { keyMeta.put(kv.getKey(), kv.getValue()); @@ -491,20 +491,17 @@ private boolean decryptDataKey(String keyName, byte[] encryptedDataKey, List encKeys = msgMetadata.getEncryptionKeysList(); - - // Go through all keys to retrieve data key from cache - for (int i = 0; i < encKeys.size(); i++) { - - byte[] msgDataKey = encKeys.get(i).getValue(); - byte[] keyDigest = digest.digest(msgDataKey); - SecretKey storedSecretKey = dataKeyCache.getIfPresent(ByteBuffer.wrap(keyDigest)); - if (storedSecretKey != null) { - - // Taking a small performance hit here if the hash collides. When it - // returns a different key, decryption fails. At this point, we would - // call decryptDataKey to refresh the cache and come here again to decrypt. - if (decryptData(storedSecretKey, msgMetadata, payload, targetBuffer)) { - // If decryption succeeded, we can already return - return true; - } - } else { - // First time, entry won't be present in cache - log.debug("{} Failed to decrypt data or data key is not in cache. Will attempt to refresh", logCtx); - } - - } - - return false; - } - /* * Decrypt the payload using the data key. Keys used to encrypt data key can be retrieved from msgMetadata * @@ -605,31 +576,73 @@ private boolean getKeyAndDecryptData(MessageMetadata msgMetadata, ByteBuffer pay @Override public boolean decrypt(Supplier messageMetadataSupplier, ByteBuffer payload, ByteBuffer outBuffer, CryptoKeyReader keyReader) { - MessageMetadata msgMetadata = messageMetadataSupplier.get(); - // If dataKey is present, attempt to decrypt using the existing key - if (dataKey != null) { - if (getKeyAndDecryptData(msgMetadata, payload, outBuffer)) { + String producerName = msgMetadata.hasProducerName() ? msgMetadata.getProducerName() : "__not_set__"; + + // Pass 1: Try last used decryption key for this producer + SecretKey lastKey = lastDecryptionKeyCache.getIfPresent(producerName); + if (lastKey != null) { + if (decryptData(lastKey, msgMetadata, payload, outBuffer)) { return true; + } else { + lastDecryptionKeyCache.invalidate(producerName); } } - // dataKey is null or decryption failed. Attempt to regenerate data key List encKeys = msgMetadata.getEncryptionKeysList(); - EncryptionKeys encKeyInfo = encKeys.stream().filter(kbv -> { + // Pass 2: Try cached keys (fast path — no CryptoKeyReader calls) + for (EncryptionKeys encKey : encKeys) { + SecretKeyCacheKey cacheKey = new SecretKeyCacheKey(encKey.getValue()); + SecretKey cachedKey = decryptionKeyCache.getIfPresent(cacheKey); + if (cachedKey != null) { + if (decryptData(cachedKey, msgMetadata, payload, outBuffer)) { + lastDecryptionKeyCache.put(producerName, cachedKey); + return true; + } + } + } - byte[] encDataKey = kbv.getValue(); - List encKeyMeta = kbv.getMetadatasList(); - return decryptDataKey(kbv.getKey(), encDataKey, encKeyMeta, keyReader); + // Pass 3: Decrypt data keys via CryptoKeyReader (slow path) + for (EncryptionKeys encKey : encKeys) { + SecretKeySpec decryptedKey = tryDecryptDataKey( + encKey.getKey(), encKey.getValue(), encKey.getMetadatasList(), keyReader); + if (decryptedKey != null) { + SecretKeyCacheKey cacheKey = new SecretKeyCacheKey(encKey.getValue()); + decryptionKeyCache.put(cacheKey, decryptedKey); + if (decryptData(decryptedKey, msgMetadata, payload, outBuffer)) { + lastDecryptionKeyCache.put(producerName, decryptedKey); + return true; + } + } + } + + return false; + } - }).findFirst().orElse(null); + // key to be used in the cache + private static final class SecretKeyCacheKey { + private final byte[] encryptedKeyBytes; + private final int hashCode; - if (encKeyInfo == null || dataKey == null) { - // Unable to decrypt data key - return false; + SecretKeyCacheKey(byte[] encryptedKeyBytes) { + this.encryptedKeyBytes = encryptedKeyBytes.clone(); + this.hashCode = Arrays.hashCode(this.encryptedKeyBytes); } - return getKeyAndDecryptData(msgMetadata, payload, outBuffer); + @Override + public int hashCode() { + return hashCode; + } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SecretKeyCacheKey)) { + return false; + } + return Arrays.equals(encryptedKeyBytes, ((SecretKeyCacheKey) o).encryptedKeyBytes); + } } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java index 2b7b1a984634f..fa72327dc47f0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java @@ -52,7 +52,6 @@ import java.security.spec.KeySpec; import java.security.spec.PKCS8EncodedKeySpec; import java.util.ArrayList; -import java.util.Arrays; import java.util.Base64; import java.util.Collection; import java.util.List; @@ -83,10 +82,7 @@ public class SecurityUtility { public static final String BC_NON_FIPS_PROVIDER_CLASS = "org.bouncycastle.jce.provider.BouncyCastleProvider"; public static final String CONSCRYPT_PROVIDER_CLASS = "org.conscrypt.OpenSSLProvider"; public static final Provider CONSCRYPT_PROVIDER = loadConscryptProvider(); - private static final List KEY_FACTORIES = Arrays.asList( - createKeyFactory("RSA"), - createKeyFactory("EC") - ); + private static final List KEY_FACTORY_ALGORITHMS = List.of("RSA", "EC"); // Security.getProvider("BC") / Security.getProvider("BCFIPS"). // also used to get Factories. e.g. CertificateFactory.getInstance("X.509", "BCFIPS") @@ -521,12 +517,12 @@ public static PrivateKey loadPrivateKeyFromPemStream(InputStream inStream) throw sb.append(currentLine); } final KeySpec keySpec = new PKCS8EncodedKeySpec(Base64.getDecoder().decode(sb.toString())); - final List failedAlgorithm = new ArrayList<>(KEY_FACTORIES.size()); - for (KeyFactory kf : KEY_FACTORIES) { + final List failedAlgorithm = new ArrayList<>(KEY_FACTORY_ALGORITHMS.size()); + for (String algorithm : KEY_FACTORY_ALGORITHMS) { try { - return kf.generatePrivate(keySpec); - } catch (InvalidKeySpecException ex) { - failedAlgorithm.add(kf.getAlgorithm()); + return KeyFactory.getInstance(algorithm).generatePrivate(keySpec); + } catch (InvalidKeySpecException | NoSuchAlgorithmException ex) { + failedAlgorithm.add(algorithm); } } throw new KeyManagementException("The private key algorithm is not supported. attempted: " @@ -596,11 +592,4 @@ public static Provider resolveProvider(String providerName) throws NoSuchAlgorit return provider; } - private static KeyFactory createKeyFactory(String algorithm) { - try { - return KeyFactory.getInstance(algorithm); - } catch (Exception e) { - throw new IllegalArgumentException(String.format("Illegal key factory algorithm " + algorithm), e); - } - } } From d8170d535c54cad0c3edc7b94cf2585e4bdce15a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 31 Mar 2026 23:20:40 +0300 Subject: [PATCH 004/201] [improve][ci] Cleanup tune-runner-vm and clean-disk actions (#25444) (cherry picked from commit 396290adabdf85f905add66b03f406b1c14cd20b) --- .github/actions/clean-disk/action.yml | 116 ++++++++++++++++------ .github/actions/tune-runner-vm/action.yml | 50 ++++------ 2 files changed, 102 insertions(+), 64 deletions(-) diff --git a/.github/actions/clean-disk/action.yml b/.github/actions/clean-disk/action.yml index e67ce59a08ddb..fe5fa9e7cc93a 100644 --- a/.github/actions/clean-disk/action.yml +++ b/.github/actions/clean-disk/action.yml @@ -23,44 +23,98 @@ inputs: mode: description: "Use 'full' to clean as much as possible" required: false + clean-threshold-gb: + description: "Skip cleaning if available disk space exceeds this threshold (in GB)" + required: false + default: '20' + full-clean-threshold-gb: + description: "In 'full' mode, skip additional cleaning steps if available disk space exceeds this threshold (in GB)" + required: false + default: '40' runs: using: composite steps: - - run: | + - shell: bash + env: + CLEAN_MODE: ${{ inputs.mode }} + CLEAN_THRESHOLD_GB: ${{ inputs.clean-threshold-gb }} + FULL_CLEAN_THRESHOLD_GB: ${{ inputs.full-clean-threshold-gb }} + run: | if [[ "$OSTYPE" == "linux-gnu"* ]]; then - directories=(/usr/local/lib/android /opt/ghc) - if [[ "${{ inputs.mode }}" == "full" ]]; then - # remove these directories only when mode is 'full' - directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL) + + available_gb() { + df --output=avail -BG / | tail -1 | tr -d ' G' + } + + show_disk() { + if mountpoint -q /mnt; then df -BM / /mnt; else df -BM /; fi + } + + remove_dir() { + local directory=$1 + if [ -d "$directory" ]; then + echo "::group::Removing $directory" + local emptydir=/tmp/empty$$/ + mkdir -p "$emptydir" + # fast way to delete a lot of files on linux + time sudo eatmydata rsync -a --delete "$emptydir" "${directory}/" + time sudo eatmydata rm -rf "${directory}" + rmdir "$emptydir" 2>/dev/null || true + time show_disk + echo "::endgroup::" + fi + } + + # Remove directories one at a time, stopping when threshold is met + clean_dirs() { + local threshold=$1 + shift + for directory in "$@"; do + if (( $(available_gb) >= threshold )); then + return + fi + remove_dir "$directory" + done + } + + if [[ "$CLEAN_MODE" == "full" ]]; then + threshold=$FULL_CLEAN_THRESHOLD_GB + else + threshold=$CLEAN_THRESHOLD_GB fi - emptydir=/tmp/empty$$/ - mkdir $emptydir - echo "::group::Available diskspace" - time df -BM / /mnt + + echo "::group::Available diskspace (threshold: ${threshold}GB)" + time show_disk echo "::endgroup::" - for directory in "${directories[@]}"; do - echo "::group::Removing $directory" - # fast way to delete a lot of files on linux - time sudo eatmydata rsync -a --delete $emptydir ${directory}/ - time sudo eatmydata rm -rf ${directory} - time df -BM / /mnt - echo "::endgroup::" - done - if [[ "${{ inputs.mode }}" == "full" ]]; then - echo "::group::Moving /var/lib/docker to /mnt/docker" - sudo systemctl stop docker - echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json - sudo mv /var/lib/docker /mnt/docker - sudo systemctl start docker - time df -BM / /mnt + + if (( $(available_gb) >= threshold )); then + echo "Available disk space meets threshold, skipping cleanup." + exit 0 + fi + + clean_dirs "$threshold" /usr/local/lib/android /opt/ghc + + if [[ "$CLEAN_MODE" == "full" ]]; then + clean_dirs "$threshold" /usr/share/dotnet /opt/hostedtoolcache/CodeQL + if (( $(available_gb) < threshold )) && mountpoint -q /mnt; then + echo "::group::Moving /var/lib/docker to /mnt/docker" + sudo systemctl stop docker + echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json + sudo mv /var/lib/docker /mnt/docker + sudo systemctl start docker + time show_disk + echo "::endgroup::" + fi + fi + + if (( $(available_gb) < threshold )); then + echo "::group::Cleaning apt state" + time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove" + time show_disk echo "::endgroup::" fi - echo "::group::Cleaning apt state" - time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove" - time df -BM / /mnt + + echo "::group::Available diskspace" + time show_disk echo "::endgroup::" fi - echo "::group::Available diskspace" - time df -BM / /mnt - echo "::endgroup::" - shell: bash diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml index ab0f65767a62d..d071f534dbb4f 100644 --- a/.github/actions/tune-runner-vm/action.yml +++ b/.github/actions/tune-runner-vm/action.yml @@ -22,7 +22,8 @@ description: tunes the GitHub Runner VM operation system runs: using: composite steps: - - run: | + - shell: bash + run: | if [[ "$OSTYPE" == "linux-gnu"* ]]; then echo "::group::Configure and tune OS" # Ensure that reverse lookups for current hostname are handled properly @@ -33,49 +34,38 @@ runs: # consumption is high. # Set vm.swappiness=1 to avoid swapping and allow high RAM usage echo 1 | sudo tee /proc/sys/vm/swappiness - ( - shopt -s nullglob - # Set swappiness to 1 for all cgroups and sub-groups - for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do - echo 1 | sudo tee $swappiness_file > /dev/null - done - ) || true # use "madvise" Linux Transparent HugePages (THP) setting # https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html # "madvise" is generally a better option than the default "always" setting - # Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp + # recommendation from https://netflixtechblog.com/bending-pause-times-to-your-will-with-generational-zgc-256629c9386b echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled - echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag + echo defer | sudo tee /sys/kernel/mm/transparent_hugepage/defrag echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds) # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes) sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true - sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true + if mountpoint -q /mnt; then + sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true + fi # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective # https://www.spinics.net/lists/linux-ide/msg52562.html for i in /sys/block/sd*/queue/discard_max_bytes; do echo 0 | sudo tee $i done - # disable any background jobs that run SSD discard/trim - sudo systemctl disable fstrim.timer || true - sudo systemctl stop fstrim.timer || true - sudo systemctl disable fstrim.service || true - sudo systemctl stop fstrim.service || true + # disable unnecessary timers + sudo systemctl stop fstrim.timer fstrim.service \ + podman-auto-update.timer sysstat-collect.timer sysstat-summary.timer \ + phpsessionclean.timer man-db.timer motd-news.timer \ + dpkg-db-backup.timer e2scrub_all.timer \ + update-notifier-download.timer update-notifier-motd.timer || true - # stop php-fpm - sudo systemctl stop php8.0-fpm.service || true - sudo systemctl stop php7.4-fpm.service || true - # stop mono-xsp4 - sudo systemctl disable mono-xsp4.service || true - sudo systemctl stop mono-xsp4.service || true - sudo killall mono || true - - # stop Azure Linux agent to save RAM - sudo systemctl stop walinuxagent.service || true + # stop unnecessary services + sudo systemctl stop php8.3-fpm.service ModemManager.service \ + multipathd.service udisks2.service walinuxagent.service || true echo '::endgroup::' @@ -87,10 +77,4 @@ runs: echo "::group::Available diskspace" df -BM echo "::endgroup::" - # show cggroup - echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP" - CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }') - sudo cgget -a $CURRENT_CGGROUP || true - echo '::endgroup::' - fi - shell: bash + fi \ No newline at end of file From 3825cbd7458e5dbd89a3d36f98fd1433e12718a8 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Tue, 7 Apr 2026 11:00:13 +0800 Subject: [PATCH 005/201] [fix][admin] Refactor namespace migration operation to async in rest api (#25478) (cherry picked from commit f72a5a78c8f705093bae68847dc1b7dbc38d7fb1) --- .../broker/admin/impl/NamespacesBase.java | 38 +++++-------------- .../pulsar/broker/admin/v2/Namespaces.java | 12 +++++- .../pulsar/broker/admin/NamespacesV2Test.java | 25 +++++++++++- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 0f626eb738581..288c2a6c40a01 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -3081,22 +3081,15 @@ protected void internalRemoveBacklogQuota(AsyncResponse asyncResponse, BacklogQu }); } - protected void internalEnableMigration(boolean migrated) { - validateSuperUserAccess(); - try { - getLocalPolicies().setLocalPoliciesWithCreate(namespaceName, oldPolicies -> oldPolicies.map( - policies -> new LocalPolicies(policies.bundles, - policies.bookieAffinityGroup, - policies.namespaceAntiAffinityGroup, - migrated)) - .orElseGet(() -> new LocalPolicies(getDefaultBundleData(), null, null, migrated))); - log.info("Successfully updated migration on namespace {}", namespaceName); - } catch (RestException re) { - throw re; - } catch (Exception e) { - log.error("Failed to update migration on namespace {}", namespaceName, e); - throw new RestException(e); - } + protected CompletableFuture internalEnableMigrationAsync(boolean migrated) { + return validateSuperUserAccessAsync().thenCompose(__ -> getDefaultBundleDataAsync().thenCompose( + defaultBundleData -> getLocalPolicies().setLocalPoliciesWithCreateAsync(namespaceName, + oldPolicies -> oldPolicies.map( + policies -> new LocalPolicies(policies.bundles, + policies.bookieAffinityGroup, + policies.namespaceAntiAffinityGroup, migrated)) + .orElseGet(() -> new LocalPolicies(defaultBundleData, null, null, migrated))))) + .thenAccept(__ -> log.info("Successfully updated migration on namespace {}", namespaceName)); } protected Policies getDefaultPolicesIfNull(Policies policies) { @@ -3183,19 +3176,6 @@ protected CompletableFuture> internalGetNamespaceAllowedClustersAsyn .thenApply(policies -> policies.allowed_clusters); } - // TODO remove this sync method after async refactor - @Deprecated - private BundlesData getDefaultBundleData() { - try { - return getDefaultBundleDataAsync().get(config().getMetadataStoreOperationTimeoutSeconds(), - TimeUnit.SECONDS); - } catch (Exception e) { - log.error("[{}] Failed to get namespace-policy configuration for namespace {}", clientAppId(), - namespaceName, e); - throw new RestException(e); - } - } - private CompletableFuture getDefaultBundleDataAsync() { return namespaceResources().getPoliciesAsync(namespaceName).thenApply( optionalPolicies -> optionalPolicies.isPresent() ? optionalPolicies.get().bundles : diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index b770c078a2876..173c22fac79e6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -3158,11 +3158,19 @@ public void removeNamespaceEntryFilters(@Suspended AsyncResponse asyncResponse, @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist") }) - public void enableMigration(@PathParam("tenant") String tenant, + public void enableMigration(@Suspended AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, boolean migrated) { validateNamespaceName(tenant, namespace); - internalEnableMigration(migrated); + internalEnableMigrationAsync(migrated) + .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) + .exceptionally(ex -> { + log.error("[{}] Failed to update migration, tenant: {}, namespace: {}", + clientAppId(), tenant, namespace, ex); + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } @POST diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java index 70f8ad90d44e2..b425ca18f6f7c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -355,7 +356,8 @@ public void testEnableMigrationWithEmptyPolicies() throws Exception { // 2.set enable migration boolean enableMigrationReq = true; - namespaces.enableMigration(testTenant, enableMigrationGroupNs, enableMigrationReq); + asyncRequests(response -> namespaces.enableMigration(response, testTenant, enableMigrationGroupNs, + enableMigrationReq)); // 3.query namespace num bundles, should be conf.getDefaultNumberOfNamespaceBundles() BundlesData bundlesData = (BundlesData) asyncRequests( @@ -379,7 +381,8 @@ public void testEnableMigrationWithExistBundlePolicies() throws Exception { // 2.set enable migration boolean enableMigrationReq = true; - namespaces.enableMigration(testTenant, enableMigrationGroupNs, enableMigrationReq); + asyncRequests(response -> namespaces.enableMigration(response, testTenant, enableMigrationGroupNs, + enableMigrationReq)); // 3.query namespace num bundles, should be policies.bundles, which we set before BundlesData bundlesData = (BundlesData) asyncRequests( @@ -498,4 +501,22 @@ public void testGetClusterAntiAffinityNamespaces() throws Exception { assertEquals(namespacesResp, namespacesWithFullPath); } + @Test + public void testEnableMigrationAndDisableMigration() throws Exception { + String enableMigrationGroupNs = "test-enable-migration-disable-migration-ns"; + asyncRequests(response -> namespaces.createNamespace(response, testTenant, enableMigrationGroupNs, null)); + + // Enable migration + asyncRequests(response -> namespaces.enableMigration(response, testTenant, enableMigrationGroupNs, true)); + Policies policiesResp = (Policies) asyncRequests( + response -> namespaces.getPolicies(response, testTenant, enableMigrationGroupNs)); + assertTrue(policiesResp.migrated); + + // Disable migration + asyncRequests(response -> namespaces.enableMigration(response, testTenant, enableMigrationGroupNs, false)); + policiesResp = (Policies) asyncRequests( + response -> namespaces.getPolicies(response, testTenant, enableMigrationGroupNs)); + assertFalse(policiesResp.migrated); + } + } From 2d9ce2b45cdb6680036726c943f9bc4def7232fe Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 8 Apr 2026 16:34:47 +0800 Subject: [PATCH 006/201] [fix][broker] Change the schema incompatible log from ERROR to WARN level (#25483) (cherry picked from commit 0b73b3a346c3f6c66107123f73bc098b53b0113d) --- .../service/schema/SchemaRegistryServiceImpl.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index b722a2df25854..a5b6bd884088b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -230,8 +230,12 @@ public CompletableFuture putSchemaIfAbsent(String schemaId, Schem }))).whenComplete((v, ex) -> { var latencyMs = this.clock.millis() - start.getValue(); if (ex != null) { - log.error("[{}] Put schema failed", schemaId, ex); - if (start.getValue() != 0) { + if (ex instanceof IncompatibleSchemaException) { + log.warn("[{}] Put schema failed due to incompatible schema", schemaId, ex); + } else { + log.error("[{}] Put schema failed", schemaId, ex); + } + if (start.longValue() != 0) { this.stats.recordPutFailed(schemaId, latencyMs); } promise.completeExceptionally(ex); @@ -458,7 +462,7 @@ private CompletableFuture checkCompatibilityWithLatest(String schemaId, Sc CompletableFuture result = new CompletableFuture<>(); result.whenComplete((__, t) -> { if (t != null) { - log.error("[{}] Schema is incompatible", schemaId); + log.warn("[{}] Schema is incompatible", schemaId); this.stats.recordSchemaIncompatible(schemaId); } else { if (log.isDebugEnabled()) { @@ -495,7 +499,7 @@ private CompletableFuture checkCompatibilityWithAll(String schemaId, Schem result.whenComplete((v, t) -> { if (t != null) { this.stats.recordSchemaIncompatible(schemaId); - log.error("[{}] Schema is incompatible, schema type {}", schemaId, schema.getType()); + log.warn("[{}] Schema is incompatible, schema type {}", schemaId, schema.getType()); } else { this.stats.recordSchemaCompatible(schemaId); if (log.isDebugEnabled()) { From 927c3e8f2694b74d3b94147ab6cf3d5027996514 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 8 Apr 2026 23:49:31 -0700 Subject: [PATCH 007/201] [fix] Upgrade Jetty to 12.1.6 to fix CVE-2026-1605 (#25485) (cherry picked from commit 5b2778e8676ede45025b8c94b563b8db258c50fc) --- .../server/src/assemble/LICENSE.bin.txt | 80 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 24 +++--- pom.xml | 2 +- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 2304153a287ee..c06300a3b0b8e 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -392,43 +392,43 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.12.4.jar - org.asynchttpclient-async-http-client-netty-utils-2.12.4.jar * Jetty - - org.eclipse.jetty-jetty-alpn-client-12.1.5.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.5.jar - - org.eclipse.jetty-jetty-alpn-server-12.1.5.jar - - org.eclipse.jetty-jetty-annotations-12.1.5.jar - - org.eclipse.jetty-jetty-client-12.1.5.jar - - org.eclipse.jetty-jetty-http-12.1.5.jar - - org.eclipse.jetty-jetty-io-12.1.5.jar - - org.eclipse.jetty-jetty-jndi-12.1.5.jar - - org.eclipse.jetty-jetty-plus-12.1.5.jar - - org.eclipse.jetty-jetty-security-12.1.5.jar - - org.eclipse.jetty-jetty-server-12.1.5.jar - - org.eclipse.jetty-jetty-session-12.1.5.jar - - org.eclipse.jetty-jetty-util-12.1.5.jar - - org.eclipse.jetty-jetty-xml-12.1.5.jar - - org.eclipse.jetty.compression-jetty-compression-common-12.1.5.jar - - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.5.jar - - org.eclipse.jetty.compression-jetty-compression-server-12.1.5.jar - - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.5.jar - - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.5.jar - - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.5.jar - - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.5.jar - - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.5.jar - - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.5.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.5.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.5.jar - - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.5.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.5.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.5.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.5.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.5.jar - - org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.6.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.5.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.5.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.5.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.5.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.5.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.5.jar + - org.eclipse.jetty-jetty-alpn-client-12.1.6.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.6.jar + - org.eclipse.jetty-jetty-alpn-server-12.1.6.jar + - org.eclipse.jetty-jetty-annotations-12.1.6.jar + - org.eclipse.jetty-jetty-client-12.1.6.jar + - org.eclipse.jetty-jetty-http-12.1.6.jar + - org.eclipse.jetty-jetty-io-12.1.6.jar + - org.eclipse.jetty-jetty-jndi-12.1.6.jar + - org.eclipse.jetty-jetty-plus-12.1.6.jar + - org.eclipse.jetty-jetty-security-12.1.6.jar + - org.eclipse.jetty-jetty-server-12.1.6.jar + - org.eclipse.jetty-jetty-session-12.1.6.jar + - org.eclipse.jetty-jetty-util-12.1.6.jar + - org.eclipse.jetty-jetty-xml-12.1.6.jar + - org.eclipse.jetty.compression-jetty-compression-common-12.1.6.jar + - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.6.jar + - org.eclipse.jetty.compression-jetty-compression-server-12.1.6.jar + - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.6.jar + - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.6.jar + - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.6.jar + - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.6.jar + - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.6.jar + - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.6.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.6.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.6.jar + - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.6.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.6.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.6.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.6.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.6.jar + - org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.6.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.6.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.6.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.6.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.6.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.6.jar * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.45.0.jar @@ -570,9 +570,9 @@ BSD 3-clause "New" or "Revised" License * JLine -- jline-jline-2.14.6.jar -- ../licenses/LICENSE-JLine.txt * JLine3 -- org.jline-jline-3.21.0.jar -- ../licenses/LICENSE-JLine.txt * OW2 ASM - - org.ow2.asm-asm-9.9.jar -- ../licenses/LICENSE-ASM.txt - - org.ow2.asm-asm-commons-9.9.jar -- ../licenses/LICENSE-ASM.txt - - org.ow2.asm-asm-tree-9.9.jar -- ../licenses/LICENSE-ASM.txt + - org.ow2.asm-asm-9.9.1.jar -- ../licenses/LICENSE-ASM.txt + - org.ow2.asm-asm-commons-9.9.1.jar -- ../licenses/LICENSE-ASM.txt + - org.ow2.asm-asm-tree-9.9.1.jar -- ../licenses/LICENSE-ASM.txt BSD 2-Clause License * HdrHistogram -- org.hdrhistogram-HdrHistogram-2.1.9.jar -- ../licenses/LICENSE-HdrHistogram.txt diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index ebeee2013ce6b..c3da3a7cc909f 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -401,18 +401,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.4.jar - async-http-client-netty-utils-2.12.4.jar * Jetty - - jetty-alpn-client-12.1.5.jar - - jetty-client-12.1.5.jar - - jetty-compression-common-12.1.5.jar - - jetty-compression-gzip-12.1.5.jar - - jetty-http-12.1.5.jar - - jetty-io-12.1.5.jar - - jetty-util-12.1.5.jar - - jetty-websocket-core-client-12.1.5.jar - - jetty-websocket-core-common-12.1.5.jar - - jetty-websocket-jetty-api-12.1.5.jar - - jetty-websocket-jetty-client-12.1.5.jar - - jetty-websocket-jetty-common-12.1.5.jar + - jetty-alpn-client-12.1.6.jar + - jetty-client-12.1.6.jar + - jetty-compression-common-12.1.6.jar + - jetty-compression-gzip-12.1.6.jar + - jetty-http-12.1.6.jar + - jetty-io-12.1.6.jar + - jetty-util-12.1.6.jar + - jetty-websocket-core-client-12.1.6.jar + - jetty-websocket-core-common-12.1.6.jar + - jetty-websocket-jetty-api-12.1.6.jar + - jetty-websocket-jetty-client-12.1.6.jar + - jetty-websocket-jetty-common-12.1.6.jar * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.45.0.jar * Javassist -- javassist-3.25.0-GA.jar diff --git a/pom.xml b/pom.xml index 2a80854cc2b17..327054dd9c2a5 100644 --- a/pom.xml +++ b/pom.xml @@ -191,7 +191,7 @@ flexible messaging model and an intuitive client API. 5.7.1 4.1.132.Final 0.0.26.Final - 12.1.5 + 12.1.6 9.4.58.v20250814 2.5.2 From 2feaf26ae2b9164f236bfde7e9663f4e69c2b101 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 15 Apr 2026 10:45:02 +0800 Subject: [PATCH 008/201] [improve][broker] Close connection when close consumer write fails (#25520) (cherry picked from commit f688ff748fe49d06ffa0829ce61d51707584f712) --- .../pulsar/broker/service/ServerCnx.java | 17 ++++++++-- .../pulsar/broker/service/ServerCnxTest.java | 33 +++++++++++++++++++ .../pulsar/common/protocol/PulsarHandler.java | 3 +- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index f84f1c080ede3..9a835370f7d48 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -3397,16 +3397,27 @@ private void closeConsumer(long consumerId, Optional assignedB if (getRemoteEndpointProtocolVersion() >= v5.getValue()) { assignedBrokerLookupData.ifPresentOrElse(lookup -> { LookupData lookupData = getLookupData(lookup); - writeAndFlush(Commands.newCloseConsumer(consumerId, -1L, + writeCloseConsumerAndCloseConnectionOnFailure(Commands.newCloseConsumer(consumerId, -1L, lookupData.getBrokerUrl(), - lookupData.getBrokerUrlTls())); + lookupData.getBrokerUrlTls()), consumerId); }, - () -> writeAndFlush(Commands.newCloseConsumer(consumerId, -1L, null, null))); + () -> writeCloseConsumerAndCloseConnectionOnFailure( + Commands.newCloseConsumer(consumerId, -1L, null, null), consumerId)); } else { close(); } } + private void writeCloseConsumerAndCloseConnectionOnFailure(ByteBuf cmd, long consumerId) { + ctx.writeAndFlush(cmd).addListener(future -> { + if (!future.isSuccess()) { + log.warn("[{}] Forcing connection to close since cannot send close consumer command for consumer {}", + remoteAddress, consumerId, future.cause()); + close(); + } + }); + } + /** * It closes the connection with client which triggers {@code channelInactive()} which clears all producers and * consumers from connection-map. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 2151eacd5d63d..bb4b8bbf0e58c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -45,10 +45,14 @@ import static org.testng.Assert.assertTrue; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; import io.netty.channel.DefaultChannelId; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.GenericFutureListener; import io.vertx.core.impl.ConcurrentHashSet; import java.io.Closeable; import java.io.IOException; @@ -2479,6 +2483,35 @@ public void testSubscribeBookieTimeout() throws Exception { channel.finish(); } + @Test + public void testCloseConsumerClosesConnectionWhenWriteFails() throws Exception { + resetChannel(); + setChannelConnected(); + + var ctx = mock(ChannelHandlerContext.class); + var writeFuture = mock(ChannelFuture.class); + var writeFailure = new RuntimeException("close consumer write failed"); + when(ctx.writeAndFlush(any())).thenReturn(writeFuture); + when(writeFuture.isSuccess()).thenReturn(false); + when(writeFuture.cause()).thenReturn(writeFailure); + when(writeFuture.addListener(any())).thenAnswer(invocation -> { + GenericFutureListener> listener = invocation.getArgument(0); + listener.operationComplete(writeFuture); + return writeFuture; + }); + serverCnx.setCtx(ctx); + + var consumer = mock(Consumer.class); + when(consumer.consumerId()).thenReturn(1L); + + serverCnx.setRemoteEndpointProtocolVersion(ProtocolVersion.v12.getValue()); + + serverCnx.closeConsumer(consumer, Optional.empty()); + + verify(ctx).writeAndFlush(any()); + verify(ctx).close(); + } + @Test(timeOut = 30000) public void testSubscribeCommand() throws Exception { final String failSubName = "failSub"; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java index e8010ea1a514f..84a9b819867f9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java @@ -53,7 +53,8 @@ public int getRemoteEndpointProtocolVersion() { return remoteEndpointProtocolVersion; } - protected void setRemoteEndpointProtocolVersion(int remoteEndpointProtocolVersion) { + @VisibleForTesting + public void setRemoteEndpointProtocolVersion(int remoteEndpointProtocolVersion) { this.remoteEndpointProtocolVersion = remoteEndpointProtocolVersion; } From a0420fb218b017c1d560637d063c7f86b92f2b6d Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 15 Apr 2026 22:49:32 +0800 Subject: [PATCH 009/201] [fix][broker] Revert "[improve][broker] Enhance advertised address resolution with fallback to localhost (#25238)" (#25523) (cherry picked from commit 352147bff422a753fd67cf0420ad1b6e7f837e0a) --- .../java/org/apache/pulsar/PulsarStandaloneStarter.java | 9 ++------- .../integration/containers/StandaloneContainer.java | 2 +- .../apache/pulsar/tests/integration/PulsarContainer.java | 2 +- .../apache/pulsar/tests/integration/PulsarContainer.java | 2 +- .../apache/pulsar/tests/integration/PulsarContainer.java | 2 +- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java index 411449d1042cb..29feac8cb46eb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java @@ -29,7 +29,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.ServiceConfigurationUtils; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.docs.tools.CmdGenerateDocs; import picocli.CommandLine; @@ -92,12 +91,8 @@ public PulsarStandaloneStarter(String[] args) throws Exception { // Use advertised address from command line config.setAdvertisedAddress(this.getAdvertisedAddress()); } else if (isBlank(config.getAdvertisedAddress()) && isBlank(config.getAdvertisedListeners())) { - try { - config.setAdvertisedAddress(ServiceConfigurationUtils.unsafeLocalhostResolve()); - } catch (Exception e) { - log.warn("Failed to resolve FQDN, using 'localhost' as advertised address", e); - config.setAdvertisedAddress("localhost"); - } + // Use advertised address as local hostname + config.setAdvertisedAddress("localhost"); } else { // Use advertised or advertisedListeners address from config file } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/StandaloneContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/StandaloneContainer.java index 3bee66adaec36..7e0b347640d59 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/StandaloneContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/StandaloneContainer.java @@ -52,7 +52,7 @@ public StandaloneContainer(String clusterName, String pulsarImageName) { @Override protected void configure() { super.configure(); - setCommand("standalone --advertised-address localhost"); + setCommand("standalone"); addEnv("PULSAR_MEM", "-Xms128M -Xmx1g -XX:MaxDirectMemorySize=1g"); } diff --git a/tests/pulsar-client-admin-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java b/tests/pulsar-client-admin-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java index c945fc369a9bb..365b4cca815b1 100644 --- a/tests/pulsar-client-admin-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java +++ b/tests/pulsar-client-admin-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java @@ -37,7 +37,7 @@ public PulsarContainer() { public PulsarContainer(final String pulsarVersion) { super(pulsarVersion); withExposedPorts(BROKER_HTTP_PORT, PULSAR_PORT); - withCommand("/pulsar/bin/pulsar standalone --advertised-address localhost"); + withCommand("/pulsar/bin/pulsar standalone"); waitingFor(new HttpWaitStrategy() .forPort(BROKER_HTTP_PORT) .forStatusCode(200) diff --git a/tests/pulsar-client-all-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java b/tests/pulsar-client-all-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java index c945fc369a9bb..365b4cca815b1 100644 --- a/tests/pulsar-client-all-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java +++ b/tests/pulsar-client-all-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java @@ -37,7 +37,7 @@ public PulsarContainer() { public PulsarContainer(final String pulsarVersion) { super(pulsarVersion); withExposedPorts(BROKER_HTTP_PORT, PULSAR_PORT); - withCommand("/pulsar/bin/pulsar standalone --advertised-address localhost"); + withCommand("/pulsar/bin/pulsar standalone"); waitingFor(new HttpWaitStrategy() .forPort(BROKER_HTTP_PORT) .forStatusCode(200) diff --git a/tests/pulsar-client-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java b/tests/pulsar-client-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java index c945fc369a9bb..365b4cca815b1 100644 --- a/tests/pulsar-client-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java +++ b/tests/pulsar-client-shade-test/src/test/java/org/apache/pulsar/tests/integration/PulsarContainer.java @@ -37,7 +37,7 @@ public PulsarContainer() { public PulsarContainer(final String pulsarVersion) { super(pulsarVersion); withExposedPorts(BROKER_HTTP_PORT, PULSAR_PORT); - withCommand("/pulsar/bin/pulsar standalone --advertised-address localhost"); + withCommand("/pulsar/bin/pulsar standalone"); waitingFor(new HttpWaitStrategy() .forPort(BROKER_HTTP_PORT) .forStatusCode(200) From 28cd4e396c3d65674f17e008572fd83fdfe18cf8 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 15 Apr 2026 17:54:51 +0300 Subject: [PATCH 010/201] [fix][sec] Upgrade Jetty to address CVE-2026-2332 (#25527) (cherry picked from commit 4c96c730aa2fa7c9b70ded7892a40797c36855e2) --- .../server/src/assemble/LICENSE.bin.txt | 72 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 24 +++---- pom.xml | 2 +- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index c06300a3b0b8e..ad2cd24a64255 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -392,43 +392,43 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.12.4.jar - org.asynchttpclient-async-http-client-netty-utils-2.12.4.jar * Jetty - - org.eclipse.jetty-jetty-alpn-client-12.1.6.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.6.jar - - org.eclipse.jetty-jetty-alpn-server-12.1.6.jar - - org.eclipse.jetty-jetty-annotations-12.1.6.jar - - org.eclipse.jetty-jetty-client-12.1.6.jar - - org.eclipse.jetty-jetty-http-12.1.6.jar - - org.eclipse.jetty-jetty-io-12.1.6.jar - - org.eclipse.jetty-jetty-jndi-12.1.6.jar - - org.eclipse.jetty-jetty-plus-12.1.6.jar - - org.eclipse.jetty-jetty-security-12.1.6.jar - - org.eclipse.jetty-jetty-server-12.1.6.jar - - org.eclipse.jetty-jetty-session-12.1.6.jar - - org.eclipse.jetty-jetty-util-12.1.6.jar - - org.eclipse.jetty-jetty-xml-12.1.6.jar - - org.eclipse.jetty.compression-jetty-compression-common-12.1.6.jar - - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.6.jar - - org.eclipse.jetty.compression-jetty-compression-server-12.1.6.jar - - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.6.jar - - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.6.jar - - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.6.jar - - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.6.jar - - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.6.jar - - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.6.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.6.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.6.jar - - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.6.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.6.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.6.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.6.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.6.jar + - org.eclipse.jetty-jetty-alpn-client-12.1.7.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.7.jar + - org.eclipse.jetty-jetty-alpn-server-12.1.7.jar + - org.eclipse.jetty-jetty-annotations-12.1.7.jar + - org.eclipse.jetty-jetty-client-12.1.7.jar + - org.eclipse.jetty-jetty-http-12.1.7.jar + - org.eclipse.jetty-jetty-io-12.1.7.jar + - org.eclipse.jetty-jetty-jndi-12.1.7.jar + - org.eclipse.jetty-jetty-plus-12.1.7.jar + - org.eclipse.jetty-jetty-security-12.1.7.jar + - org.eclipse.jetty-jetty-server-12.1.7.jar + - org.eclipse.jetty-jetty-session-12.1.7.jar + - org.eclipse.jetty-jetty-util-12.1.7.jar + - org.eclipse.jetty-jetty-xml-12.1.7.jar + - org.eclipse.jetty.compression-jetty-compression-common-12.1.7.jar + - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.7.jar + - org.eclipse.jetty.compression-jetty-compression-server-12.1.7.jar + - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.7.jar + - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.7.jar + - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.7.jar + - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.7.jar + - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.7.jar + - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.7.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.7.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.7.jar + - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.7.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.7.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.7.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.7.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.7.jar - org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.6.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.6.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.6.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.6.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.6.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.6.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.7.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.7.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.7.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.7.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.7.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.7.jar * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.45.0.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index c3da3a7cc909f..cfe03c118d477 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -401,18 +401,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.4.jar - async-http-client-netty-utils-2.12.4.jar * Jetty - - jetty-alpn-client-12.1.6.jar - - jetty-client-12.1.6.jar - - jetty-compression-common-12.1.6.jar - - jetty-compression-gzip-12.1.6.jar - - jetty-http-12.1.6.jar - - jetty-io-12.1.6.jar - - jetty-util-12.1.6.jar - - jetty-websocket-core-client-12.1.6.jar - - jetty-websocket-core-common-12.1.6.jar - - jetty-websocket-jetty-api-12.1.6.jar - - jetty-websocket-jetty-client-12.1.6.jar - - jetty-websocket-jetty-common-12.1.6.jar + - jetty-alpn-client-12.1.7.jar + - jetty-client-12.1.7.jar + - jetty-compression-common-12.1.7.jar + - jetty-compression-gzip-12.1.7.jar + - jetty-http-12.1.7.jar + - jetty-io-12.1.7.jar + - jetty-util-12.1.7.jar + - jetty-websocket-core-client-12.1.7.jar + - jetty-websocket-core-common-12.1.7.jar + - jetty-websocket-jetty-api-12.1.7.jar + - jetty-websocket-jetty-client-12.1.7.jar + - jetty-websocket-jetty-common-12.1.7.jar * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.45.0.jar * Javassist -- javassist-3.25.0-GA.jar diff --git a/pom.xml b/pom.xml index 327054dd9c2a5..8f5ab369d18f6 100644 --- a/pom.xml +++ b/pom.xml @@ -191,7 +191,7 @@ flexible messaging model and an intuitive client API. 5.7.1 4.1.132.Final 0.0.26.Final - 12.1.6 + 12.1.7 9.4.58.v20250814 2.5.2 From 7fd11c55b861c55a645fd145d70a9c49463c5fa0 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 15 Apr 2026 18:54:44 +0300 Subject: [PATCH 011/201] [cleanup][build] Bumped version to 4.2.1-SNAPSHOT Signed-off-by: Lari Hotari --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- jetcd-core-shaded/pom.xml | 2 +- jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml | 2 +- jetty-upgrade/pom.xml | 2 +- jetty-upgrade/zookeeper-prometheus-metrics/pom.xml | 2 +- jetty-upgrade/zookeeper-with-patched-admin/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- microbench/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-bom/pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-cli-utils/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-dependencies-minimized/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-docs-tools/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/azure-data-explorer/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis-kpl-shaded/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-opentelemetry/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 135 files changed, 138 insertions(+), 138 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 3592df42f4f5c..7b4538d1aa3fd 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.0 + 4.2.1-SNAPSHOT bouncy-castle-bc diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 4088d7fbf760f..fac2c9def607d 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.0 + 4.2.1-SNAPSHOT bcfips-include-test diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 5511d63822e75..b07fbe49221e1 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.0 + 4.2.1-SNAPSHOT bouncy-castle-bcfips diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 95c20f2edd6c1..e85913fc1976f 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.0 + 4.2.1-SNAPSHOT diff --git a/buildtools/pom.xml b/buildtools/pom.xml index c891a628b4ea0..2440826767827 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 4.2.0 + 4.2.1-SNAPSHOT jar Pulsar Build Tools @@ -43,7 +43,7 @@ - 2026-03-24T17:03:02Z + 2026-04-15T15:54:00Z 1.8 1.8 8 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 6ffa58ce8eb49..e27ac2f154bb3 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.0 + 4.2.1-SNAPSHOT pulsar-io-distribution diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 26b377bdc67e2..80f523d9ea207 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.0 + 4.2.1-SNAPSHOT pulsar-offloader-distribution diff --git a/distribution/pom.xml b/distribution/pom.xml index 7c602833fb7ba..0b9c17ec902bc 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.0 + 4.2.1-SNAPSHOT distribution diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index cac6b8ff2ed4d..3422b25323e33 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.0 + 4.2.1-SNAPSHOT pulsar-server-distribution diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 9c2c22dab301f..e68805857a59b 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.0 + 4.2.1-SNAPSHOT pulsar-shell-distribution diff --git a/docker/pom.xml b/docker/pom.xml index 5889db2968171..fe8fbe27b9189 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.0 + 4.2.1-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 84fc39a77d973..e42af25f2fc6d 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 0b4a345771901..d8cd369156090 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.0 + 4.2.1-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 5b96d9e6d813d..0c39663aae3f4 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.0 + 4.2.1-SNAPSHOT jclouds-shaded diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml index 6651614e0d060..aad21693fe241 100644 --- a/jetcd-core-shaded/pom.xml +++ b/jetcd-core-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.0 + 4.2.1-SNAPSHOT jetcd-core-shaded diff --git a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml index 5a44e6563e0fa..cfa40d1acef45 100644 --- a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml +++ b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.0 + 4.2.1-SNAPSHOT pulsar-bookkeeper-prometheus-metrics-provider Apache Pulsar :: BookKeeper Stats Providers :: Prometheus diff --git a/jetty-upgrade/pom.xml b/jetty-upgrade/pom.xml index 8889ac3625237..ef7bb6d489ef9 100644 --- a/jetty-upgrade/pom.xml +++ b/jetty-upgrade/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.0 + 4.2.1-SNAPSHOT jetty-upgrade diff --git a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml index 32dadb8d96124..376c5a3779aa5 100755 --- a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml +++ b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.0 + 4.2.1-SNAPSHOT pulsar-zookeeper-prometheus-metrics diff --git a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml index 6061e690330be..f5cf6273e79ab 100644 --- a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml +++ b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar jetty-upgrade - 4.2.0 + 4.2.1-SNAPSHOT zookeeper-with-patched-admin diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index b80620b706123..22e2b9cedff5a 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.0 + 4.2.1-SNAPSHOT managed-ledger diff --git a/microbench/pom.xml b/microbench/pom.xml index 4ed1580e9acb9..97675fc284b7a 100644 --- a/microbench/pom.xml +++ b/microbench/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.0 + 4.2.1-SNAPSHOT microbench diff --git a/pom.xml b/pom.xml index 8f5ab369d18f6..cb274cd465118 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 4.2.0 + 4.2.1-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -94,7 +94,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - 2026-03-24T17:02:57Z + 2026-04-15T15:53:54Z true 1.78.1 1.81 From ab397b1f08642c8281a1e75dc9d6bedda5bb5a93 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 16 Apr 2026 01:26:27 +0300 Subject: [PATCH 015/201] [fix][sec] Upgrade Jetty to address CVE-2026-5795 (#25532) (cherry picked from commit e05c2129aad5a8620b1fda88ee2a5f79bc86a365) --- .../server/src/assemble/LICENSE.bin.txt | 72 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 24 +++---- pom.xml | 2 +- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 79fcdb0e9db3c..4698da389ce0c 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -392,43 +392,43 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.12.4.jar - org.asynchttpclient-async-http-client-netty-utils-2.12.4.jar * Jetty - - org.eclipse.jetty-jetty-alpn-client-12.1.7.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.7.jar - - org.eclipse.jetty-jetty-alpn-server-12.1.7.jar - - org.eclipse.jetty-jetty-annotations-12.1.7.jar - - org.eclipse.jetty-jetty-client-12.1.7.jar - - org.eclipse.jetty-jetty-http-12.1.7.jar - - org.eclipse.jetty-jetty-io-12.1.7.jar - - org.eclipse.jetty-jetty-jndi-12.1.7.jar - - org.eclipse.jetty-jetty-plus-12.1.7.jar - - org.eclipse.jetty-jetty-security-12.1.7.jar - - org.eclipse.jetty-jetty-server-12.1.7.jar - - org.eclipse.jetty-jetty-session-12.1.7.jar - - org.eclipse.jetty-jetty-util-12.1.7.jar - - org.eclipse.jetty-jetty-xml-12.1.7.jar - - org.eclipse.jetty.compression-jetty-compression-common-12.1.7.jar - - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.7.jar - - org.eclipse.jetty.compression-jetty-compression-server-12.1.7.jar - - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.7.jar - - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.7.jar - - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.7.jar - - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.7.jar - - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.7.jar - - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.7.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.7.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.7.jar - - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.7.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.7.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.7.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.7.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.7.jar + - org.eclipse.jetty-jetty-alpn-client-12.1.8.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.8.jar + - org.eclipse.jetty-jetty-alpn-server-12.1.8.jar + - org.eclipse.jetty-jetty-annotations-12.1.8.jar + - org.eclipse.jetty-jetty-client-12.1.8.jar + - org.eclipse.jetty-jetty-http-12.1.8.jar + - org.eclipse.jetty-jetty-io-12.1.8.jar + - org.eclipse.jetty-jetty-jndi-12.1.8.jar + - org.eclipse.jetty-jetty-plus-12.1.8.jar + - org.eclipse.jetty-jetty-security-12.1.8.jar + - org.eclipse.jetty-jetty-server-12.1.8.jar + - org.eclipse.jetty-jetty-session-12.1.8.jar + - org.eclipse.jetty-jetty-util-12.1.8.jar + - org.eclipse.jetty-jetty-xml-12.1.8.jar + - org.eclipse.jetty.compression-jetty-compression-common-12.1.8.jar + - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.8.jar + - org.eclipse.jetty.compression-jetty-compression-server-12.1.8.jar + - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.8.jar + - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.8.jar + - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.8.jar + - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.8.jar + - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.8.jar + - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.8.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.8.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.8.jar + - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.8.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.8.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.8.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.8.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.8.jar - org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.7.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.7.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.7.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.7.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.7.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.7.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.8.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.8.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.8.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.8.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.8.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.8.jar * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.45.0.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index a6b14a4cb363a..1846799c30249 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -401,18 +401,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.4.jar - async-http-client-netty-utils-2.12.4.jar * Jetty - - jetty-alpn-client-12.1.7.jar - - jetty-client-12.1.7.jar - - jetty-compression-common-12.1.7.jar - - jetty-compression-gzip-12.1.7.jar - - jetty-http-12.1.7.jar - - jetty-io-12.1.7.jar - - jetty-util-12.1.7.jar - - jetty-websocket-core-client-12.1.7.jar - - jetty-websocket-core-common-12.1.7.jar - - jetty-websocket-jetty-api-12.1.7.jar - - jetty-websocket-jetty-client-12.1.7.jar - - jetty-websocket-jetty-common-12.1.7.jar + - jetty-alpn-client-12.1.8.jar + - jetty-client-12.1.8.jar + - jetty-compression-common-12.1.8.jar + - jetty-compression-gzip-12.1.8.jar + - jetty-http-12.1.8.jar + - jetty-io-12.1.8.jar + - jetty-util-12.1.8.jar + - jetty-websocket-core-client-12.1.8.jar + - jetty-websocket-core-common-12.1.8.jar + - jetty-websocket-jetty-api-12.1.8.jar + - jetty-websocket-jetty-client-12.1.8.jar + - jetty-websocket-jetty-common-12.1.8.jar * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.45.0.jar * Javassist -- javassist-3.25.0-GA.jar diff --git a/pom.xml b/pom.xml index 59c5105c7cdf5..f06a774f3d9dc 100644 --- a/pom.xml +++ b/pom.xml @@ -191,7 +191,7 @@ flexible messaging model and an intuitive client API. 5.7.1 4.1.132.Final 0.0.26.Final - 12.1.7 + 12.1.8 9.4.58.v20250814 2.5.2 From 6d8533981813cd5c14facd8bd1e6186ce2fb1ff9 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 16 Apr 2026 12:11:20 +0300 Subject: [PATCH 016/201] [cleanup][ci] Remove ready-to-test label enforcement (#25470) (cherry picked from commit 865e276510bb5ab0a55d0d89aa3a927203e70b10) --- .github/PULL_REQUEST_TEMPLATE.md | 26 -------- .github/workflows/ci-go-functions.yaml | 8 --- .github/workflows/pulsar-ci-flaky.yaml | 8 --- .github/workflows/pulsar-ci.yaml | 8 --- build/pulsar_ci_tool.sh | 91 -------------------------- 5 files changed, 141 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c148e8f61da14..01bb7fe0476ec 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -70,29 +70,3 @@ This change added tests and can be verified as follows: - [ ] The admin CLI options - [ ] The metrics - [ ] Anything that affects deployment - -### Documentation - - - -- [ ] `doc` -- [ ] `doc-required` -- [ ] `doc-not-needed` -- [ ] `doc-complete` - -### Matching PR in forked repository - -PR in forked repository: - - diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index d3a1da2fa4de2..44dc55f567073 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -60,14 +60,6 @@ jobs: echo docs_only=false >> $GITHUB_OUTPUT fi - - name: Check if the PR has been approved for testing - if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} - env: - GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - GITHUB_TOKEN: ${{ github.token }} - run: | - build/pulsar_ci_tool.sh check_ready_to_test - check-style: needs: preconditions if: ${{ needs.preconditions.outputs.docs_only != 'true' }} diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index cc2174d931247..2d35d63c2713b 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -153,14 +153,6 @@ jobs: github.event_name == 'workflow_dispatch' && github.event.inputs.netty_leak_detection || 'report' }}" >> $GITHUB_OUTPUT - - name: Check if the PR has been approved for testing - if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} - env: - GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - GITHUB_TOKEN: ${{ github.token }} - run: | - build/pulsar_ci_tool.sh check_ready_to_test - build-and-test: needs: preconditions name: Flaky tests suite diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 03e83e11a5bc9..f8cb682896c08 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -201,14 +201,6 @@ jobs: github.event_name == 'workflow_dispatch' && github.event.inputs.netty_leak_detection || 'report' }}" >> $GITHUB_OUTPUT - - name: Check if the PR has been approved for testing - if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} - env: - GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - GITHUB_TOKEN: ${{ github.token }} - run: | - build/pulsar_ci_tool.sh check_ready_to_test - build-and-license-check: needs: preconditions name: Build and License check diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh index ac2b9173af0c0..672289d3ac5c8 100755 --- a/build/pulsar_ci_tool.sh +++ b/build/pulsar_ci_tool.sh @@ -170,97 +170,6 @@ function ci_move_test_reports() { ) } -function ci_check_ready_to_test() { - if [[ -z "$GITHUB_EVENT_PATH" ]]; then - >&2 echo "GITHUB_EVENT_PATH isn't set" - return 1 - fi - - PR_JSON_URL=$(jq -r '.pull_request.url' "${GITHUB_EVENT_PATH}") - echo "Refreshing $PR_JSON_URL..." - PR_JSON=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "${PR_JSON_URL}") - - if printf "%s" "${PR_JSON}" | jq -e '.draft | select(. == true)' &> /dev/null; then - echo "PR is draft." - elif ! ( printf "%s" "${PR_JSON}" | jq -e '.mergeable | select(. == true)' &> /dev/null ); then - echo "PR isn't mergeable." - else - # check ready-to-test label - if printf "%s" "${PR_JSON}" | jq -e '.labels[] | .name | select(. == "ready-to-test")' &> /dev/null; then - echo "Found ready-to-test label." - return 0 - else - echo "There is no ready-to-test label on the PR." - fi - - # check if the PR has been approved - PR_NUM=$(jq -r '.pull_request.number' "${GITHUB_EVENT_PATH}") - REPO_FULL_NAME=$(jq -r '.repository.full_name' "${GITHUB_EVENT_PATH}") - REPO_NAME=$(basename "${REPO_FULL_NAME}") - REPO_OWNER=$(dirname "${REPO_FULL_NAME}") - # use graphql query to find out reviewDecision - PR_REVIEW_DECISION=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" -X POST -d '{"query": "query { repository(name: \"'${REPO_NAME}'\", owner: \"'${REPO_OWNER}'\") { pullRequest(number: '${PR_NUM}') { reviewDecision } } }"}' https://api.github.com/graphql |jq -r '.data.repository.pullRequest.reviewDecision') - echo "Review decision for PR #${PR_NUM} in repository ${REPO_OWNER}/${REPO_NAME} is ${PR_REVIEW_DECISION}" - if [[ "$PR_REVIEW_DECISION" == "APPROVED" ]]; then - return 0 - fi - fi - - FORK_REPO_URL=$(jq -r '.pull_request.head.repo.html_url' "$GITHUB_EVENT_PATH") - PR_BRANCH_LABEL=$(jq -r '.pull_request.head.label' "$GITHUB_EVENT_PATH") - PR_BASE_BRANCH=$(jq -r '.pull_request.base.ref' "$GITHUB_EVENT_PATH") - PR_URL=$(jq -r '.pull_request.html_url' "$GITHUB_EVENT_PATH") - FORK_PR_TITLE_URL_ENCODED=$(printf "%s" "${PR_JSON}" | jq -r '"[run-tests] " + .title | @uri') - FORK_PR_BODY_URL_ENCODED=$(jq -n -r "\"This PR is for running tests for upstream PR ${PR_URL}.\n\n\" | @uri") - if [[ "$PR_BASE_BRANCH" != "master" ]]; then - sync_non_master_fork_docs=$(cat <&2 tee -a "$GITHUB_STEP_SUMMARY" < Date: Tue, 7 Apr 2026 05:35:39 -0700 Subject: [PATCH 017/201] [cleanup][ci] Remove documentation label bot (#25469) Co-authored-by: Lari Hotari (cherry picked from commit 20a380af2fa53d12b6f8d661486ce8742bb40956) --- .github/workflows/ci-documentbot.yml | 46 ---------------------------- 1 file changed, 46 deletions(-) delete mode 100644 .github/workflows/ci-documentbot.yml diff --git a/.github/workflows/ci-documentbot.yml b/.github/workflows/ci-documentbot.yml deleted file mode 100644 index 98e86eb6a07ad..0000000000000 --- a/.github/workflows/ci-documentbot.yml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Documentation Bot - -on: - pull_request_target : - types: - - opened - - edited - - labeled - - unlabeled - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.number }} - cancel-in-progress: true - -jobs: - label: - if: (github.repository == 'apache/pulsar') && (github.event.pull_request.state == 'open') - permissions: - pull-requests: write - runs-on: ubuntu-24.04 - steps: - - name: Labeling - uses: apache/pulsar-test-infra/docbot@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - LABEL_WATCH_LIST: 'doc,doc-required,doc-not-needed,doc-complete' - LABEL_MISSING: 'doc-label-missing' \ No newline at end of file From 52d4ac1d3035c84fe18b97292a5321c9d48d58bc Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 16 Apr 2026 13:37:36 +0300 Subject: [PATCH 018/201] [fix][build][branch-4.2] Use correct Jetty ee8 BOM coordinates --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f06a774f3d9dc..71368671658b3 100644 --- a/pom.xml +++ b/pom.xml @@ -1744,7 +1744,7 @@ flexible messaging model and an intuitive client API. org.eclipse.jetty.ee8 - jetty-ee8 + jetty-ee8-bom ${jetty.version} pom import From 5736b60085f538b74c0236904d1c39959f35ce48 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 17 Apr 2026 13:39:15 +0300 Subject: [PATCH 019/201] [fix][sec] Upgrade to async-http-client 2.14.5 to address CVE-2026-40490 (#25546) (cherry picked from commit a1613bc2e5fd26cc16fc95b4a3c61bc5e1ae090d) --- .../server/src/assemble/LICENSE.bin.txt | 6 +- .../shell/src/assemble/LICENSE.bin.txt | 6 +- pom.xml | 2 +- .../internal/http/AsyncHttpConnectorTest.java | 56 +++++++ pulsar-client/pom.xml | 6 + .../apache/pulsar/client/impl/HttpClient.java | 89 +++++++++-- .../pulsar/client/impl/HttpClientTest.java | 150 ++++++++++++++++++ .../client/impl/HttpLookupServiceTest.java | 142 +++++++++++++++++ 8 files changed, 435 insertions(+), 22 deletions(-) create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/HttpClientTest.java create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/HttpLookupServiceTest.java diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 4698da389ce0c..6645d0d5392fe 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -389,8 +389,8 @@ The Apache Software License, Version 2.0 * AirCompressor - io.airlift-aircompressor-2.0.3.jar * AsyncHttpClient - - org.asynchttpclient-async-http-client-2.12.4.jar - - org.asynchttpclient-async-http-client-netty-utils-2.12.4.jar + - org.asynchttpclient-async-http-client-2.14.5.jar + - org.asynchttpclient-async-http-client-netty-utils-2.14.5.jar * Jetty - org.eclipse.jetty-jetty-alpn-client-12.1.8.jar - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.8.jar @@ -628,7 +628,7 @@ Eclipse Public License - v2.0 -- ../licenses/LICENSE-EPL-2.0.txt * Jakarta Transactions API -- jakarta.transaction-jakarta.transaction-api-1.3.3.jar Public Domain (CC0) -- ../licenses/LICENSE-CC0.txt - * Reactive Streams -- org.reactivestreams-reactive-streams-1.0.3.jar + * Reactive Streams -- org.reactivestreams-reactive-streams-1.0.4.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 1846799c30249..b86a20eb9f3c6 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -398,8 +398,8 @@ The Apache Software License, Version 2.0 * AirCompressor - aircompressor-2.0.3.jar * AsyncHttpClient - - async-http-client-2.12.4.jar - - async-http-client-netty-utils-2.12.4.jar + - async-http-client-2.14.5.jar + - async-http-client-netty-utils-2.14.5.jar * Jetty - jetty-alpn-client-12.1.8.jar - jetty-client-12.1.8.jar @@ -463,7 +463,7 @@ Eclipse Public License - v2.0 -- ../licenses/LICENSE-EPL-2.0.txt * Jakarta Injection -- jakarta.inject-2.6.1.jar Public Domain (CC0) -- ../licenses/LICENSE-CC0.txt - * Reactive Streams -- reactive-streams-1.0.3.jar + * Reactive Streams -- reactive-streams-1.0.4.jar diff --git a/pom.xml b/pom.xml index 71368671658b3..c505d2e6f253d 100644 --- a/pom.xml +++ b/pom.xml @@ -263,7 +263,7 @@ flexible messaging model and an intuitive client API. 0.16.1 8.1.1 2.0.3 - 2.12.4 + 2.14.5 3.19.0 2.21.0 1.20.0 diff --git a/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorTest.java b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorTest.java index e0b4b13a03dae..38243976f5698 100644 --- a/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorTest.java +++ b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorTest.java @@ -19,7 +19,9 @@ package org.apache.pulsar.client.admin.internal.http; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.post; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static org.testng.Assert.assertEquals; @@ -282,6 +284,60 @@ private void doTestRedirect(String location) throws InterruptedException, Execut assertEquals(response.getResponseBody(), "OK"); } + /** + * Locks in that AsyncHttpConnector forwards the {@code Authorization} header across a cross-origin + * HTTP redirect (different host:port — serverA → serverB). The admin connector disables AHC's + * built-in follow-redirect and runs its own redirect loop that copies the original request headers, + * so it must remain unaffected by the async-http-client >= 2.14.5 {@code Authorization} stripping + * on cross-origin redirects (CVE-2026-40490 fix). Regressing {@code setFollowRedirect(true)} would + * break this test. + */ + @Test + void testAuthorizationHeaderOnCrossOriginRedirect() throws ExecutionException, InterruptedException { + WireMockServer serverB = new WireMockServer(WireMockConfiguration.wireMockConfig().port(0)); + serverB.start(); + try { + server.stubFor(get(urlEqualTo("/admin/v2/clusters")) + .willReturn(aResponse() + .withStatus(307) + .withHeader("Location", + "http://127.0.0.1:" + serverB.port() + "/admin/v2/clusters"))); + + serverB.stubFor(get(urlEqualTo("/admin/v2/clusters")) + .atPriority(2) + .willReturn(aResponse().withStatus(401).withBody("missing auth"))); + serverB.stubFor(get(urlEqualTo("/admin/v2/clusters")) + .atPriority(1) + .withHeader("Authorization", equalTo("Bearer test-token")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody("[\"test-cluster\"]"))); + + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setServiceUrl("http://127.0.0.1:" + server.port()); + + @Cleanup + AsyncHttpConnector connector = new AsyncHttpConnector(5000, 5000, + 5000, 0, conf, false, null); + + Request request = new RequestBuilder("GET") + .setUrl("http://127.0.0.1:" + server.port() + "/admin/v2/clusters") + .addHeader("Authorization", "Bearer test-token") + .build(); + + Response response = connector.executeRequest(request).get(); + + assertEquals(response.getStatusCode(), 200, + "cross-origin redirect should forward Authorization and return the stubbed body"); + assertEquals(response.getResponseBody(), "[\"test-cluster\"]"); + serverB.verify(getRequestedFor(urlEqualTo("/admin/v2/clusters")) + .withHeader("Authorization", equalTo("Bearer test-token"))); + } finally { + serverB.stop(); + } + } + @Test void testRedirectWithBody() throws ExecutionException, InterruptedException { server.stubFor(post(urlEqualTo("/path1")) diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 94e2f21f85bb0..f6ccb1e166466 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -228,6 +228,12 @@ fastutil + + com.github.tomakehurst + wiremock-jre8-standalone + ${wiremock.version} + test + diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java index 51b267699c2c6..e02748d4fec1a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java @@ -56,6 +56,7 @@ import org.asynchttpclient.DefaultAsyncHttpClient; import org.asynchttpclient.DefaultAsyncHttpClientConfig; import org.asynchttpclient.Request; +import org.asynchttpclient.Response; import org.asynchttpclient.SslEngineFactory; import org.asynchttpclient.channel.DefaultKeepAliveStrategy; @@ -88,7 +89,12 @@ protected HttpClient(ClientConfigurationData conf, EventLoopGroup eventLoopGroup DefaultAsyncHttpClientConfig.Builder confBuilder = new DefaultAsyncHttpClientConfig.Builder(); confBuilder.setCookieStore(null); confBuilder.setUseProxyProperties(true); - confBuilder.setFollowRedirect(true); + // Follow redirects manually in executeGet(...) so we can re-invoke authentication per hop and + // carry the Authorization header across cross-origin redirects. async-http-client >= 2.14.5 + // (CVE-2026-40490 fix) strips the Authorization header when it follows redirects itself; Pulsar + // HTTP lookups routinely redirect to another broker's httpUrl/httpUrlTls which is a different + // host/port, i.e. cross-origin. + confBuilder.setFollowRedirect(false); confBuilder.setMaxRedirects(conf.getMaxLookupRedirects()); confBuilder.setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_IN_SECONDS * 1000); confBuilder.setReadTimeout(DEFAULT_READ_TIMEOUT_IN_SECONDS * 1000); @@ -168,10 +174,28 @@ public CompletableFuture get(String path, Class clazz) { try { URI hostUri = serviceNameResolver.resolveHostUri(); String requestUrl = new URL(hostUri.toURL(), path).toString(); - String remoteHostName = hostUri.getHost(); + InetSocketAddress originalHost = InetSocketAddress.createUnresolved(hostUri.getHost(), hostUri.getPort()); + executeGet(requestUrl, originalHost, clientConf.getMaxLookupRedirects(), future, clazz); + } catch (Exception e) { + log.warn("[{}] Failed to initiate HTTP get request: {}", path, e.getMessage()); + if (e instanceof PulsarClientException) { + future.completeExceptionally(e); + } else { + future.completeExceptionally(new PulsarClientException(e)); + } + } + + return future; + } + + private void executeGet(String requestUrl, InetSocketAddress originalHost, + int redirectsRemaining, CompletableFuture future, Class clazz) { + try { + URI currentUri = URI.create(requestUrl); + String remoteHostName = currentUri.getHost(); AuthenticationDataProvider authData = authentication.getAuthData(remoteHostName); - CompletableFuture> authFuture = new CompletableFuture<>(); + CompletableFuture> authFuture = new CompletableFuture<>(); // bring a authenticationStage for sasl auth. if (authData.hasDataForHttp()) { @@ -180,18 +204,15 @@ public CompletableFuture get(String path, Class clazz) { authFuture.complete(null); } - // auth complete, do real request authFuture.whenComplete((respHeaders, ex) -> { if (ex != null) { - serviceNameResolver.markHostAvailability( - InetSocketAddress.createUnresolved(hostUri.getHost(), hostUri.getPort()), false); + serviceNameResolver.markHostAvailability(originalHost, false); log.warn("[{}] Failed to perform http request at authentication stage: {}", requestUrl, ex.getMessage()); future.completeExceptionally(new PulsarClientException(ex)); return; } - // auth complete, use a new builder BoundRequestBuilder builder = httpClient.prepareGet(requestUrl) // share the DNS resolver and cache with Pulsar client .setNameResolver(nameResolver) @@ -218,17 +239,22 @@ public CompletableFuture get(String path, Class clazz) { builder.execute().toCompletableFuture().whenComplete((response2, t) -> { if (t != null) { - serviceNameResolver.markHostAvailability( - InetSocketAddress.createUnresolved(hostUri.getHost(), hostUri.getPort()), false); + serviceNameResolver.markHostAvailability(originalHost, false); log.warn("[{}] Failed to perform http request: {}", requestUrl, t.getMessage()); future.completeExceptionally(new PulsarClientException(t)); return; } - serviceNameResolver.markHostAvailability( - InetSocketAddress.createUnresolved(hostUri.getHost(), hostUri.getPort()), true); + serviceNameResolver.markHostAvailability(originalHost, true); + + int statusCode = response2.getStatusCode(); + if (isRedirectStatusCode(statusCode)) { + handleRedirect(requestUrl, currentUri, response2, originalHost, + redirectsRemaining, future, clazz); + return; + } // request not success - if (response2.getStatusCode() != HttpURLConnection.HTTP_OK) { + if (statusCode != HttpURLConnection.HTTP_OK) { String errorReason = response2.getStatusText(); if ("application/json".equals(response2.getContentType()) || "text/json".equals( response2.getContentType())) { @@ -250,7 +276,7 @@ public CompletableFuture get(String path, Class clazz) { } log.warn("[{}] HTTP get request failed: {}", requestUrl, errorReason); Exception e; - if (response2.getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) { + if (statusCode == HttpURLConnection.HTTP_NOT_FOUND) { e = new NotFoundException("Not found: " + errorReason); } else { e = new PulsarClientException("HTTP get request failed: " + errorReason); @@ -270,15 +296,48 @@ public CompletableFuture get(String path, Class clazz) { }); }); } catch (Exception e) { - log.warn("[{}]PulsarClientImpl: {}", path, e.getMessage()); + log.warn("[{}] HTTP request setup failed: {}", requestUrl, e.getMessage()); if (e instanceof PulsarClientException) { future.completeExceptionally(e); } else { future.completeExceptionally(new PulsarClientException(e)); } } + } - return future; + private void handleRedirect(String requestUrl, URI currentUri, + Response response, + InetSocketAddress originalHost, int redirectsRemaining, + CompletableFuture future, Class clazz) { + String location = response.getHeader("Location"); + if (location == null || location.isEmpty()) { + future.completeExceptionally(new PulsarClientException( + "HTTP redirect " + response.getStatusCode() + " without Location header: " + requestUrl)); + return; + } + if (redirectsRemaining <= 0) { + future.completeExceptionally(new PulsarClientException( + "Maximum redirects exceeded (" + clientConf.getMaxLookupRedirects() + + ") while following HTTP redirect for " + requestUrl)); + return; + } + String newUrl; + try { + newUrl = currentUri.resolve(location).toString(); + } catch (Exception e) { + future.completeExceptionally(new PulsarClientException( + "Invalid redirect Location \"" + location + "\" for " + requestUrl)); + return; + } + executeGet(newUrl, originalHost, redirectsRemaining - 1, future, clazz); + } + + private static boolean isRedirectStatusCode(int statusCode) { + return statusCode == HttpURLConnection.HTTP_MOVED_PERM // 301 + || statusCode == HttpURLConnection.HTTP_MOVED_TEMP // 302 + || statusCode == HttpURLConnection.HTTP_SEE_OTHER // 303 + || statusCode == 307 // Temporary Redirect + || statusCode == 308; // Permanent Redirect } protected PulsarSslConfiguration buildSslConfiguration(ClientConfigurationData config, String host) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/HttpClientTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/HttpClientTest.java new file mode 100644 index 0000000000000..ec3c8b0b78ecc --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/HttpClientTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.resolver.NameResolver; +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timer; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.net.InetAddress; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.lookup.data.LookupData; +import org.apache.pulsar.common.util.netty.DnsResolverUtil; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Verifies that {@link HttpClient} carries the {@code Authorization} header across cross-origin HTTP redirects. + * + *

Pulsar's HTTP lookup endpoint returns {@code 307 Temporary Redirect} to whichever broker owns the bundle for a + * topic. The redirect target is that broker's {@code httpUrl}/{@code httpUrlTls}, i.e. typically a different host or + * port from the original request. Auth plugins ({@code AuthenticationToken}, {@code AuthenticationBasic}, + * {@code AuthenticationOAuth2}, {@code AuthenticationAthenz}) inject the {@code Authorization} header — that header + * must reach the redirect target for lookup to succeed. + * + *

async-http-client 2.14.5 strips {@code Authorization} on cross-origin redirects when its built-in follow-redirect + * is enabled (CVE-2026-40490 fix). This test drives two WireMock servers on different ports to exercise that path. + */ +public class HttpClientTest { + + private static final String LOOKUP_PATH = "/lookup/v2/topic/persistent/public/default/test-topic"; + private static final String EXPECTED_BODY = "{\"brokerUrl\":\"pulsar://broker-b:6650\"," + + "\"brokerUrlTls\":\"pulsar+ssl://broker-b:6651\"," + + "\"httpUrl\":\"http://broker-b:8080\"," + + "\"httpUrlTls\":\"https://broker-b:8443\"," + + "\"nativeUrl\":\"pulsar://broker-b:6650\"}"; + + private WireMockServer serverA; + private WireMockServer serverB; + private EventLoopGroup eventLoopGroup; + private Timer timer; + private NameResolver nameResolver; + + @BeforeClass(alwaysRun = true) + void beforeClass() { + eventLoopGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("HttpClientTest")); + timer = new HashedWheelTimer(new DefaultThreadFactory("HttpClientTest-timer")); + // Default JDK-backed resolver is sufficient; we only hit 127.0.0.1. + nameResolver = DnsResolverUtil.adaptToNameResolver(null); + } + + @AfterClass(alwaysRun = true) + void afterClass() { + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully(); + } + if (timer != null) { + timer.stop(); + } + } + + @BeforeMethod(alwaysRun = true) + void beforeMethod() { + serverA = new WireMockServer(WireMockConfiguration.wireMockConfig().port(0)); + serverB = new WireMockServer(WireMockConfiguration.wireMockConfig().port(0)); + serverA.start(); + serverB.start(); + } + + @AfterMethod(alwaysRun = true) + void afterMethod() { + if (serverA != null) { + serverA.stop(); + } + if (serverB != null) { + serverB.stop(); + } + } + + @Test + public void testCrossOriginRedirectCarriesAuthorizationHeader() throws Exception { + // serverA (origin host:port) returns 307 Temporary Redirect to serverB (different port -> cross-origin). + serverA.stubFor(get(urlPathMatching(LOOKUP_PATH)) + .willReturn(aResponse() + .withStatus(307) + .withHeader("Location", + "http://127.0.0.1:" + serverB.port() + LOOKUP_PATH))); + + // serverB only responds 200 when the Authorization header is present with the expected token. + // Priorities: lower = higher priority; the specific-Authorization stub must be checked first. + serverB.stubFor(get(urlPathMatching(LOOKUP_PATH)) + .atPriority(2) + .willReturn(aResponse().withStatus(401).withBody("missing auth"))); + serverB.stubFor(get(urlPathMatching(LOOKUP_PATH)) + .atPriority(1) + .withHeader("Authorization", equalTo("Bearer test-token")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(EXPECTED_BODY))); + + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setServiceUrl("http://127.0.0.1:" + serverA.port()); + conf.setAuthentication(AuthenticationFactory.token("test-token")); + + try (HttpClient httpClient = new HttpClient(conf, eventLoopGroup, timer, nameResolver)) { + LookupData result = httpClient.get(LOOKUP_PATH, LookupData.class) + .get(30, TimeUnit.SECONDS); + + assertNotNull(result, "Expected lookup payload after cross-origin redirect"); + assertEquals(result.getBrokerUrl(), "pulsar://broker-b:6650"); + assertEquals(result.getHttpUrl(), "http://broker-b:8080"); + + // Lock the invariant: the final hop on serverB must carry the Authorization header. + serverB.verify(getRequestedFor(urlPathMatching(LOOKUP_PATH)) + .withHeader("Authorization", equalTo("Bearer test-token"))); + } + } +} diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/HttpLookupServiceTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/HttpLookupServiceTest.java new file mode 100644 index 0000000000000..72f7953f16a17 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/HttpLookupServiceTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timer; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.netty.DnsResolverUtil; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * End-to-end check that {@link HttpLookupService#getBroker} carries the {@code Authorization} header + * across a cross-origin redirect — the production scenario where a broker returns + * {@code 307 Temporary Redirect} with a {@code Location} pointing at another broker's + * {@code httpUrl}/{@code httpUrlTls}. + */ +public class HttpLookupServiceTest { + + private static final String TOPIC = "persistent://public/default/cross-origin-lookup-topic"; + private static final String LOOKUP_PATH_REGEX = + "/lookup/v2/topic/persistent/public/default/cross-origin-lookup-topic"; + private static final String LOOKUP_BODY = "{\"brokerUrl\":\"pulsar://broker-b.example:6650\"," + + "\"brokerUrlTls\":\"pulsar+ssl://broker-b.example:6651\"," + + "\"httpUrl\":\"http://broker-b.example:8080\"," + + "\"httpUrlTls\":\"https://broker-b.example:8443\"," + + "\"nativeUrl\":\"pulsar://broker-b.example:6650\"}"; + + private WireMockServer serverA; + private WireMockServer serverB; + private EventLoopGroup eventLoopGroup; + private Timer timer; + + @BeforeClass(alwaysRun = true) + void beforeClass() { + eventLoopGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("HttpLookupServiceTest")); + timer = new HashedWheelTimer(new DefaultThreadFactory("HttpLookupServiceTest-timer")); + } + + @AfterClass(alwaysRun = true) + void afterClass() { + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully(); + } + if (timer != null) { + timer.stop(); + } + } + + @BeforeMethod(alwaysRun = true) + void beforeMethod() { + serverA = new WireMockServer(WireMockConfiguration.wireMockConfig().port(0)); + serverB = new WireMockServer(WireMockConfiguration.wireMockConfig().port(0)); + serverA.start(); + serverB.start(); + } + + @AfterMethod(alwaysRun = true) + void afterMethod() { + if (serverA != null) { + serverA.stop(); + } + if (serverB != null) { + serverB.stop(); + } + } + + @Test + public void testGetBrokerFollowsCrossOriginRedirect() throws Exception { + serverA.stubFor(get(urlPathMatching(LOOKUP_PATH_REGEX)) + .willReturn(aResponse() + .withStatus(307) + .withHeader("Location", + "http://127.0.0.1:" + serverB.port() + LOOKUP_PATH_REGEX))); + + serverB.stubFor(get(urlPathMatching(LOOKUP_PATH_REGEX)) + .atPriority(2) + .willReturn(aResponse().withStatus(401).withBody("missing auth"))); + serverB.stubFor(get(urlPathMatching(LOOKUP_PATH_REGEX)) + .atPriority(1) + .withHeader("Authorization", equalTo("Bearer test-token")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(LOOKUP_BODY))); + + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setServiceUrl("http://127.0.0.1:" + serverA.port()); + conf.setAuthentication(AuthenticationFactory.token("test-token")); + + HttpLookupService lookupService = new HttpLookupService( + InstrumentProvider.NOOP, conf, eventLoopGroup, timer, DnsResolverUtil.adaptToNameResolver(null)); + try { + LookupTopicResult result = lookupService.getBroker(TopicName.get(TOPIC), null) + .get(30, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(result.getLogicalAddress().getHostString(), "broker-b.example"); + assertEquals(result.getLogicalAddress().getPort(), 6650); + + serverB.verify(getRequestedFor(urlPathMatching(LOOKUP_PATH_REGEX)) + .withHeader("Authorization", equalTo("Bearer test-token"))); + } finally { + lookupService.close(); + } + } +} From f7e790b6e876c9893b1cdd441102670f6a5472de Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:56:01 +0800 Subject: [PATCH 020/201] [improve][common] Optimize TopicName.get() to reduce lock contention on cache lookup (#25367) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation `TopicName.get()` previously used `ConcurrentHashMap.computeIfAbsent()` to populate the topic-name cache. Although `computeIfAbsent` is atomic, it holds the internal bin-lock for the entire duration of the mapping function, which includes the non-trivial `TopicName` construction (string splitting, validation, etc.). Under high-concurrency workloads where many threads simultaneously encounter the same uncached topic name, this causes unnecessary lock contention and can degrade throughput. ### Modifications Replace `computeIfAbsent` with an explicit two-step pattern: 1. **Fast path**: call `cache.get(topic)` first — a single volatile read with no locking — and return immediately on a cache hit (steady-state case). 2. **Slow path** (cache miss): construct `TopicName` *outside* the lock, then use `cache.put()` to insert. (cherry picked from commit 8c4e83d2374a6431953684bfadbf5abc301dd26f) --- .../broker/naming/TopicNameGetBenchmark.java | 88 +++++++++++++++++++ .../pulsar/broker/naming/package-info.java | 23 +++++ .../pulsar/common/naming/TopicName.java | 7 +- 3 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 microbench/src/main/java/org/apache/pulsar/broker/naming/TopicNameGetBenchmark.java create mode 100644 microbench/src/main/java/org/apache/pulsar/broker/naming/package-info.java diff --git a/microbench/src/main/java/org/apache/pulsar/broker/naming/TopicNameGetBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/naming/TopicNameGetBenchmark.java new file mode 100644 index 0000000000000..74188e16d3aa9 --- /dev/null +++ b/microbench/src/main/java/org/apache/pulsar/broker/naming/TopicNameGetBenchmark.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.naming; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.pulsar.common.naming.TopicName; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * JMH benchmark for {@link TopicName#get(String)} cold-start (cache-miss) performance + * under 50-thread contention. + * + *

Uses {@code Mode.SingleShotTime} with {@code @Fork(10)} to measure + * the total time of a fixed batch of cold-start calls. No cache clearing is + * needed — each fork is a fresh JVM with an empty cache, and the batch size + * is bounded to avoid OOM. + * + *

Run with: + * 

+ *   ./gradlew :microbench:shadowJar
+ *   java -jar microbench/build/libs/microbench-*-benchmarks.jar TopicNameGetBenchmark
+ *
+ */ +@Fork(10) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Warmup(iterations = 2) +@Measurement(iterations = 5) +@Threads(50) +@State(Scope.Thread) +public class TopicNameGetBenchmark { + + /** + * Each thread processes 10,000 unique topics per invocation. + * 50 threads × 10,000 = 500,000 total entries per invocation — well within memory. + */ + private static final int BATCH_SIZE = 10_000; + private static final AtomicInteger COUNTER = new AtomicInteger(); + + private String[] topics; + + @Setup(Level.Invocation) + public void prepare() { + int base = COUNTER.getAndAdd(BATCH_SIZE); + topics = new String[BATCH_SIZE]; + for (int i = 0; i < BATCH_SIZE; i++) { + topics[i] = "persistent://public/default/topic-" + (base + i); + } + } + + @Benchmark + @OperationsPerInvocation(BATCH_SIZE) + public void coldStartGet(Blackhole bh) { + for (int i = 0; i < BATCH_SIZE; i++) { + bh.consume(TopicName.get(topics[i])); + } + } +} diff --git a/microbench/src/main/java/org/apache/pulsar/broker/naming/package-info.java b/microbench/src/main/java/org/apache/pulsar/broker/naming/package-info.java new file mode 100644 index 0000000000000..f655bf0490db8 --- /dev/null +++ b/microbench/src/main/java/org/apache/pulsar/broker/naming/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Benchmarks for {@link org.apache.pulsar.common.naming.TopicName#get(String)} cold-start (cache-miss) performance + */ +package org.apache.pulsar.broker.naming; \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java index a1c7055a8972c..984a3d830e2f2 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java @@ -72,11 +72,16 @@ public static TopicName get(String domain, String tenant, String namespace, Stri } public static TopicName get(String topic) { + // Fast path: already cached — single volatile read, no lock. TopicName tp = cache.get(topic); if (tp != null) { return tp; } - return cache.computeIfAbsent(topic, k -> new TopicName(k)); + // Use get()+put() instead of computeIfAbsent() to keep construction outside the bin-lock. + // Duplicate instances from racing threads are safe to discard since TopicName is immutable. + TopicName newTp = new TopicName(topic); + TopicName existing = cache.put(topic, newTp); + return existing != null ? existing : newTp; } public static TopicName getPartitionedTopicName(String topic) { From f042ba2f8a4f013f4eac27994b87bb9f4ddfd862 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 21 Apr 2026 19:08:11 +0800 Subject: [PATCH 021/201] [improve][broker] Improve the performance of TopicName constructor (#24463) (cherry picked from commit 3130a93b1abc053fc5e5c9687df044a6091d0a3c) --- .../pulsar/common/naming/TopicName.java | 63 ++++++++++--------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java index 984a3d830e2f2..b53bbbd93e99c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java @@ -114,44 +114,45 @@ private TopicName(String completeTopicName) { try { // The topic name can be in two different forms, one is fully qualified topic name, // the other one is short topic name - if (!completeTopicName.contains("://")) { + int index = completeTopicName.indexOf("://"); + if (index < 0) { // The short topic name can be: // - // - // - String[] parts = StringUtils.split(completeTopicName, '/'); - if (parts.length == 3) { - completeTopicName = TopicDomain.persistent.name() + "://" + completeTopicName; - } else if (parts.length == 1) { - completeTopicName = TopicDomain.persistent.name() + "://" - + PUBLIC_TENANT + "/" + DEFAULT_NAMESPACE + "/" + parts[0]; + List parts = splitBySlash(completeTopicName, 0); + this.domain = TopicDomain.persistent; + if (parts.size() == 3) { + this.tenant = parts.get(0); + this.namespacePortion = parts.get(1); + this.localName = parts.get(2); + } else if (parts.size() == 1) { + this.tenant = PUBLIC_TENANT; + this.namespacePortion = DEFAULT_NAMESPACE; + this.localName = parts.get(0); } else { throw new IllegalArgumentException( "Invalid short topic name '" + completeTopicName + "', it should be in the format of " + "// or "); } - } - - // Expected format: persistent://tenant/namespace/topic - List parts = Splitter.on("://").limit(2).splitToList(completeTopicName); - this.domain = TopicDomain.getEnum(parts.get(0)); - - String rest = parts.get(1); - - // Expected format: tenant/namespace/ - parts = Splitter.on("/").limit(4).splitToList(rest); - if (parts.size() == 4) { - throw new IllegalArgumentException( - "V1 topic names (with cluster component) are no longer supported. " - + "Please use the V2 format: '://tenant/namespace/topic'. Got: " - + completeTopicName); - } else if (parts.size() == 3) { - this.tenant = parts.get(0); - this.namespacePortion = parts.get(1); - this.localName = parts.get(2); - this.partitionIndex = getPartitionIndex(completeTopicName); - this.namespaceName = NamespaceName.get(tenant, namespacePortion); + this.completeTopicName = domain.name() + "://" + tenant + "/" + namespacePortion + "/" + localName; } else { - throw new IllegalArgumentException("Invalid topic name: " + completeTopicName); + // Expected format: persistent://tenant/namespace/topic + List parts = splitBySlash(completeTopicName.substring(index + "://".length()), 4); + if (parts.size() == 4) { + throw new IllegalArgumentException( + "V1 topic names (with cluster component) are no longer supported. " + + "Please use the V2 format: '://tenant/namespace/topic'. Got: " + + completeTopicName); + } + if (parts.size() == 3) { + this.tenant = parts.get(0); + this.namespacePortion = parts.get(1); + this.localName = parts.get(2); + } else { + throw new IllegalArgumentException("Invalid topic name " + completeTopicName); + } + this.completeTopicName = completeTopicName; + this.domain = TopicDomain.getEnum(completeTopicName.substring(0, index)); } if (StringUtils.isBlank(localName)) { @@ -159,11 +160,11 @@ private TopicName(String completeTopicName) { + " be blank.", completeTopicName)); } + this.partitionIndex = getPartitionIndex(localName); + this.namespaceName = NamespaceName.get(tenant, namespacePortion); } catch (NullPointerException e) { throw new IllegalArgumentException("Invalid topic name: " + completeTopicName, e); } - this.completeTopicName = String.format("%s://%s/%s/%s", - domain, tenant, namespacePortion, localName); } public boolean isPersistent() { From c95d317b454045308df88860f805c8a5a15be3ee Mon Sep 17 00:00:00 2001 From: Hao Zhang Date: Tue, 7 Apr 2026 11:36:30 +0800 Subject: [PATCH 022/201] [fix][broker] Fix backlog clearing for unloaded namespace bundles (#25272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 张浩 (cherry picked from commit 8e37e3ee63d8fcebed493daa6e577fdc08591746) --- .../pulsar/broker/admin/AdminResource.java | 7 - .../broker/admin/impl/NamespacesBase.java | 295 +++++++++--------- .../pulsar/broker/admin/v2/Namespaces.java | 57 ++-- .../broker/namespace/NamespaceService.java | 19 ++ .../pulsar/broker/admin/AdminApiTest.java | 126 ++++++++ 5 files changed, 333 insertions(+), 171 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 190ed01114329..5ac94210c965e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.admin; -import static org.apache.commons.lang3.StringUtils.isBlank; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.errorprone.annotations.CanIgnoreReturnValue; @@ -820,12 +819,6 @@ protected void checkNotNull(Object o, String errorMessage) { } } - protected void checkNotBlank(String str, String errorMessage) { - if (isBlank(str)) { - throw new RestException(Status.PRECONDITION_FAILED, errorMessage); - } - } - protected boolean isManagedLedgerNotFoundException(Throwable cause) { return cause instanceof ManagedLedgerException.MetadataNotFoundException || cause instanceof MetadataStoreException.NotFoundException; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 288c2a6c40a01..a1b35d64fe90d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -79,7 +79,6 @@ import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; -import org.apache.pulsar.common.naming.NamespaceBundles; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; @@ -1824,129 +1823,108 @@ private CompletableFuture doUpdatePersistenceAsync(PersistencePolicies per ); } - protected void internalClearNamespaceBacklog(AsyncResponse asyncResponse, boolean authoritative) { - validateNamespaceOperation(namespaceName, NamespaceOperation.CLEAR_BACKLOG); - - final List> futures = new ArrayList<>(); - try { - NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory() - .getBundles(namespaceName); - for (NamespaceBundle nsBundle : bundles.getBundles()) { - // check if the bundle is owned by any broker, if not then there is no backlog on this bundle to clear - if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) { - futures.add(pulsar().getAdminClient().namespaces() - .clearNamespaceBundleBacklogAsync(namespaceName.toString(), nsBundle.getBundleRange())); - } - } - } catch (WebApplicationException wae) { - asyncResponse.resume(wae); - return; - } catch (Exception e) { - asyncResponse.resume(new RestException(e)); - return; - } - - FutureUtil.waitForAll(futures).handle((result, exception) -> { - if (exception != null) { - log.warn("[{}] Failed to clear backlog on the bundles for namespace {}: {}", clientAppId(), - namespaceName, exception.getCause().getMessage()); - if (exception.getCause() instanceof PulsarAdminException) { - asyncResponse.resume(new RestException((PulsarAdminException) exception.getCause())); - return null; - } else { - asyncResponse.resume(new RestException(exception.getCause())); - return null; - } - } - log.info("[{}] Successfully cleared backlog on all the bundles for namespace {}", clientAppId(), - namespaceName); - asyncResponse.resume(Response.noContent().build()); - return null; - }); + protected CompletableFuture internalClearNamespaceBacklogAsync(boolean authoritative) { + return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.CLEAR_BACKLOG) + .thenCompose(__ -> pulsar().getNamespaceService().getNamespaceBundleFactory() + .getBundlesAsync(namespaceName)) + .thenCompose(bundles -> { + final List> futures = new ArrayList<>(); + for (NamespaceBundle nsBundle : bundles.getBundles()) { + try { + futures.add(pulsar().getAdminClient().namespaces() + .clearNamespaceBundleBacklogAsync(namespaceName.toString(), + nsBundle.getBundleRange())); + } catch (PulsarServerException e) { + return CompletableFuture.failedFuture(e); + } + } + return FutureUtil.waitForAll(futures); + }).thenRun(() -> log.info("[{}] Successfully cleared backlog on all the bundles for namespace {}", + clientAppId(), namespaceName)); } @SuppressWarnings("deprecation") - protected void internalClearNamespaceBundleBacklog(String bundleRange, boolean authoritative) { - validateNamespaceOperation(namespaceName, NamespaceOperation.CLEAR_BACKLOG); - checkNotNull(bundleRange, "BundleRange should not be null"); - - Policies policies = getNamespacePolicies(namespaceName); - - // check cluster ownership for a given global namespace: redirect if peer-cluster owns it - validateGlobalNamespaceOwnership(namespaceName); - - validateNamespaceBundleOwnership(namespaceName, policies.bundles, bundleRange, authoritative, true); + protected CompletableFuture internalClearNamespaceBundleBacklogAsync(String bundleRange, + boolean authoritative) { + if (bundleRange == null) { + return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "BundleRange should not be null")); + } - clearBacklog(namespaceName, bundleRange, null); - log.info("[{}] Successfully cleared backlog on namespace bundle {}/{}", clientAppId(), namespaceName, - bundleRange); + return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.CLEAR_BACKLOG) + .thenCompose(__ -> { + // check cluster ownership for a given global namespace: redirect if peer-cluster owns it + return validateGlobalNamespaceOwnershipAsync(namespaceName); + }) + .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) + .thenCompose(policies -> + // Allow acquiring ownership for an unassigned bundle so backlog can be cleared + // even if not loaded. + validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, + authoritative, false)) + .thenCompose(bundle -> clearBacklogAsync(bundle, null)) + .thenRun(() -> log.info("[{}] Successfully cleared backlog on namespace bundle {}/{}", clientAppId(), + namespaceName, bundleRange)); } - protected void internalClearNamespaceBacklogForSubscription(AsyncResponse asyncResponse, String subscription, - boolean authoritative) { - validateNamespaceOperation(namespaceName, NamespaceOperation.CLEAR_BACKLOG); - checkNotNull(subscription, "Subscription should not be null"); - - final List> futures = new ArrayList<>(); - try { - NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory() - .getBundles(namespaceName); - for (NamespaceBundle nsBundle : bundles.getBundles()) { - // check if the bundle is owned by any broker, if not then there is no backlog on this bundle to clear - if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) { - futures.add(pulsar().getAdminClient().namespaces().clearNamespaceBundleBacklogForSubscriptionAsync( - namespaceName.toString(), nsBundle.getBundleRange(), subscription)); - } - } - } catch (WebApplicationException wae) { - asyncResponse.resume(wae); - return; - } catch (Exception e) { - asyncResponse.resume(new RestException(e)); - return; + protected CompletableFuture internalClearNamespaceBacklogForSubscriptionAsync(String subscription, + boolean authoritative) { + if (subscription == null) { + return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "Subscription should not be null")); } - FutureUtil.waitForAll(futures).handle((result, exception) -> { - if (exception != null) { - log.warn("[{}] Failed to clear backlog for subscription {} on the bundles for namespace {}: {}", - clientAppId(), subscription, namespaceName, exception.getCause().getMessage()); - if (exception.getCause() instanceof PulsarAdminException) { - asyncResponse.resume(new RestException((PulsarAdminException) exception.getCause())); - return null; - } else { - asyncResponse.resume(new RestException(exception.getCause())); - return null; - } - } - log.info("[{}] Successfully cleared backlog for subscription {} on all the bundles for namespace {}", - clientAppId(), subscription, namespaceName); - asyncResponse.resume(Response.noContent().build()); - return null; - }); + return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.CLEAR_BACKLOG) + .thenCompose(__ -> pulsar().getNamespaceService().getNamespaceBundleFactory() + .getBundlesAsync(namespaceName)) + .thenCompose(bundles -> { + final List> futures = new ArrayList<>(); + for (NamespaceBundle nsBundle : bundles.getBundles()) { + try { + futures.add(pulsar().getAdminClient().namespaces() + .clearNamespaceBundleBacklogForSubscriptionAsync( + namespaceName.toString(), nsBundle.getBundleRange(), subscription)); + } catch (PulsarServerException e) { + return CompletableFuture.failedFuture(e); + } + } + return FutureUtil.waitForAll(futures); + }).thenRun(() -> log.info( + "[{}] Successfully cleared backlog for subscription {} on all the bundles for namespace {}", + clientAppId(), subscription, namespaceName)); } @SuppressWarnings("deprecation") - protected void internalClearNamespaceBundleBacklogForSubscription(String subscription, String bundleRange, - boolean authoritative) { - validateNamespaceOperation(namespaceName, NamespaceOperation.CLEAR_BACKLOG); - checkNotNull(subscription, "Subscription should not be null"); - checkNotNull(bundleRange, "BundleRange should not be null"); - - Policies policies = getNamespacePolicies(namespaceName); - - // check cluster ownership for a given global namespace: redirect if peer-cluster owns it - validateGlobalNamespaceOwnership(namespaceName); - - validateNamespaceBundleOwnership(namespaceName, policies.bundles, bundleRange, authoritative, true); + protected CompletableFuture internalClearNamespaceBundleBacklogForSubscriptionAsync(String subscription, + String bundleRange, + boolean authoritative) { + if (subscription == null) { + return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "Subscription should not be null")); + } + if (bundleRange == null) { + return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "BundleRange should not be null")); + } - clearBacklog(namespaceName, bundleRange, subscription); - log.info("[{}] Successfully cleared backlog for subscription {} on namespace bundle {}/{}", clientAppId(), - subscription, namespaceName, bundleRange); + return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.CLEAR_BACKLOG) + .thenCompose(__ -> { + // check cluster ownership for a given global namespace: redirect if peer-cluster owns it + return validateGlobalNamespaceOwnershipAsync(namespaceName); + }) + .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) + .thenCompose(policies -> + // Allow acquiring ownership for an unassigned bundle so backlog can be cleared + // even if not loaded. + validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, + authoritative, false)) + .thenCompose(bundle -> clearBacklogAsync(bundle, subscription)) + .thenRun(() -> log.info( + "[{}] Successfully cleared backlog for subscription {} on namespace bundle {}/{}", + clientAppId(), subscription, namespaceName, bundleRange)); } protected CompletableFuture internalUnsubscribeNamespaceAsync(String subscription, boolean authoritative) { - checkNotNull(subscription, "Subscription should not be null"); + if (subscription == null) { + return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "Subscription should not be null")); + } return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.UNSUBSCRIBE) .thenCompose(__ -> pulsar().getNamespaceService().getNamespaceBundleFactory() @@ -1969,8 +1947,12 @@ protected CompletableFuture internalUnsubscribeNamespaceAsync(String subsc @SuppressWarnings("deprecation") protected CompletableFuture internalUnsubscribeNamespaceBundleAsync(String subscription, String bundleRange, boolean authoritative) { - checkNotNull(subscription, "Subscription should not be null"); - checkNotNull(bundleRange, "BundleRange should not be null"); + if (subscription == null) { + return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "Subscription should not be null")); + } + if (bundleRange == null) { + return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "BundleRange should not be null")); + } return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.UNSUBSCRIBE) .thenCompose(__ -> validateGlobalNamespaceOwnershipAsync(namespaceName)) @@ -2062,8 +2044,14 @@ protected void internalSetDelayedDelivery(DelayedDeliveryPolicies delayedDeliver } protected CompletableFuture internalSetNamespaceAntiAffinityGroupAsync(String antiAffinityGroup) { - checkNotNull(antiAffinityGroup, "Anti-affinity group should not be null"); - checkNotBlank(antiAffinityGroup, "Anti-affinity group can't be empty"); + if (antiAffinityGroup == null) { + return FutureUtil.failedFuture( + new RestException(Status.BAD_REQUEST, "Anti-affinity group should not be null")); + } + if (StringUtils.isBlank(antiAffinityGroup)) { + return FutureUtil.failedFuture( + new RestException(Status.PRECONDITION_FAILED, "Anti-affinity group can't be empty")); + } return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.ANTI_AFFINITY, PolicyOperation.WRITE) .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()).thenCompose( __ -> getDefaultBundleDataAsync().thenCompose( @@ -2101,10 +2089,20 @@ protected CompletableFuture internalRemoveNamespaceAntiAffinityGroupAsync( protected CompletableFuture> internalGetAntiAffinityNamespacesAsync(String cluster, String antiAffinityGroup, String tenant) { - checkNotNull(cluster, "Cluster should not be null"); - checkNotNull(antiAffinityGroup, "Anti-affinity group should not be null"); - checkNotNull(tenant, "Tenant should not be null"); - checkNotBlank(antiAffinityGroup, "Anti-affinity group can't be empty"); + if (cluster == null) { + return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "Cluster should not be null")); + } + if (antiAffinityGroup == null) { + return FutureUtil.failedFuture( + new RestException(Status.BAD_REQUEST, "Anti-affinity group should not be null")); + } + if (tenant == null) { + return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "Tenant should not be null")); + } + if (StringUtils.isBlank(antiAffinityGroup)) { + return FutureUtil.failedFuture( + new RestException(Status.PRECONDITION_FAILED, "Anti-affinity group can't be empty")); + } return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.ANTI_AFFINITY, PolicyOperation.READ) .thenCompose(__ -> validateClusterExistsAsync(cluster)) @@ -2135,39 +2133,48 @@ private boolean checkQuotas(Policies policies, RetentionPolicies retention) { return checkBacklogQuota(quota, retention); } - private void clearBacklog(NamespaceName nsName, String bundleRange, String subscription) { - try { - List topicList = pulsar().getBrokerService().getAllTopicsFromNamespaceBundle(nsName.toString(), - nsName.toString() + "/" + bundleRange); - - List> futures = new ArrayList<>(); - if (subscription != null) { - if (subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) { - subscription = PersistentReplicator.getRemoteCluster(subscription); - } - for (Topic topic : topicList) { - if (topic instanceof PersistentTopic - && !pulsar().getBrokerService().isSystemTopic(TopicName.get(topic.getName()))) { - futures.add(((PersistentTopic) topic).clearBacklog(subscription)); + private CompletableFuture clearBacklogAsync(NamespaceBundle bundle, String subscription) { + return pulsar().getNamespaceService().getOwnedPersistentTopicListForNamespaceBundle(bundle) + .thenCompose(topicsInBundle -> { + List> futures = new ArrayList<>(); + String effectiveSubscription = subscription; + if (effectiveSubscription != null + && effectiveSubscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) { + effectiveSubscription = PersistentReplicator.getRemoteCluster(effectiveSubscription); } - } - } else { - for (Topic topic : topicList) { - if (topic instanceof PersistentTopic - && !pulsar().getBrokerService().isSystemTopic(TopicName.get(topic.getName()))) { - futures.add(((PersistentTopic) topic).clearBacklog()); + final String finalSubscription = effectiveSubscription; + + for (String topic : topicsInBundle) { + TopicName topicName = TopicName.get(topic); + if (pulsar().getBrokerService().isSystemTopic(topicName)) { + continue; + } + futures.add(pulsar().getBrokerService().getTopic(topicName.toString(), false) + .thenCompose(optTopic -> { + if (optTopic.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + Topic loaded = optTopic.get(); + if (!(loaded instanceof PersistentTopic persistentTopic)) { + return CompletableFuture.completedFuture(null); + } + return finalSubscription != null + ? persistentTopic.clearBacklog(finalSubscription) + : persistentTopic.clearBacklog(); + })); } - } - } - FutureUtil.waitForAll(futures).get(); - } catch (Exception e) { - log.error("[{}] Failed to clear backlog for namespace {}/{}, subscription: {}", clientAppId(), - nsName.toString(), bundleRange, subscription, e); - throw new RestException(e); - } + return FutureUtil.waitForAll(futures); + }).exceptionally(ex -> { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + if (cause instanceof RestException) { + throw (RestException) cause; + } + throw new RestException(cause); + }); } + private CompletableFuture unsubscribeAsync(NamespaceBundle bundle, String subscription) { if (subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) { return CompletableFuture.failedFuture( diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 173c22fac79e6..ad01dd6f39c12 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -1516,14 +1516,14 @@ public void getPersistence( public void clearNamespaceBacklog(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { - try { - validateNamespaceName(tenant, namespace); - internalClearNamespaceBacklog(asyncResponse, authoritative); - } catch (WebApplicationException wae) { - asyncResponse.resume(wae); - } catch (Exception e) { - asyncResponse.resume(new RestException(e)); - } + validateNamespaceName(tenant, namespace); + internalClearNamespaceBacklogAsync(authoritative) + .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) + .exceptionally(ex -> { + log.error("[{}] Failed to clear backlog on namespace {}", clientAppId(), namespaceName, ex); + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } @POST @@ -1534,11 +1534,19 @@ public void clearNamespaceBacklog(@Suspended final AsyncResponse asyncResponse, @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin or operate permission on the namespace"), @ApiResponse(code = 404, message = "Namespace does not exist") }) - public void clearNamespaceBundleBacklog(@PathParam("tenant") String tenant, + public void clearNamespaceBundleBacklog(@Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("bundle") String bundleRange, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateNamespaceName(tenant, namespace); - internalClearNamespaceBundleBacklog(bundleRange, authoritative); + internalClearNamespaceBundleBacklogAsync(bundleRange, authoritative) + .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) + .exceptionally(ex -> { + log.error("[{}] Failed to clear backlog on namespace bundle {}/{}", clientAppId(), + namespaceName, bundleRange, ex); + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } @POST @@ -1552,14 +1560,15 @@ public void clearNamespaceBacklogForSubscription(@Suspended final AsyncResponse @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("subscription") String subscription, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { - try { - validateNamespaceName(tenant, namespace); - internalClearNamespaceBacklogForSubscription(asyncResponse, subscription, authoritative); - } catch (WebApplicationException wae) { - asyncResponse.resume(wae); - } catch (Exception e) { - asyncResponse.resume(new RestException(e)); - } + validateNamespaceName(tenant, namespace); + internalClearNamespaceBacklogForSubscriptionAsync(subscription, authoritative) + .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) + .exceptionally(ex -> { + log.error("[{}] Failed to clear backlog for subscription {} on namespace {}", clientAppId(), + subscription, namespaceName, ex); + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } @POST @@ -1570,12 +1579,20 @@ public void clearNamespaceBacklogForSubscription(@Suspended final AsyncResponse @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin or operate permission on the namespace"), @ApiResponse(code = 404, message = "Namespace does not exist") }) - public void clearNamespaceBundleBacklogForSubscription(@PathParam("tenant") String tenant, + public void clearNamespaceBundleBacklogForSubscription(@Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("subscription") String subscription, @PathParam("bundle") String bundleRange, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateNamespaceName(tenant, namespace); - internalClearNamespaceBundleBacklogForSubscription(subscription, bundleRange, authoritative); + internalClearNamespaceBundleBacklogForSubscriptionAsync(subscription, bundleRange, authoritative) + .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) + .exceptionally(ex -> { + log.error("[{}] Failed to clear backlog for subscription {} on namespace bundle {}/{}", + clientAppId(), subscription, namespaceName, bundleRange, ex); + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } @POST diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index c16c9338bf994..44988d8fd5c3b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1378,6 +1378,25 @@ public CompletableFuture> getOwnedTopicListForNamespaceBundle(Names }); } + public CompletableFuture> getOwnedPersistentTopicListForNamespaceBundle(NamespaceBundle bundle) { + return getListOfPersistentTopics(bundle.getNamespaceObject()).thenCompose(topics -> + CompletableFuture.completedFuture( + topics.stream() + .filter(topic -> bundle.includes(TopicName.get(topic))) + .collect(Collectors.toList()))) + .thenCombine(getPartitions(bundle.getNamespaceObject(), TopicDomain.persistent).thenCompose(topics -> + CompletableFuture.completedFuture( + topics.stream().filter(topic -> bundle.includes(TopicName.get(topic))) + .collect(Collectors.toList()))), (left, right) -> { + for (String topic : right) { + if (!left.contains(topic)) { + left.add(topic); + } + } + return left; + }); + } + /*** * Checks whether the topic exists( partitioned or non-partitioned ). */ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index 2b7b404e655b0..38caf1f0facfe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -2285,6 +2285,132 @@ public void testClearBacklogOnNamespace(Integer numBundles) throws Exception { assertEquals(backlog, 0); } + @Test(dataProvider = "numBundles") + public void testClearNamespaceBundleBacklogOnUnloadedBundle(Integer numBundles) throws Exception { + String namespace = "prop-xyz/ns1-bundles"; + admin.namespaces().createNamespace("prop-xyz/ns1-bundles", numBundles); + + String topic = "persistent://" + namespace + "/t1"; + String subscription = "sub1"; + + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subscription) + .subscribe(); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + for (int i = 0; i < 10; i++) { + producer.send(("message-" + i).getBytes()); + } + producer.close(); + consumer.close(); + + long backlog = admin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog(); + assertEquals(backlog, 10); + + NamespaceBundle bundle = + pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(TopicName.get(topic)); + admin.namespaces().unloadNamespaceBundle(namespace, bundle.getBundleRange()); + + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .pollInterval(200, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + assertFalse(pulsar.getNamespaceService().isServiceUnitOwned(bundle), + "Bundle should not be owned by current broker"); + assertFalse(otherPulsar.getNamespaceService().isServiceUnitOwned(bundle), + "Bundle should not be owned by other broker"); + }); + + admin.namespaces().clearNamespaceBundleBacklog(namespace, bundle.getBundleRange()); + + @Cleanup + Consumer consumer1 = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subscription) + .subscribe(); + + long backlogAfter = admin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog(); + assertEquals(backlogAfter, 0); + } + + @Test(dataProvider = "numBundles") + public void testClearNamespaceBundleBacklogForSubscriptionOnUnloadedBundle(Integer numBundles) throws Exception { + String namespace = "prop-xyz/ns1-bundles"; + admin.namespaces().createNamespace("prop-xyz/ns1-bundles", numBundles); + + String topic = "persistent://" + namespace + "/t1"; + String subscription = "sub1"; + String otherSubscription = "sub2"; + + Consumer consumer1 = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subscription) + .subscribe(); + + Consumer consumer2 = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(otherSubscription) + .subscribe(); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + for (int i = 0; i < 10; i++) { + producer.send(("message-" + i).getBytes()); + } + producer.close(); + consumer1.close(); + consumer2.close(); + + long backlog = admin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog(); + assertEquals(backlog, 10); + long otherBacklog = admin.topics().getStats(topic).getSubscriptions().get(otherSubscription).getMsgBacklog(); + assertEquals(otherBacklog, 10); + + NamespaceBundle bundle = + pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(TopicName.get(topic)); + admin.namespaces().unloadNamespaceBundle(namespace, bundle.getBundleRange()); + + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .pollInterval(200, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + assertFalse(pulsar.getNamespaceService().isServiceUnitOwned(bundle), + "Bundle should not be owned by current broker"); + assertFalse(otherPulsar.getNamespaceService().isServiceUnitOwned(bundle), + "Bundle should not be owned by other broker"); + }); + + admin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, bundle.getBundleRange(), + subscription); + + @Cleanup + Consumer consumer3 = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subscription) + .subscribe(); + + @Cleanup + Consumer consumer4 = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(otherSubscription) + .subscribe(); + + long backlogAfter = admin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog(); + assertEquals(backlogAfter, 0); + long otherBacklogAfter = + admin.topics().getStats(topic).getSubscriptions().get(otherSubscription).getMsgBacklog(); + assertEquals(otherBacklogAfter, 10); + } + + @Test(dataProvider = "bundling") public void testUnsubscribeOnNamespace(Integer numBundles) throws Exception { admin.namespaces().createNamespace("prop-xyz/ns1-bundles", numBundles); From eae4ef87d87c34c13b6e3c34f630cdb510be833a Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 20 Apr 2026 21:20:09 +0800 Subject: [PATCH 023/201] [fix][broker]Namespaces can be created with may empty replication_clusters policy (#25551) (cherry picked from commit 90332238719502d1c094d6ddd4fc3c7ba431c2fc) --- .../broker/admin/impl/NamespacesBase.java | 4 +-- .../pulsar/broker/service/BrokerService.java | 12 +++---- .../pulsar/broker/admin/AdminApi2Test.java | 35 +++++++++++++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index a1b35d64fe90d..53d8bc49d9123 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -480,7 +480,7 @@ private CompletableFuture precheckWhenDeleteNamespace(NamespaceName ns // There are still more than one clusters configured for the global namespace throw new RestException(Status.PRECONDITION_FAILED, "Cannot delete the global namespace " + nsName + ". There are still more than " - + "one replication clusters configured."); + + "one replication clusters configured or replication clusters is empty."); } if (!cluster.equals(config().getClusterName())) { // the only replication cluster is other cluster, redirect @@ -2249,7 +2249,7 @@ private CompletableFuture validatePoliciesAsync(NamespaceName ns, Policies log.info(msg); return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, msg)); } - pulsar().getBrokerService().setCurrentClusterAllowedIfNoClusterIsAllowed(ns, policies); + pulsar().getBrokerService().setCurrentClusterAllowedWhenCreating(ns, policies); // Validate cluster names and permissions return Stream.concat(policies.replication_clusters.stream(), policies.allowed_clusters.stream()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index d6226254f3c45..d21477f000d50 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -4029,14 +4029,14 @@ public boolean isCurrentClusterAllowed(NamespaceName nsName, Policies nsPolicies return nsPolicies.replication_clusters.contains(pulsar.getConfig().getClusterName()); } - public void setCurrentClusterAllowedIfNoClusterIsAllowed(NamespaceName nsName, Policies nsPolicies) { - if (nsPolicies.replication_clusters.contains(pulsar.getConfig().getClusterName()) - || nsPolicies.allowed_clusters.contains(pulsar.getConfig().getClusterName())) { - return; - } + public void setCurrentClusterAllowedWhenCreating(NamespaceName nsName, Policies nsPolicies) { if (nsPolicies.replication_clusters.isEmpty()) { nsPolicies.replication_clusters.add(pulsar.getConfig().getClusterName()); - } else { + } + if (nsPolicies.allowed_clusters.isEmpty()) { + return; + } + if (!nsPolicies.allowed_clusters.contains(pulsar.getConfig().getClusterName())) { nsPolicies.allowed_clusters.add(pulsar.getConfig().getClusterName()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index a2aa358cc0088..774c195b7fbb9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -41,7 +41,11 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.lang.reflect.Field; +import java.net.URI; import java.net.URL; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.time.Clock; import java.util.ArrayList; @@ -137,6 +141,7 @@ import org.apache.pulsar.common.policies.data.PartitionedTopicStats; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; +import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -1732,6 +1737,36 @@ public void testCreateNamespaceWithNoClusters() throws PulsarAdminException { Collections.singletonList(localCluster)); } + @Test + public void testCreateNamespaceWithEmptyReplicationClustersByHttp() throws Exception { + String localCluster = pulsar.getConfiguration().getClusterName(); + String namespacePart = newUniqueName("ns"); + String namespace = defaultTenant + "/" + namespacePart; + + // Create namespace with "allowed_cluster", and the param "replication_clusters" is empty. + HttpClient httpClient = HttpClient.newHttpClient(); + URI adminV2Uri = URI.create(brokerUrl.toString()).resolve("/admin/v2/"); + String namespaceRequestBody = "{\"allowed_clusters\": [\"" + localCluster + "\"]}"; + HttpRequest createNamespaceRequest = + HttpRequest.newBuilder(adminV2Uri.resolve("namespaces/" + namespace)) + .header("Content-Type", "application/json") + .PUT(HttpRequest.BodyPublishers.ofString(namespaceRequestBody)) + .build(); + HttpResponse createNamespaceResponse = httpClient.send(createNamespaceRequest, + HttpResponse.BodyHandlers.ofString()); + assertEquals(createNamespaceResponse.statusCode(), Status.NO_CONTENT.getStatusCode(), + "Failed to create namespace by HTTP: " + createNamespaceResponse.body()); + + // Verify: replication_clusters is not empty. + Awaitility.await().untilAsserted(() -> { + Policies policies = admin.namespaces().getPolicies(namespace); + assertEquals(policies.replication_clusters.size(), 1); + assertEquals(policies.allowed_clusters.size(), 1); + assertTrue(policies.replication_clusters.contains(localCluster)); + assertTrue(policies.allowed_clusters.contains(localCluster)); + }); + } + @Test(timeOut = 30000) public void testConsumerStatsLastTimestamp() throws PulsarClientException, PulsarAdminException, InterruptedException { From 49c0894d119c23862e5ac3cfacaf820489543a13 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 21 Apr 2026 19:25:59 +0300 Subject: [PATCH 024/201] [fix][io] Restore lz4 compression with Kafka IO connector after #25198 exclusion - lz4 compression with Kafka would break without this change --- pulsar-io/kafka/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index b8b3701bd8a41..9a4d7ecd2ea6c 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -91,6 +91,11 @@ + + at.yawk.lz4 + lz4-java + + io.confluent kafka-schema-registry-client From c451abf3d690bd4f226ac1dd740fd665ae2178b3 Mon Sep 17 00:00:00 2001 From: sinan liu Date: Sat, 4 Apr 2026 06:11:47 +0800 Subject: [PATCH 025/201] [fix][broker] Prevent timed-out producer creation from racing with retry (#25460) (cherry picked from commit 56442c27907f3e80e4876404ff31063811a92fba) --- .../java/org/apache/pulsar/broker/service/ServerCnx.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 9a835370f7d48..de1e16a0d2fbb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1875,6 +1875,13 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ ProducerAccessMode producerAccessMode, Optional topicEpoch, boolean supportsPartialProducer, CompletableFuture producerFuture){ + if (producerFuture.isCompletedExceptionally()) { + log.info("[{}] Skipped producer creation after timeout on client side. producerId={}, producerName={}", + remoteAddress, producerId, producerName); + producers.remove(producerId, producerFuture); + return; + } + CompletableFuture producerQueuedFuture = new CompletableFuture<>(); Producer producer = new Producer(topic, ServerCnx.this, producerId, producerName, getPrincipal(), isEncrypted, metadata, schemaVersion, epoch, From a4e111979c2b086526536cf00223725fe5e7981c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 9 Apr 2026 01:39:36 +0300 Subject: [PATCH 026/201] [fix][test] Fix flaky ServerCnxTest.testCreateProducerTimeoutThenCreateSameNamedProducerShouldFail (#25497) (cherry picked from commit 7ce8a8c7a03b2cef1360d05f918436c84d7627fe) --- .../org/apache/pulsar/broker/service/ServerCnxTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index bb4b8bbf0e58c..824bfca9c2016 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -2135,6 +2135,13 @@ public void testCreateProducerTimeoutThenCreateSameNamedProducerShouldFail() thr producerName, Collections.emptyMap(), false); channel.writeInbound(createProducer1); + // Run pending tasks to ensure the producer is registered in the producers map + // before the close command is processed. Without this, the close command may not + // find the producer (since registration happens in a thenApplyAsync callback), + // causing it to skip cancellation and leading to a race between the first and + // second producer creation. + channel.runPendingTasks(); + ByteBuf closeProducer = Commands.newCloseProducer(1 /* producer id */, 2 /* request id */); channel.writeInbound(closeProducer); From 79bd60285f0ca17f8a6acb722a854c40ed8ef220 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 30 Mar 2026 18:06:51 -0700 Subject: [PATCH 027/201] [fix][test] Fix flaky testLoadBalancerServiceUnitTableViewSyncer (#25427) (cherry picked from commit 0c285f4cd8ed4f43fc20f5be3a5f64196aea2efd) --- .../ExtensibleLoadManagerImplBaseTest.java | 14 ++- .../ExtensibleLoadManagerImplTest.java | 91 ++++++++++++++----- 2 files changed, 83 insertions(+), 22 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java index 4dedba4f995b8..b50081de89e64 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java @@ -87,6 +87,7 @@ protected ServiceConfiguration getDefaultConf() { } protected ArrayList clients = new ArrayList<>(); + private final java.util.Map originalLookupServices = new java.util.HashMap<>(); @DataProvider(name = "serviceUnitStateTableViewClassName") public static Object[][] serviceUnitStateTableViewClassName() { @@ -152,7 +153,10 @@ protected void setup() throws Exception { lookupService = (LookupService) FieldUtils.readDeclaredField(pulsarClient, "lookup", true); for (int i = 0; i < 4; i++) { - clients.add(pulsarClient(lookupUrl.toString(), 100)); + PulsarClient client = pulsarClient(lookupUrl.toString(), 100); + clients.add(client); + originalLookupServices.put(client, + (LookupService) FieldUtils.readDeclaredField(client, "lookup", true)); } } @@ -195,6 +199,14 @@ protected void initializeState() throws PulsarAdminException, IllegalAccessExcep admin.namespaces().unload(defaultTestNamespace); reset(primaryLoadManager, secondaryLoadManager); FieldUtils.writeDeclaredField(pulsarClient, "lookup", lookupService, true); + // Restore original lookup services for all shared clients to prevent state leakage + // between tests when a previous test fails before resetting spied lookup services. + for (PulsarClient client : clients) { + LookupService original = originalLookupServices.get(client); + if (original != null) { + FieldUtils.writeDeclaredField(client, "lookup", original, true); + } + } pulsar1.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); pulsar2.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 82fed30cb356e..c99443ac137a2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -88,6 +88,7 @@ import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateMetadataStoreTableViewImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl; @@ -771,7 +772,9 @@ public static void testUnloadClientReconnectionWithLookup(List cli for (var consumer : consumers) { consumer.close(); } - resetLookupService(pulsarClient, lookup.getLeft()); + if (lookup != null) { + resetLookupService(pulsarClient, lookup.getLeft()); + } } } @@ -1239,31 +1242,54 @@ public void testDeployAndRollbackLoadManager() throws Exception { pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); } } - // Check if the broker is available + // Simulate broker going offline: clean ownerships first (as disableBroker() does), + // then unregister from the broker registry. var wrapper = (ExtensibleLoadManagerWrapper) pulsar4.getLoadManager().get(); var loadManager4 = spy((ExtensibleLoadManagerImpl) FieldUtils.readField(wrapper, "loadManager", true)); - loadManager4.getBrokerRegistry().unregister(); + ServiceUnitStateChannel channel4 = (ServiceUnitStateChannel) + FieldUtils.readField(loadManager4, "serviceUnitStateChannel", true); + channel4.cleanOwnerships(); + // Simulate broker going offline by removing it from the broker registry. + // Set state to Closed BEFORE deleting the ZK node to prevent the notification + // handler's session-expiry recovery from auto-re-registering broker4. + // In production, the PulsarService shuts down after unregister(), so the handler + // never fires. In tests, the service stays running, creating a race. + var registry4 = (BrokerRegistryImpl) loadManager4.getBrokerRegistry(); + registry4.state.set(BrokerRegistryImpl.State.Closed); + String brokerZkPath = "/loadbalance/brokers/" + pulsar4.getBrokerId(); + pulsar4.getLocalMetadataStore().delete(brokerZkPath, Optional.empty()).get(); NamespaceName slaMonitorNamespace = getSLAMonitorNamespace(pulsar4.getBrokerId(), pulsar.getConfiguration()); String slaMonitorTopic = slaMonitorNamespace.getPersistentTopicName("test"); + // Wait for ownership to be reassigned after broker unregistration + String pulsar4BrokerUrl = pulsar4.getBrokerServiceUrl(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + String lookupResult = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + assertNotNull(lookupResult); + assertNotEquals(lookupResult, pulsar4BrokerUrl); + }); String result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); - assertNotNull(result); log.info("{} Namespace is re-owned by {}", slaMonitorTopic, result); - assertNotEquals(result, pulsar4.getBrokerServiceUrl()); Producer producer = pulsar.getClient().newProducer(Schema.STRING) .topic(slaMonitorTopic).create(); producer.send("t1"); // Test re-register broker and check the lookup result - loadManager4.getBrokerRegistry().registerAsync().get(); - + // Reset state to Started to allow re-registration. + registry4.state.set(BrokerRegistryImpl.State.Started); + registry4.registerAsync().get(); + + // Wait for ownership to be reassigned back to pulsar4 after re-registration + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + String reRegResult = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + assertNotNull(reRegResult); + assertEquals(reRegResult, pulsar4.getBrokerServiceUrl()); + }); result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); - assertNotNull(result); log.info("{} Namespace is re-owned by {}", slaMonitorTopic, result); - assertEquals(result, pulsar4.getBrokerServiceUrl()); producer.send("t2"); Producer producer1 = pulsar.getClient().newProducer(Schema.STRING) @@ -1327,10 +1353,10 @@ public void testLoadBalancerServiceUnitTableViewSyncer() throws Exception { assertTrue(pulsar2.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled())); makeSecondaryAsLeader(); makePrimaryAsLeader(); - Awaitility.await() + Awaitility.await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(primaryLoadManager.getServiceUnitStateTableViewSyncer() .isActive())); - Awaitility.await() + Awaitility.await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer() .isActive())); ServiceConfiguration defaultConf = getDefaultConf(); @@ -1462,31 +1488,54 @@ public void testLoadBalancerServiceUnitTableViewSyncer() throws Exception { pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); } } - // Check if the broker is available + // Simulate broker going offline: clean ownerships first (as disableBroker() does), + // then unregister from the broker registry. var wrapper = (ExtensibleLoadManagerWrapper) pulsar4.getLoadManager().get(); var loadManager4 = spy((ExtensibleLoadManagerImpl) FieldUtils.readField(wrapper, "loadManager", true)); - loadManager4.getBrokerRegistry().unregister(); + ServiceUnitStateChannel channel4 = (ServiceUnitStateChannel) + FieldUtils.readField(loadManager4, "serviceUnitStateChannel", true); + channel4.cleanOwnerships(); + // Simulate broker going offline by removing it from the broker registry. + // Set state to Closed BEFORE deleting the ZK node to prevent the notification + // handler's session-expiry recovery from auto-re-registering broker4. + // In production, the PulsarService shuts down after unregister(), so the handler + // never fires. In tests, the service stays running, creating a race. + var registry4 = (BrokerRegistryImpl) loadManager4.getBrokerRegistry(); + registry4.state.set(BrokerRegistryImpl.State.Closed); + String brokerZkPath = "/loadbalance/brokers/" + pulsar4.getBrokerId(); + pulsar4.getLocalMetadataStore().delete(brokerZkPath, Optional.empty()).get(); NamespaceName slaMonitorNamespace = getSLAMonitorNamespace(pulsar4.getBrokerId(), pulsar.getConfiguration()); String slaMonitorTopic = slaMonitorNamespace.getPersistentTopicName("test"); + // Wait for ownership to be reassigned after broker unregistration + String pulsar4BrokerUrl = pulsar4.getBrokerServiceUrl(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + String lookupResult = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + assertNotNull(lookupResult); + assertNotEquals(lookupResult, pulsar4BrokerUrl); + }); String result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); - assertNotNull(result); log.info("{} Namespace is re-owned by {}", slaMonitorTopic, result); - assertNotEquals(result, pulsar4.getBrokerServiceUrl()); Producer producer = pulsar.getClient().newProducer(Schema.STRING) .topic(slaMonitorTopic).create(); producer.send("t1"); // Test re-register broker and check the lookup result - loadManager4.getBrokerRegistry().registerAsync().get(); - + // Reset state to Started to allow re-registration. + registry4.state.set(BrokerRegistryImpl.State.Started); + registry4.registerAsync().get(); + + // Wait for ownership to be reassigned back to pulsar4 after re-registration + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + String reRegResult = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + assertNotNull(reRegResult); + assertEquals(reRegResult, pulsar4.getBrokerServiceUrl()); + }); result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); - assertNotNull(result); log.info("{} Namespace is re-owned by {}", slaMonitorTopic, result); - assertEquals(result, pulsar4.getBrokerServiceUrl()); producer.send("t2"); Producer producer1 = pulsar.getClient().newProducer(Schema.STRING) @@ -1516,10 +1565,10 @@ public void testLoadBalancerServiceUnitTableViewSyncer() throws Exception { Awaitility.await().untilAsserted(() -> assertFalse(pulsar2.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled())); makeSecondaryAsLeader(); - Awaitility.await() + Awaitility.await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> assertFalse(primaryLoadManager.getServiceUnitStateTableViewSyncer() .isActive())); - Awaitility.await() + Awaitility.await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer() .isActive())); } From 495186887d0f335f3c2ab1f2ea584b87d6711977 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Mon, 13 Apr 2026 10:42:57 +0800 Subject: [PATCH 028/201] [fix][test] Fix flaky ExtensibleLoadManagerImpl client reconnection tests: PulsarClientException$AlreadyClosedException: Client already closed (#25509) (cherry picked from commit bf459748671ce62f6cb18f0b9e3b68cc06abf4b4) --- .../ExtensibleLoadManagerImplBaseTest.java | 52 ++++++++----------- .../ExtensibleLoadManagerImplTest.java | 29 ++++++++--- ...anagerImplWithAdvertisedListenersTest.java | 47 +++++++++++------ 3 files changed, 76 insertions(+), 52 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java index b50081de89e64..e32e43cccce4c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java @@ -47,7 +47,6 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.util.FutureUtil; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -86,8 +85,28 @@ protected ServiceConfiguration getDefaultConf() { return conf; } - protected ArrayList clients = new ArrayList<>(); - private final java.util.Map originalLookupServices = new java.util.HashMap<>(); + /** + * Create fresh PulsarClient instances for use within a single test method. + * Each test creates and closes its own clients to avoid shared mutable state + * that causes "Client already closed" flakiness. + */ + protected List createTestClients(int count) throws Exception { + List testClients = new ArrayList<>(); + for (int i = 0; i < count; i++) { + testClients.add(pulsarClient(lookupUrl.toString(), 100)); + } + return testClients; + } + + protected static void closeTestClients(List testClients) { + for (PulsarClient client : testClients) { + try { + client.close(); + } catch (Exception e) { + // ignore + } + } + } @DataProvider(name = "serviceUnitStateTableViewClassName") public static Object[][] serviceUnitStateTableViewClassName() { @@ -151,13 +170,6 @@ protected void setup() throws Exception { admin.namespaces().setNamespaceReplicationClusters(defaultTestNamespace, Sets.newHashSet(this.conf.getClusterName()), false); lookupService = (LookupService) FieldUtils.readDeclaredField(pulsarClient, "lookup", true); - - for (int i = 0; i < 4; i++) { - PulsarClient client = pulsarClient(lookupUrl.toString(), 100); - clients.add(client); - originalLookupServices.put(client, - (LookupService) FieldUtils.readDeclaredField(client, "lookup", true)); - } } private static PulsarClient pulsarClient(String url, int intervalInMillis) throws PulsarClientException { @@ -171,27 +183,15 @@ private static PulsarClient pulsarClient(String url, int intervalInMillis) throw @Override @AfterClass(alwaysRun = true) protected void cleanup() throws Exception { - List> futures = new ArrayList<>(); - for (PulsarClient client : clients) { - futures.add(client.closeAsync()); - } - futures.add(pulsar2.closeAsync()); - if (additionalPulsarTestContext != null) { additionalPulsarTestContext.close(); additionalPulsarTestContext = null; } super.internalCleanup(); - try { - FutureUtil.waitForAll(futures).join(); - } catch (Throwable e) { - // skip error - } pulsar1 = pulsar2 = null; primaryLoadManager = secondaryLoadManager = null; channel1 = channel2 = null; lookupService = null; - } @BeforeMethod(alwaysRun = true) @@ -199,14 +199,6 @@ protected void initializeState() throws PulsarAdminException, IllegalAccessExcep admin.namespaces().unload(defaultTestNamespace); reset(primaryLoadManager, secondaryLoadManager); FieldUtils.writeDeclaredField(pulsarClient, "lookup", lookupService, true); - // Restore original lookup services for all shared clients to prevent state leakage - // between tests when a previous test fails before resetting spied lookup services. - for (PulsarClient client : clients) { - LookupService original = originalLookupServices.get(client); - if (original != null) { - FieldUtils.writeDeclaredField(client, "lookup", original, true); - } - } pulsar1.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); pulsar2.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index c99443ac137a2..ea417a13c6df7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -532,8 +532,14 @@ public Object[][] isPersistentTopicSubscriptionTypeTest() { @Test(timeOut = 30_000, dataProvider = "isPersistentTopicSubscriptionTypeTest") public void testTransferClientReconnectionWithoutLookup(TopicDomain topicDomain, SubscriptionType subscriptionType) throws Exception { - testTransferClientReconnectionWithoutLookup(clients, topicDomain, subscriptionType, defaultTestNamespace, - admin, lookupUrl.toString(), pulsar1, pulsar2, primaryLoadManager, secondaryLoadManager); + var testClients = createTestClients(4); + try { + testTransferClientReconnectionWithoutLookup(testClients, topicDomain, subscriptionType, + defaultTestNamespace, admin, lookupUrl.toString(), pulsar1, pulsar2, + primaryLoadManager, secondaryLoadManager); + } finally { + closeTestClients(testClients); + } } @Test(enabled = false) @@ -689,8 +695,13 @@ public static void testTransferClientReconnectionWithoutLookup( @Test(timeOut = 30 * 1000, dataProvider = "isPersistentTopicSubscriptionTypeTest") public void testUnloadClientReconnectionWithLookup(TopicDomain topicDomain, SubscriptionType subscriptionType) throws Exception { - testUnloadClientReconnectionWithLookup(clients, topicDomain, subscriptionType, defaultTestNamespace, - admin, lookupUrl.toString(), pulsar1); + var testClients = createTestClients(1); + try { + testUnloadClientReconnectionWithLookup(testClients, topicDomain, subscriptionType, + defaultTestNamespace, admin, lookupUrl.toString(), pulsar1); + } finally { + closeTestClients(testClients); + } } @Test(enabled = false) @@ -711,6 +722,7 @@ public static void testUnloadClientReconnectionWithLookup(List cli PulsarClient pulsarClient = null; try { pulsarClient = clients.get(0); + @Cleanup var producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); var consumerCount = subscriptionType == SubscriptionType.Exclusive ? 1 : 3; @@ -785,8 +797,13 @@ public Object[][] isPersistentTopicTest() { @Test(timeOut = 30 * 1000, dataProvider = "isPersistentTopicTest") public void testOptimizeUnloadDisable(TopicDomain topicDomain) throws Exception { - testOptimizeUnloadDisable(clients, topicDomain, defaultTestNamespace, admin, lookupUrl.toString(), pulsar1, - pulsar2); + var testClients = createTestClients(1); + try { + testOptimizeUnloadDisable(testClients, topicDomain, defaultTestNamespace, admin, + lookupUrl.toString(), pulsar1, pulsar2); + } finally { + closeTestClients(testClients); + } } @Test(enabled = false) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplWithAdvertisedListenersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplWithAdvertisedListenersTest.java index 54079c71c664c..ac1edb6af69e2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplWithAdvertisedListenersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplWithAdvertisedListenersTest.java @@ -72,23 +72,33 @@ public Object[][] isPersistentTopicSubscriptionTypeTest() { @Test(timeOut = 30_000, dataProvider = "isPersistentTopicSubscriptionTypeTest") public void testTransferClientReconnectionWithoutLookup(TopicDomain topicDomain, SubscriptionType subscriptionType) throws Exception { - ExtensibleLoadManagerImplTest.testTransferClientReconnectionWithoutLookup( - clients, - topicDomain, subscriptionType, - defaultTestNamespace, admin, - brokerServiceUrl, - pulsar1, pulsar2, primaryLoadManager, secondaryLoadManager); + var testClients = createTestClients(4); + try { + ExtensibleLoadManagerImplTest.testTransferClientReconnectionWithoutLookup( + testClients, + topicDomain, subscriptionType, + defaultTestNamespace, admin, + brokerServiceUrl, + pulsar1, pulsar2, primaryLoadManager, secondaryLoadManager); + } finally { + closeTestClients(testClients); + } } @Test(timeOut = 30 * 1000, dataProvider = "isPersistentTopicSubscriptionTypeTest") public void testUnloadClientReconnectionWithLookup(TopicDomain topicDomain, SubscriptionType subscriptionType) throws Exception { - ExtensibleLoadManagerImplTest.testUnloadClientReconnectionWithLookup( - clients, - topicDomain, subscriptionType, - defaultTestNamespace, admin, - brokerServiceUrl, - pulsar1); + var testClients = createTestClients(1); + try { + ExtensibleLoadManagerImplTest.testUnloadClientReconnectionWithLookup( + testClients, + topicDomain, subscriptionType, + defaultTestNamespace, admin, + brokerServiceUrl, + pulsar1); + } finally { + closeTestClients(testClients); + } } @DataProvider(name = "isPersistentTopicTest") @@ -98,10 +108,15 @@ public Object[][] isPersistentTopicTest() { @Test(timeOut = 30 * 1000, dataProvider = "isPersistentTopicTest") public void testOptimizeUnloadDisable(TopicDomain topicDomain) throws Exception { - ExtensibleLoadManagerImplTest.testOptimizeUnloadDisable( - clients, - topicDomain, defaultTestNamespace, admin, - brokerServiceUrl, pulsar1, pulsar2); + var testClients = createTestClients(1); + try { + ExtensibleLoadManagerImplTest.testOptimizeUnloadDisable( + testClients, + topicDomain, defaultTestNamespace, admin, + brokerServiceUrl, pulsar1, pulsar2); + } finally { + closeTestClients(testClients); + } } } From 205f1252e2234aa3d2f87283eb47049f849c34b4 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 9 Apr 2026 11:17:29 +0300 Subject: [PATCH 029/201] [fix][test] Fix flaky ExtensibleLoadManagerTest.startBroker timeout (#25500) (cherry picked from commit 12bcd6bcc9e8379c49555b1d40e6103de9b26384) --- .../ExtensibleLoadManagerTest.java | 55 +++++++++++-------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index 90301a1861712..33f72d7a2ad9e 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -135,33 +135,40 @@ public void cleanup() { @BeforeMethod(alwaysRun = true) public void startBroker() { - if (pulsarCluster != null) { - pulsarCluster.getBrokers().forEach(brokerContainer -> { - if (!brokerContainer.isRunning()) { - brokerContainer.start(); - } - }); - String topicName = "persistent://" + DEFAULT_NAMESPACE + "/startBrokerCheck"; - Awaitility.await().atMost(120, TimeUnit.SECONDS).ignoreExceptions().until( - () -> { - for (BrokerContainer brokerContainer : pulsarCluster.getBrokers()) { - try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl( - brokerContainer.getHttpServiceUrl()).build()) { - if (admin.brokers().getActiveBrokers(clusterName).size() != NUM_BROKERS) { - return false; - } - try { - admin.topics().createPartitionedTopic(topicName, 10); - } catch (PulsarAdminException.ConflictException e) { - // expected - } - admin.lookups().lookupPartitionedTopic(topicName); + if (pulsarCluster == null) { + return; + } + pulsarCluster.getBrokers().forEach(brokerContainer -> { + if (!brokerContainer.isRunning()) { + brokerContainer.start(); + } + }); + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/startBrokerCheck"; + Awaitility.await().atMost(180, TimeUnit.SECONDS).until( + () -> { + for (BrokerContainer brokerContainer : pulsarCluster.getBrokers()) { + try (PulsarAdmin brokerAdmin = PulsarAdmin.builder().serviceHttpUrl( + brokerContainer.getHttpServiceUrl()).build()) { + if (brokerAdmin.brokers().getActiveBrokers(clusterName).size() != NUM_BROKERS) { + log.info("Broker {} does not see {} active brokers yet", + brokerContainer.getHostName(), NUM_BROKERS); + return false; + } + try { + brokerAdmin.topics().createPartitionedTopic(topicName, 10); + } catch (PulsarAdminException.ConflictException e) { + // expected - topic already exists } + brokerAdmin.lookups().lookupPartitionedTopic(topicName); + } catch (Exception e) { + log.warn("Broker {} is not ready yet: {}", + brokerContainer.getHostName(), e.getMessage()); + return false; } - return true; } - ); - } + return true; + } + ); } @Test(timeOut = 40 * 1000) From a62103d89c7aa103c2452e397ebb05f3c045e71e Mon Sep 17 00:00:00 2001 From: Vishal Kumar Singh Date: Wed, 22 Apr 2026 04:01:05 +0530 Subject: [PATCH 030/201] [fix][broker] Lower log level of DrainingHashesTracker not-found entry to DEBUG (#25558) (cherry picked from commit 423a74698e1b44652425f030788ac54484052f85) --- .../apache/pulsar/broker/service/DrainingHashesTracker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/DrainingHashesTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/DrainingHashesTracker.java index 51c458173689b..8570e2f4d36fb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/DrainingHashesTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/DrainingHashesTracker.java @@ -185,8 +185,8 @@ public void updateConsumerStats(Consumer consumer, ConsumerStatsImpl consumerSta int hash = hashIterator.nextInt(); DrainingHashEntry entry = getEntry(hash); if (entry == null) { - log.warn("[{}] Draining hash {} not found in the tracker for consumer {}", dispatcherName, hash, - consumer); + log.debug("[{}] Draining hash {} not found in the tracker for consumer {}", dispatcherName, + hash, consumer); continue; } int unackedMessages = entry.getRefCount(); From 96b8d3b634460d0369a56ac761f7bb6462910f8c Mon Sep 17 00:00:00 2001 From: Wenzhi Feng Date: Wed, 15 Apr 2026 14:28:56 +0800 Subject: [PATCH 031/201] [fix][broker] Unthrottle producers immediately when publish rate limiting is disabled (#25502) Co-authored-by: fengwenzhi (cherry picked from commit 85937d8ea8ecf07480f51df6c76654370e35c870) --- .../service/PublishRateLimiterImpl.java | 20 ++- .../service/PublishRateLimiterTest.java | 124 ++++++++++++++++++ 2 files changed, 142 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java index 096418191dc44..5d8fb4687b269 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java @@ -43,6 +43,12 @@ public class PublishRateLimiterImpl implements PublishRateLimiter { private final AtomicInteger throttledProducersCount = new AtomicInteger(0); private final AtomicBoolean processingQueuedProducers = new AtomicBoolean(false); + + /** + * Executor used for the last {@link #scheduleUnthrottling} from this limiter (set when throttling starts). + * Used to schedule an immediate follow-up run after publish-rate limits change. + */ + private volatile ScheduledExecutorService lastUnthrottleExecutor; private final Consumer throttleAction; private final Consumer unthrottleAction; @@ -88,6 +94,7 @@ private void scheduleDecrementThrottleCount(Producer producer) { // this is to avoid scheduling unthrottling multiple times for concurrent producers if (throttledProducersCount.incrementAndGet() == 1) { ScheduledExecutorService executor = producer.getCnx().getBrokerService().executor().next(); + lastUnthrottleExecutor = executor; scheduleUnthrottling(executor, calculateThrottlingDurationNanos()); } } @@ -167,12 +174,18 @@ public void update(Policies policies, String clusterName) { update(maxPublishRate); } + private void scheduleImmediateUnthrottling() { + ScheduledExecutorService executor = lastUnthrottleExecutor; + if (executor != null) { + scheduleUnthrottling(executor, 0L); + } + } + public void update(PublishRate maxPublishRate) { if (maxPublishRate != null) { updateTokenBuckets(maxPublishRate.publishThrottlingRateInMsg, maxPublishRate.publishThrottlingRateInByte); } else { - tokenBucketOnMessage = null; - tokenBucketOnByte = null; + updateTokenBuckets(0L, 0L); } } @@ -189,6 +202,9 @@ protected void updateTokenBuckets(long publishThrottlingRateInMsg, long publishT } else { tokenBucketOnByte = null; } + // After any bucket rebuild, wake unthrottling: + // old scheduled delay may be invalid and cause unnecessary wait time for producers to be unthrottled. + scheduleImmediateUnthrottling(); } @VisibleForTesting diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java index 573e3980c7383..270286419f754 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java @@ -19,19 +19,26 @@ package org.apache.pulsar.broker.service; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.DefaultEventLoop; import io.netty.channel.EventLoop; import io.netty.channel.EventLoopGroup; import io.netty.util.concurrent.DefaultThreadFactory; +import io.netty.util.concurrent.ScheduledFuture; import java.util.HashMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; import org.testng.annotations.AfterMethod; @@ -149,4 +156,121 @@ public void testPublishRateLimiterImplUpdate() throws Exception { }); future.get(5, TimeUnit.SECONDS); } + + /** + * When the token bucket is deeply depleted, the first scheduled unthrottle uses a long delay. Disabling limits + * must schedule an immediate unthrottle (delay 0) so producers are not stuck until that delay elapses. + */ + @Test + public void shouldUnthrottleImmediatelyAfterDisablingLimitsDespiteLongPendingDelay() { + AtomicLong manualClock = new AtomicLong(TimeUnit.SECONDS.toNanos(100)); + AtomicInteger unthrottleCalls = new AtomicInteger(); + + PublishRateLimiterImpl limiter = new PublishRateLimiterImpl( + manualClock::get, + p -> { }, + p -> unthrottleCalls.incrementAndGet()); + + EventLoop scheduler = mock(EventLoop.class); + AtomicInteger longDelaySchedules = new AtomicInteger(); + doAnswer(invocation -> { + Runnable task = invocation.getArgument(0); + long delay = invocation.getArgument(1); + TimeUnit unit = invocation.getArgument(2); + long delayNanos = unit.toNanos(delay); + if (delayNanos == 0L) { + task.run(); + } else { + longDelaySchedules.incrementAndGet(); + } + @SuppressWarnings("unchecked") + ScheduledFuture scheduled = mock(ScheduledFuture.class); + return scheduled; + }).when(scheduler).schedule(any(Runnable.class), anyLong(), any()); + + Producer p = mock(Producer.class); + ServerCnx cnx = mock(ServerCnx.class); + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + doAnswer(a -> ctx).when(cnx).ctx(); + doAnswer(a -> cnx).when(p).getCnx(); + when(p.getCnx()).thenReturn(cnx); + doAnswer(a -> { + ((Runnable) a.getArgument(0)).run(); + return null; + }).when(cnx).execute(any(Runnable.class)); + + BrokerService brokerService = mock(BrokerService.class); + when(cnx.getBrokerService()).thenReturn(brokerService); + EventLoopGroup eventLoopGroup = mock(EventLoopGroup.class); + when(brokerService.executor()).thenReturn(eventLoopGroup); + when(eventLoopGroup.next()).thenReturn(scheduler); + + limiter.update(new PublishRate(1, 0)); + manualClock.addAndGet(TimeUnit.SECONDS.toNanos(1)); + + limiter.handlePublishThrottling(p, 100_000, 0L); + assertEquals(unthrottleCalls.get(), 0); + assertTrue(longDelaySchedules.get() >= 1, + "Expected a long-delay unthrottle to be scheduled while the bucket is deeply depleted"); + + limiter.update(new PublishRate(0, 0)); + assertEquals(unthrottleCalls.get(), 1); + } + + /** + * Relaxing only the byte limit still invalidates a previously scheduled long unthrottle delay; an immediate + * unthrottle pass must run after buckets are rebuilt. + */ + @Test + public void shouldUnthrottleImmediatelyAfterRaisingByteLimitDespiteLongPendingDelay() { + AtomicLong manualClock = new AtomicLong(TimeUnit.SECONDS.toNanos(100)); + AtomicInteger unthrottleCalls = new AtomicInteger(); + + PublishRateLimiterImpl limiter = new PublishRateLimiterImpl( + manualClock::get, + p -> { }, + p -> unthrottleCalls.incrementAndGet()); + + EventLoop scheduler = mock(EventLoop.class); + doAnswer(invocation -> { + Runnable task = invocation.getArgument(0); + long delay = invocation.getArgument(1); + TimeUnit unit = invocation.getArgument(2); + if (unit.toNanos(delay) == 0L) { + task.run(); + } + @SuppressWarnings("unchecked") + ScheduledFuture scheduled = mock(ScheduledFuture.class); + return scheduled; + }).when(scheduler).schedule(any(Runnable.class), anyLong(), any()); + + Producer p = mock(Producer.class); + ServerCnx cnx = mock(ServerCnx.class); + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + doAnswer(a -> ctx).when(cnx).ctx(); + doAnswer(a -> cnx).when(p).getCnx(); + when(p.getCnx()).thenReturn(cnx); + doAnswer(a -> { + ((Runnable) a.getArgument(0)).run(); + return null; + }).when(cnx).execute(any(Runnable.class)); + + BrokerService brokerService = mock(BrokerService.class); + when(cnx.getBrokerService()).thenReturn(brokerService); + EventLoopGroup eventLoopGroup = mock(EventLoopGroup.class); + when(brokerService.executor()).thenReturn(eventLoopGroup); + when(eventLoopGroup.next()).thenReturn(scheduler); + + limiter.update(new PublishRate(0, 1)); + manualClock.addAndGet(TimeUnit.SECONDS.toNanos(1)); + + limiter.handlePublishThrottling(p, 0, 100_000L); + assertEquals(unthrottleCalls.get(), 0); + + limiter.update(new PublishRate(0, 1_000_000)); + assertEquals(unthrottleCalls.get(), 1); + + AsyncTokenBucket byteBucket = limiter.getTokenBucketOnByte(); + assertNotNull(byteBucket); + } } From 4787e225cff76c48d33eac9a3ab52147a358dc3c Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 21 Apr 2026 15:54:17 -0700 Subject: [PATCH 032/201] [fix][test] Recreate EventLoop in PublishRateLimiterTest setup (#25560) (cherry picked from commit 293eafc08c933bbb4ffebb37c128c4b76b6a3947) --- .../apache/pulsar/broker/service/PublishRateLimiterTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java index 270286419f754..03283865761d6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java @@ -57,11 +57,12 @@ public class PublishRateLimiterTest { private ServerCnx serverCnx; private PublishRateLimiterImpl publishRateLimiter; private ServerCnxThrottleTracker throttleTracker; - private DefaultThreadFactory threadFactory = new DefaultThreadFactory("pulsar-io"); - private EventLoop eventLoop = new DefaultEventLoop(threadFactory); + private final DefaultThreadFactory threadFactory = new DefaultThreadFactory("pulsar-io"); + private EventLoop eventLoop; @BeforeMethod public void setup() throws Exception { + eventLoop = new DefaultEventLoop(threadFactory); policies.publishMaxMessageRate = new HashMap<>(); policies.publishMaxMessageRate.put(CLUSTER_NAME, publishRate); manualClockSource = new AtomicLong(TimeUnit.SECONDS.toNanos(100)); From 7de44804eed4f7aa38bfa3dea61518faaff44d96 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 6 Apr 2026 19:47:23 -0700 Subject: [PATCH 033/201] [fix][test] Fix flaky BrokerRegistryIntegrationTest port binding race (#25463) (cherry picked from commit aaf2defe4bad3fc06a080718b53599bd2aeb4333) --- .../extensions/BrokerRegistryIntegrationTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java index e975671fa12e8..588b1c7730e78 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Optional; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.util.PortManager; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LoadManager; @@ -41,14 +40,14 @@ public class BrokerRegistryIntegrationTest { private static final String clusterName = "test"; - private final int zkPort = PortManager.nextFreePort(); - private final LocalBookkeeperEnsemble bk = new LocalBookkeeperEnsemble(2, zkPort, PortManager::nextFreePort); + private LocalBookkeeperEnsemble bk; private PulsarService pulsar; private BrokerRegistry brokerRegistry; private String brokerMetadataPath; @BeforeClass protected void setup() throws Exception { + bk = new LocalBookkeeperEnsemble(2, 0, () -> 0); bk.start(); pulsar = new PulsarService(brokerConfig()); pulsar.start(); @@ -68,7 +67,9 @@ protected void cleanup() throws Exception { pulsar.close(); } final var elapsedMs = System.currentTimeMillis() - startMs; - bk.stop(); + if (bk != null) { + bk.stop(); + } if (elapsedMs > 5000) { throw new RuntimeException("Broker took " + elapsedMs + "ms to close"); } From 35049645986c10b2521a8cbf4de0e950ea9836ae Mon Sep 17 00:00:00 2001 From: Rahul <84563516+programmerahul@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:43:50 +0530 Subject: [PATCH 034/201] [fix][broker] pulsar admin stats internal with metadata command (#25557) Co-authored-by: Rahul Prasad (cherry picked from commit 9cf1cc9b5bf5d5581fe4e40318b85dfdd2ec77f7) --- .../pulsar/broker/service/persistent/PersistentTopic.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index cb7157314c080..91264a8f7c3f7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3029,6 +3029,9 @@ public CompletableFuture getInternalStats(boolean info.ledgerId = compactedTopicContext.getLedger().getId(); info.entries = compactedTopicContext.getLedger().getLastAddConfirmed() + 1; info.size = compactedTopicContext.getLedger().getLength(); + if (includeLedgerMetadata) { + info.metadata = compactedTopicContext.getLedger().getLedgerMetadata().toSafeString(); + } } stats.compactedLedger = info; @@ -3110,7 +3113,7 @@ public CompletableFuture getInternalStats(boolean schemaLedgerInfo.entries = metadata.getLastEntryId() + 1; schemaLedgerInfo.size = metadata.getLength(); if (includeLedgerMetadata) { - info.metadata = metadata.toSafeString(); + schemaLedgerInfo.metadata = metadata.toSafeString(); } stats.schemaLedgers.add(schemaLedgerInfo); completableFuture.complete(null); From 1ee8793b25ff03d957548d9da8863002bfed61fc Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 22 Apr 2026 03:28:49 -0700 Subject: [PATCH 035/201] [fix][test] Relax BrokerRegistryIntegrationTest broker-close threshold (#25562) (cherry picked from commit f26e3bac7d44c9c16bc0eb7f4b2720d763cf7eb1) --- .../extensions/BrokerRegistryIntegrationTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java index 588b1c7730e78..1c9d9a9980424 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java @@ -70,7 +70,11 @@ protected void cleanup() throws Exception { if (bk != null) { bk.stop(); } - if (elapsedMs > 5000) { + // Guard against regressions where the broker hangs on shutdown (e.g. blocked health + // checks or stuck registrations). Keep the threshold generous enough to tolerate CI + // variability in BrokerRegistryMetadataStoreIntegrationTest, while still catching the + // multi-minute hangs this assertion was originally added to detect. + if (elapsedMs > 15000) { throw new RuntimeException("Broker took " + elapsedMs + "ms to close"); } } From 8fbe0f2c77815b7abc8f2682508435ae642f6f42 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 22 Apr 2026 04:23:53 -0700 Subject: [PATCH 036/201] [fix][test] Extend SameAuthParamsLookupAutoClusterFailoverTest phase timeouts (#25563) (cherry picked from commit 43c7ad86ebfc0c7c47f0f6ccc92c5f0aa62177a3) --- .../SameAuthParamsLookupAutoClusterFailoverTest.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java index 0a2a74d9a22be..755b3edaa3c15 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -63,7 +63,11 @@ public Object[][] enabledTls () { }; } - @Test(dataProvider = "enabledTls", timeOut = 240 * 1000) + // Each state-convergence phase below waits up to 2 minutes. With 3 phases plus + // cluster startup/teardown, the 240s overall timeout can be tight on slow CI agents + // (the probe timeout is 3s and recoverThreshold=5, so a single slow probe can + // stretch a phase to ~30s). Allow 8 minutes overall to match the per-phase budget. + @Test(dataProvider = "enabledTls", timeOut = 480 * 1000) public void testAutoClusterFailover(boolean enabledTls) throws Exception { // Start clusters. setup(); @@ -116,7 +120,7 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { // Test failover 0 --> 2. pulsar1.close(); - Awaitility.await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { + Awaitility.await().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { CompletableFuture checkStatesFuture2 = new CompletableFuture<>(); executor.submit(() -> { boolean res = stateArray[0] == PulsarServiceState.Failed; @@ -133,7 +137,7 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { executor.execute(() -> { urlArray[1] = url2; }); - Awaitility.await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { + Awaitility.await().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { CompletableFuture checkStatesFuture3 = new CompletableFuture<>(); executor.submit(() -> { boolean res = stateArray[0] == PulsarServiceState.Failed; @@ -150,7 +154,7 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { executor.execute(() -> { urlArray[0] = url2; }); - Awaitility.await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { + Awaitility.await().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { CompletableFuture checkStatesFuture4 = new CompletableFuture<>(); executor.submit(() -> { boolean res = stateArray[0] == PulsarServiceState.Healthy; From ff7054f3dcce6c23d98336ac561889441cf6175f Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 22 Apr 2026 04:26:54 -0700 Subject: [PATCH 037/201] [fix][test] Fix flaky OffloadPrefixTest.testPositionOnEdgeOfLedger race with ledger rollover (#25561) (cherry picked from commit 77999fc8a9464b5af7ba0e541d3beb5906f83714) --- .../apache/bookkeeper/mledger/impl/OffloadPrefixTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java index 38d7c17b9d49c..b9c95f445cc33 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java @@ -56,6 +56,7 @@ import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -234,7 +235,11 @@ public void testPositionOnEdgeOfLedger() throws Exception { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } - assertEquals(ledger.getLedgersInfoAsList().size(), 2); + // After filling exactly 2 ledgers, the 2nd is closed and a 3rd empty ledger starts + // being created asynchronously. Wait for that to finish so the rest of the test runs + // against a stable state (l1 full, l2 full, l3 empty) instead of racing with rollover. + Awaitility.await().untilAsserted(() -> + assertEquals(ledger.getLedgersInfoAsList().size(), 3)); Position p = ledger.getLastConfirmedEntry(); // position at end of second ledger From 83427869cdd199b227822203695e6fb4ecadf7e2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 22 Apr 2026 16:50:28 +0300 Subject: [PATCH 038/201] [fix][test] Flaky SameAuthParamsLookupAutoClusterFailoverTest (#25566) (cherry picked from commit 1071cc94b62921ede294c17391e19fca69c289d6) --- .../broker/SameAuthParamsLookupAutoClusterFailoverTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java index 755b3edaa3c15..febd0c4aad0fa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -83,7 +83,7 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { .pulsarServiceUrlArray(urlArray) .failoverThreshold(5) .recoverThreshold(5) - .checkHealthyIntervalMs(300) + .checkHealthyIntervalMs(100) .testTopic("a/b/c") .markTopicNotFoundAsAvailable(true) .build(); From ec45e06d580efef49cfeb012c793a95881a18aa9 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Thu, 23 Apr 2026 04:47:27 +0800 Subject: [PATCH 039/201] [improve][broker] Use full bundle name for namespace bundle destination affinity in ModularLoadManagerImpl (#25518) (cherry picked from commit db94b9534f0900a5ec4eb9fa5191296513aa0575) --- .../broker/admin/impl/NamespacesBase.java | 6 +++++- .../impl/ModularLoadManagerWrapper.java | 3 +-- .../impl/ModularLoadManagerImplTest.java | 20 +++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 53d8bc49d9123..0b8a0591ae12b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -1395,6 +1395,7 @@ public CompletableFuture setNamespaceBundleAffinityAsync(String bundleRang if (StringUtils.isBlank(destinationBroker)) { return CompletableFuture.completedFuture(null); } + return pulsar().getLoadManager().get().getAvailableBrokersAsync() .thenCompose(brokers -> { if (!brokers.contains(destinationBroker)) { @@ -1415,8 +1416,11 @@ public CompletableFuture setNamespaceBundleAffinityAsync(String bundleRang if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar())) { return; } + final String bundleName = pulsar().getNamespaceService().getNamespaceBundleFactory() + .getBundle(namespaceName.toString(), bundleRange) + .toString(); // For ExtensibleLoadManager, this operation will be ignored. - pulsar().getLoadManager().get().setNamespaceBundleAffinity(bundleRange, destinationBroker); + pulsar().getLoadManager().get().setNamespaceBundleAffinity(bundleName, destinationBroker); }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java index c8d81bda1bc13..b5bb4d52717f9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java @@ -64,8 +64,7 @@ public LoadManagerReport generateLoadReport() { @Override public Optional getLeastLoaded(final ServiceUnitId serviceUnit) { - String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(serviceUnit.toString()); - String affinityBroker = loadManager.setNamespaceBundleAffinity(bundleRange, null); + String affinityBroker = loadManager.setNamespaceBundleAffinity(serviceUnit.toString(), null); if (!StringUtils.isBlank(affinityBroker)) { return Optional.of(buildBrokerResourceUnit(affinityBroker)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java index da12c84e19791..1bf40fab1a111 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java @@ -413,6 +413,26 @@ public void testBrokerAffinity() throws Exception { Assert.assertEquals(brokerServiceUrl, topicLookupAfterUnload); } + @Test + public void testBrokerAffinityLookupUsesFullBundleName() throws Exception { + String affinityBroker1 = "affinity-broker-1"; + String affinityBroker2 = "affinity-broker-2"; + NamespaceBundle bundle1 = makeBundle("tenant-1", "ns-1"); + NamespaceBundle bundle2 = makeBundle("tenant-1", "ns-2"); + + LoadManager wrapper = pulsar1.getLoadManager().get(); + wrapper.setNamespaceBundleAffinity(bundle1.toString(), affinityBroker1); + wrapper.setNamespaceBundleAffinity(bundle2.toString(), affinityBroker2); + + Optional leastLoadedBroker1 = wrapper.getLeastLoaded(bundle1); + Optional leastLoadedBroker2 = wrapper.getLeastLoaded(bundle2); + + Assert.assertTrue(leastLoadedBroker1.isPresent()); + Assert.assertTrue(leastLoadedBroker2.isPresent()); + Assert.assertEquals(leastLoadedBroker1.get().getResourceId(), affinityBroker1); + Assert.assertEquals(leastLoadedBroker2.get().getResourceId(), affinityBroker2); + } + /** * It verifies that once broker owns max-number of topics: load-manager doesn't allocates new bundles to that broker * unless all the brokers are in same state. From 1fb3a6b5343b7d52e70f112c18bcac62dbc0221d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 23 Apr 2026 04:59:16 +0300 Subject: [PATCH 040/201] [fix][sec] Upgrade BouncyCastle to 1.84 (CVE-2026-5588, CVE-2026-0636) (#25569) (cherry picked from commit a14f6ccf31464a645dcb9af012ccdde73fc5124e) --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 12 ++++----- .../server/src/assemble/LICENSE.bin.txt | 6 ++--- .../shell/src/assemble/LICENSE.bin.txt | 6 ++--- pom.xml | 27 ++++++++++++++----- 5 files changed, 32 insertions(+), 21 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 7b4538d1aa3fd..04284f49ddef3 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -52,7 +52,7 @@ org.bouncycastle - bcprov-ext-jdk18on + bcprov-jdk18on diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index b07fbe49221e1..846f2b45f77ba 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -32,11 +32,6 @@ Apache Pulsar :: Bouncy Castle :: BC-FIPS - - org.bouncycastle - bcutil-fips - 2.0.5 - ${project.groupId} pulsar-common @@ -53,13 +48,16 @@ org.bouncycastle bc-fips - ${bouncycastle.bc-fips.version} org.bouncycastle bcpkix-fips - ${bouncycastle.bcpkix-fips.version} + + + + org.bouncycastle + bcutil-fips diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 6645d0d5392fe..c18f46674879d 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -634,9 +634,9 @@ Public Domain (CC0) -- ../licenses/LICENSE-CC0.txt Bouncy Castle License * Bouncy Castle -- ../licenses/LICENSE-bouncycastle.txt - - org.bouncycastle-bcpkix-jdk18on-1.81.jar - - org.bouncycastle-bcprov-jdk18on-1.78.1.jar - - org.bouncycastle-bcutil-jdk18on-1.81.jar + - org.bouncycastle-bcpkix-jdk18on-1.84.jar + - org.bouncycastle-bcprov-jdk18on-1.84.jar + - org.bouncycastle-bcutil-jdk18on-1.84.jar ------------------------ diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index b86a20eb9f3c6..38de80b932c6d 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -469,9 +469,9 @@ Public Domain (CC0) -- ../licenses/LICENSE-CC0.txt Bouncy Castle License * Bouncy Castle -- ../licenses/LICENSE-bouncycastle.txt - - bcpkix-jdk18on-1.81.jar - - bcprov-jdk18on-1.78.1.jar - - bcutil-jdk18on-1.81.jar + - bcpkix-jdk18on-1.84.jar + - bcprov-jdk18on-1.84.jar + - bcutil-jdk18on-1.84.jar ------------------------ diff --git a/pom.xml b/pom.xml index c505d2e6f253d..9404694088dfe 100644 --- a/pom.xml +++ b/pom.xml @@ -204,11 +204,12 @@ flexible messaging model and an intuitive client API. 4.5.0 2.25.4 - 1.78.1 - 1.81 - 1.81 - 1.78.1 - 2.0.10 + 1.84 + ${bouncycastle.version} + ${bouncycastle.version} + ${bouncycastle.version} + 2.0.11 + 2.0.6 2.0.1 2.18.6 8.5.16 @@ -999,8 +1000,20 @@ flexible messaging model and an intuitive client API. org.bouncycastle - bcprov-ext-jdk18on - ${bouncycastle.bcprov-ext-jdk18on.version} + bc-fips + ${bouncycastle.bc-fips.version} + + + + org.bouncycastle + bcpkix-fips + ${bouncycastle.bcpkix-fips.version} + + + + org.bouncycastle + bcutil-fips + ${bouncycastle.bcutil-fips.version} From aabd80bb483810f9597c82fcf20808a5980bbb19 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 23 Apr 2026 12:31:16 +0300 Subject: [PATCH 041/201] Release 4.2.1 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- jetcd-core-shaded/pom.xml | 2 +- jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml | 2 +- jetty-upgrade/pom.xml | 2 +- jetty-upgrade/zookeeper-prometheus-metrics/pom.xml | 2 +- jetty-upgrade/zookeeper-with-patched-admin/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- microbench/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-bom/pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-cli-utils/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-dependencies-minimized/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-docs-tools/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/azure-data-explorer/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis-kpl-shaded/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-opentelemetry/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 135 files changed, 138 insertions(+), 138 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 04284f49ddef3..8fecf8a65b565 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.1-SNAPSHOT + 4.2.1 bouncy-castle-bc diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index fac2c9def607d..f0e07c5c8ad27 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.1-SNAPSHOT + 4.2.1 bcfips-include-test diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 846f2b45f77ba..0446dcd609fbb 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.1-SNAPSHOT + 4.2.1 bouncy-castle-bcfips diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index e85913fc1976f..3b773df4e3801 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.1-SNAPSHOT + 4.2.1 diff --git a/buildtools/pom.xml b/buildtools/pom.xml index f066747948794..6f9f8c2bf077e 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 4.2.1-SNAPSHOT + 4.2.1 jar Pulsar Build Tools @@ -43,7 +43,7 @@ - 2026-04-15T15:54:00Z + 2026-04-23T09:31:13Z 1.8 1.8 8 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index e27ac2f154bb3..34ac94b564bb7 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.1-SNAPSHOT + 4.2.1 pulsar-io-distribution diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 80f523d9ea207..d146025b55269 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.1-SNAPSHOT + 4.2.1 pulsar-offloader-distribution diff --git a/distribution/pom.xml b/distribution/pom.xml index 0b9c17ec902bc..43b6eccb3a3db 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.1-SNAPSHOT + 4.2.1 distribution diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 3422b25323e33..66705af2819fc 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.1-SNAPSHOT + 4.2.1 pulsar-server-distribution diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index e68805857a59b..e5ae96bb5bba1 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.1-SNAPSHOT + 4.2.1 pulsar-shell-distribution diff --git a/docker/pom.xml b/docker/pom.xml index fe8fbe27b9189..e4e627e1a0aaa 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.1-SNAPSHOT + 4.2.1 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index e42af25f2fc6d..b84aac69c06ae 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.1-SNAPSHOT + 4.2.1 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index d8cd369156090..da84c97de1930 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.1-SNAPSHOT + 4.2.1 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 0c39663aae3f4..096907895925b 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.1-SNAPSHOT + 4.2.1 jclouds-shaded diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml index aad21693fe241..ce6ccfb9581cd 100644 --- a/jetcd-core-shaded/pom.xml +++ b/jetcd-core-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.1-SNAPSHOT + 4.2.1 jetcd-core-shaded diff --git a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml index cfa40d1acef45..ea8c8de0e1c77 100644 --- a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml +++ b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.1-SNAPSHOT + 4.2.1 pulsar-bookkeeper-prometheus-metrics-provider Apache Pulsar :: BookKeeper Stats Providers :: Prometheus diff --git a/jetty-upgrade/pom.xml b/jetty-upgrade/pom.xml index ef7bb6d489ef9..93f19e07c7af7 100644 --- a/jetty-upgrade/pom.xml +++ b/jetty-upgrade/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.1-SNAPSHOT + 4.2.1 jetty-upgrade diff --git a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml index 376c5a3779aa5..f8f48ef2ce610 100755 --- a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml +++ b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.1-SNAPSHOT + 4.2.1 pulsar-zookeeper-prometheus-metrics diff --git a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml index f5cf6273e79ab..9807f5f98fffb 100644 --- a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml +++ b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar jetty-upgrade - 4.2.1-SNAPSHOT + 4.2.1 zookeeper-with-patched-admin diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 22e2b9cedff5a..fcd106225e954 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.1-SNAPSHOT + 4.2.1 managed-ledger diff --git a/microbench/pom.xml b/microbench/pom.xml index 97675fc284b7a..180a491833fe4 100644 --- a/microbench/pom.xml +++ b/microbench/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.1-SNAPSHOT + 4.2.1 microbench diff --git a/pom.xml b/pom.xml index 9404694088dfe..0245aed552b87 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 4.2.1-SNAPSHOT + 4.2.1 Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -94,7 +94,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - 2026-04-15T15:53:54Z + 2026-04-23T09:31:08Z true 4.1.12.1 5.7.1 - 4.1.132.Final + 4.1.133.Final 0.0.26.Final 12.1.8 From d04ed91cdcb5c70cd400acecf784b67ab619c321 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 6 May 2026 00:42:40 +0800 Subject: [PATCH 048/201] [fix][broker]Replication is stuck because failed to read entries (#25625) (cherry picked from commit efc9299dcd67630373316a9689e06e43bbabe0fc) --- .../mledger/impl/ManagedLedgerTest.java | 21 +++++- .../persistent/PersistentReplicator.java | 11 +-- .../broker/service/OneWayReplicatorTest.java | 73 +++++++++++++++++++ ...yReplicatorUsingGlobalPartitionedTest.java | 6 ++ .../OneWayReplicatorUsingGlobalZKTest.java | 6 ++ 5 files changed, 111 insertions(+), 6 deletions(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 353768a964ebc..a03de59736e9d 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -87,6 +88,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; +import java.util.function.Supplier; import lombok.Cleanup; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -167,7 +169,7 @@ public Object[][] checkOwnershipFlagProvider() { return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; } - private void makeAddEntryTimeout(ManagedLedgerImpl ml, AtomicBoolean addEntryFinished) throws Exception { + public static void makeAddEntryTimeout(ManagedLedgerImpl ml, AtomicBoolean addEntryFinished) throws Exception { LedgerHandle currentLedger = ml.currentLedger; final LedgerHandle spyLedgerHandle = spy(currentLedger); doAnswer(invocation -> { @@ -183,6 +185,23 @@ private void makeAddEntryTimeout(ManagedLedgerImpl ml, AtomicBoolean addEntryFin ml.currentLedger = spyLedgerHandle; } + public static void makeReadEntryProbFail(ManagedLedgerImpl ml, Supplier errorOrNot) + throws Exception { + ml.entryCache.clear(); + LedgerHandle currentLedger = ml.currentLedger; + final LedgerHandle spyLedgerHandle = spy(currentLedger); + doAnswer(invocation -> { + long ledgerId = (long) invocation.getArguments()[0]; + long entryId = (long) invocation.getArguments()[1]; + ManagedLedgerException mightError = errorOrNot.get(); + if (mightError != null) { + return CompletableFuture.failedFuture(mightError); + } + return currentLedger.readUnconfirmedAsync(ledgerId, entryId); + }).when(spyLedgerHandle).readUnconfirmedAsync(anyLong(), anyLong()); + ml.currentLedger = spyLedgerHandle; + } + @Data private static class DeleteLedgerInfo{ volatile boolean hasCalled; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index d56f214751456..4202f812a4aac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -501,6 +501,7 @@ public CompletableFuture getFuture() { @Override public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + InFlightTask inFlightTask = (InFlightTask) ctx; if (state != Started) { log.info("[{}] Replicator was disconnected while reading entries." + " Stop reading. Replicator state: {}", @@ -521,14 +522,14 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { terminate(); return; } else if (!(exception instanceof TooManyRequestsException)) { + inFlightTask.setEntries(Collections.emptyList()); log.error("[{}] Error reading entries at {}. Retrying to read in {}s. ({})", replicatorId, ctx, waitTimeMillis / 1000.0, exception.getMessage(), exception); } else { - if (log.isDebugEnabled()) { - log.debug("[{}] Throttled by bookies while reading at {}. Retrying to read in {}s. ({})", - replicatorId, ctx, waitTimeMillis / 1000.0, exception.getMessage(), - exception); - } + inFlightTask.setEntries(Collections.emptyList()); + log.debug("[{}] Throttled by bookies while reading at {}. Retrying to read in {}s. ({})", + replicatorId, ctx, waitTimeMillis / 1000.0, exception.getMessage(), + exception); } brokerService.executor().schedule(this::readMoreEntries, waitTimeMillis, TimeUnit.MILLISECONDS); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index 195b28413736a..9358d18b27192 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -72,6 +72,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerTest; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.resources.ClusterResources; @@ -106,6 +107,7 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; import org.apache.pulsar.common.policies.data.PublishRate; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; @@ -445,6 +447,77 @@ public void testCreateRemoteConsumerFirst() throws Exception { }); } + @Test(timeOut = 45 * 1000) + public void testProbBKErrorWhenReplicating() throws Exception { + // creates topics. + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + nonReplicatedNamespace + "/tp_"); + final String subscription = "s1"; + final int totalMsg = 10_000; + admin1.topics().createNonPartitionedTopic(topicName); + admin2.topics().createNonPartitionedTopic(topicName); + RetentionPolicies retentionPolicies = new RetentionPolicies(10, -1); + admin1.topicPolicies().setRetention(topicName, retentionPolicies); + admin2.topicPolicies().setRetention(topicName, retentionPolicies); + PersistentTopic topic1 = (PersistentTopic) broker1.getTopic(topicName, false).join().get(); + ManagedLedgerImpl ml1 = (ManagedLedgerImpl) topic1.getManagedLedger(); + PersistentTopic topic2 = (PersistentTopic) broker2.getTopic(topicName, false).join().get(); + Awaitility.await().untilAsserted(() -> { + HierarchyTopicPolicies policies1 = topic1.getHierarchyTopicPolicies(); + HierarchyTopicPolicies policies2 = topic2.getHierarchyTopicPolicies(); + assertEquals(policies1.getRetentionPolicies().get().getRetentionTimeInMinutes(), 10); + assertEquals(policies2.getRetentionPolicies().get().getRetentionTimeInMinutes(), 10); + }); + // Publishes messages. + Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).create(); + Set msgPublished = new HashSet<>(); + for (int i = 0; i < totalMsg; i++) { + msgPublished.add("msg" + i); + producer1.send("msg" + i); + } + + // Inject a probable error. + AtomicInteger roundrobin = new AtomicInteger(); + Supplier bkErrorOrNot = () -> { + if (roundrobin.incrementAndGet() % 2 == 0) { + return null; + } + return new ManagedLedgerException.TooManyRequestsException("mocked error"); + }; + ManagedLedgerTest.makeReadEntryProbFail(ml1, bkErrorOrNot); + + // Verify: the replication will finish even though received ManagedLedgerException.TooManyRequestsException. + pulsar1.getConfig().setReplicationStartAt("earliest"); + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); + waitReplicatorStarted(topicName); + Awaitility.await().atMost(Duration.ofSeconds(600)).pollInterval(Duration.ofSeconds(1)).untilAsserted(() -> { + TopicStats topicStats = admin1.topics().getStats(topicName); + assertEquals(topicStats.getReplication().get(cluster2).getReplicationBacklog(), 0); + }); + + // Verify: messages were replicated. + admin2.topics().createSubscription(topicName, subscription, MessageId.earliest); + Set received = new HashSet<>(); + Consumer consumer2 = client2.newConsumer(Schema.STRING) + .subscriptionName(subscription).topic(topicName).subscribe(); + while (true) { + Message msg = consumer2.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + received.add(msg.getValue()); + } + assertEquals(received.size(), msgPublished.size()); + assertEquals(received, msgPublished); + + // cleanup. + producer1.close(); + consumer2.close(); + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1)); + waitReplicatorStopped(topicName, false); + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + } + /** * Since {@link NonPersistentReplicator} never implement the rate limitation, the config * "replicationProducerQueueSize" should not affect {@link NonPersistentReplicator}. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java index 7b59eb5098404..1dab6b1699428 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java @@ -102,6 +102,12 @@ public void testCreateRemoteConsumerFirst() throws Exception { super.testReplicatorProducerStatInTopic(); } + @Override + @Test(enabled = false) + public void testProbBKErrorWhenReplicating() throws Exception { + super.testProbBKErrorWhenReplicating(); + } + @Override @Test(enabled = false) public void testTopicCloseWhenInternalProducerCloseErrorOnce() throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java index aa8d0aabcb67a..3bda2b8c6e3f6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java @@ -315,6 +315,12 @@ public void testCreateRemoteConsumerFirst() throws Exception { super.testReplicatorProducerStatInTopic(); } + @Override + @Test(enabled = false) + public void testProbBKErrorWhenReplicating() throws Exception { + super.testProbBKErrorWhenReplicating(); + } + @Test(enabled = false) public void testTopicCloseWhenInternalProducerCloseErrorOnce() throws Exception { super.testReplicatorProducerStatInTopic(); From 70479e641abe356f2d24a41d4d51253f28eb5548 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 6 May 2026 00:43:09 +0800 Subject: [PATCH 049/201] [fix][broker][fix][broker]Replication stats is empty when the cluster is the target cluster of a one-way replication (#25583) (cherry picked from commit f43398e8b1da50dfce0306a50d3de9df05019f2b) --- .../service/persistent/PersistentTopic.java | 28 ++++++++--- .../broker/service/OneWayReplicatorTest.java | 49 +++++++++++++++++++ ...yReplicatorUsingGlobalPartitionedTest.java | 6 +++ .../OneWayReplicatorUsingGlobalZKTest.java | 6 +++ 4 files changed, 82 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 91264a8f7c3f7..03adb2a9c5ed5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -27,6 +27,7 @@ import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION; import com.carrotsearch.hppc.ObjectObjectHashMap; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; @@ -56,6 +57,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.function.BiConsumer; import java.util.function.BiFunction; import lombok.Getter; import lombok.Value; @@ -2846,24 +2848,36 @@ public CompletableFuture asyncGetStats(GetStatsOptions stats.abortedTxnCount = txnBuffer.getAbortedTxnCount(); stats.committedTxnCount = txnBuffer.getCommittedTxnCount(); - replicators.forEach((cluster, replicator) -> { - ReplicatorStatsImpl replicatorStats = replicator.computeStats(); - - // Add incoming msg rates - PublisherStatsImpl pubStats = remotePublishersStats.get(replicator.getRemoteCluster()); - if (pubStats != null) { + BiConsumer replicationInboundSetter = + (ReplicatorStatsImpl replicatorStats, PublisherStatsImpl pubStats) -> { + if (replicatorStats == null || pubStats == null) { + return; + } replicatorStats.msgRateIn = pubStats.msgRateIn; replicatorStats.msgThroughputIn = pubStats.msgThroughputIn; replicatorStats.inboundConnection = pubStats.getAddress(); replicatorStats.inboundConnectedSince = pubStats.getConnectedSince(); - } + }; + + replicators.forEach((cluster, replicator) -> { + ReplicatorStatsImpl replicatorStats = replicator.computeStats(); + // Add incoming msg rates + PublisherStatsImpl pubStats = remotePublishersStats.remove(replicator.getRemoteCluster()); + replicationInboundSetter.accept(replicatorStats, pubStats); stats.msgRateOut += replicatorStats.msgRateOut; stats.msgThroughputOut += replicatorStats.msgThroughputOut; stats.replication.put(replicator.getRemoteCluster(), replicatorStats); }); + for (ObjectObjectCursor inboundReplication : remotePublishersStats) { + PublisherStatsImpl pubStats = inboundReplication.value; + ReplicatorStatsImpl replicatorStats = new ReplicatorStatsImpl(); + replicationInboundSetter.accept(replicatorStats, pubStats); + stats.replication.put(inboundReplication.key, replicatorStats); + } + stats.storageSize = ledger.getTotalSize(); stats.backlogSize = ledger.getEstimatedBacklogSize(); stats.deduplicationStatus = messageDeduplication.getStatus().toString(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index 9358d18b27192..06c4411aae034 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -109,6 +109,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; import org.apache.pulsar.common.policies.data.PublishRate; +import org.apache.pulsar.common.policies.data.ReplicatorStats; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.TenantInfo; @@ -146,6 +147,54 @@ public void cleanup() throws Exception { super.cleanup(); } + @Test(timeOut = 45 * 1000) + public void testReceiverSideReplicationStats() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + admin1.topics().createNonPartitionedTopic(topic); + Producer producer1 = client1.newProducer(Schema.STRING).topic(topic) + .batchingMaxPublishDelay(1, TimeUnit.SECONDS).create(); + waitReplicatorStarted(topic); + + // Keep publishing to cluster-1. + AtomicBoolean keepPublishing = new AtomicBoolean(true); + Thread publisherThread = new Thread(() -> { + while (keepPublishing.get()) { + try { + producer1.send("msg"); + Thread.sleep(100); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + publisherThread.start(); + + // Verify: in-bound replication stats. + PersistentTopic persistentTopic2 = (PersistentTopic) broker2.getTopic(topic, false).join().get(); + Awaitility.await().untilAsserted(() -> { + persistentTopic2.getProducers().values().forEach(org.apache.pulsar.broker.service.Producer::updateRates); + TopicStats topicStats = admin2.topics().getStats(topic); + assertNotNull(topicStats); + assertNotNull(topicStats.getReplication()); + ReplicatorStats replicatorStats = topicStats.getReplication().get(cluster1); + assertNotNull(replicatorStats); + assertTrue(replicatorStats.getMsgRateIn() > 0); + assertTrue(replicatorStats.getMsgThroughputIn() > 0); + assertNotNull(replicatorStats.getInboundConnection()); + assertNotNull(replicatorStats.getInboundConnectedSince()); + // The connected attribute means out-bound connection so far. + assertFalse(replicatorStats.isConnected()); + }); + + // cleanup. + keepPublishing.set(false); + producer1.close(); + cleanupTopics(() -> { + admin1.topics().delete(topic); + admin2.topics().delete(topic); + }); + } + @Test(timeOut = 45 * 1000) public void testDeleteTopicWhenReplicating() throws Exception { final String topicName1 = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java index 1dab6b1699428..32ac30fece8c0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java @@ -65,6 +65,12 @@ public void cleanup() throws Exception { super.cleanup(); } + @Override + @Test(enabled = false) + public void testReceiverSideReplicationStats() throws Exception { + super.testReceiverSideReplicationStats(); + } + @Override protected void setConfigDefaults(ServiceConfiguration config, String clusterName, LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java index 3bda2b8c6e3f6..aaab17d006467 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java @@ -84,6 +84,12 @@ public void cleanup() throws Exception { super.cleanup(); } + @Override + @Test(enabled = false) + public void testReceiverSideReplicationStats() throws Exception { + super.testReceiverSideReplicationStats(); + } + protected void setConfigDefaults(ServiceConfiguration config, String clusterName, LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { super.setConfigDefaults(config, clusterName, bookkeeperEnsemble, brokerConfigZk); From 75494f05f79156f414552887a8a89a9193244bb7 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Sat, 9 May 2026 16:11:21 +0800 Subject: [PATCH 050/201] [fix][broker][branch-4.2] URL-encode sub-name in Txn pending-ack topic #25727 (#25728) --- .../service/persistent/PersistentTopic.java | 3 +- .../pendingack/impl/MLPendingAckStore.java | 11 +++++-- .../impl/MLPendingAckStoreTest.java | 30 +++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 03adb2a9c5ed5..e7a006185af77 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1277,8 +1277,7 @@ public CompletableFuture unsubscribe(String subscriptionName) { CompletableFuture unsubscribeFuture = new CompletableFuture<>(); TopicName tn = TopicName.get(MLPendingAckStore - .getTransactionPendingAckStoreSuffix(topic, - Codec.encode(subscriptionName))); + .getTransactionPendingAckStoreSuffix(topic, subscriptionName)); if (brokerService.pulsar().getConfiguration().isTransactionCoordinatorEnabled()) { ManagedLedgerConfig managedLedgerConfig = ledger.getConfig(); ManagedLedgerFactory managedLedgerFactory = getBrokerService() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index 25c7727259db3..b4f63bf64c132 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -59,6 +59,7 @@ import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.transaction.coordinator.impl.TxnBatchedPositionImpl; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriter; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterConfig; @@ -516,7 +517,13 @@ public CompletableFuture getManagedLedger() { } public static String getTransactionPendingAckStoreSuffix(String originTopicName, String subName) { - return TopicName.get(originTopicName) + "-" + subName + SystemTopicNames.PENDING_ACK_STORE_SUFFIX; + // URL-encode the subscription name so that any '/' characters it contains do not create + // extra path segments when the resulting string is parsed as a topic name. TopicName + // always decodes the local-name component on parse (via Codec.decode) and re-encodes it + // on output (via getEncodedLocalName / getPersistenceNamingEncoding), so encoding here + // produces a valid round-trip with no double-encoding. + String encodedSubName = Codec.encode(subName); + return TopicName.get(originTopicName) + "-" + encodedSubName + SystemTopicNames.PENDING_ACK_STORE_SUFFIX; } public static String getTransactionPendingAckStoreCursorName() { @@ -569,4 +576,4 @@ public ByteBuf serialize(ArrayList dataArray) { return buf; } } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTest.java index 154cbe39d408c..14e1ceade5036 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTest.java @@ -45,6 +45,9 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.common.api.proto.CommandSubscribe; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterConfig; import org.awaitility.Awaitility; @@ -140,6 +143,33 @@ private MLPendingAckStore createPendingAckStore(TxnLogBufferedWriterConfig txnLo return (MLPendingAckStore) mlPendingAckStoreProvider.newPendingAckStore(persistentSubscriptionMock).get(); } + @Test + public void testPendingAckStoreWithSlashSubscriptionName() throws Exception { + String slashSubName = "tenant/namespace/my-function"; + when(persistentSubscriptionMock.getName()).thenReturn(slashSubName); + + MLPendingAckStoreProvider provider = new MLPendingAckStoreProvider(); + + // Should not throw — subscription names containing '/' must be URL-encoded so the + // resulting pending-ack topic name is a valid V2 persistent topic name. + MLPendingAckStore store = (MLPendingAckStore) provider.newPendingAckStore(persistentSubscriptionMock).get(); + + // Verify the managed ledger persistence path encodes the subscription name correctly. + // Expected: tenant/namespace/persistent/ + // where localName = "-__transaction_pending_ack" + String originTopicName = persistentSubscriptionMock.getTopic().getName(); + TopicName origin = TopicName.get(originTopicName); + String encodedSubName = Codec.encode(slashSubName); + String expectedLocalName = origin.getLocalName() + "-" + encodedSubName + + SystemTopicNames.PENDING_ACK_STORE_SUFFIX; + // getPersistenceNamingEncoding() = tenant/namespace/persistent/encodedLocalName + String expectedMlName = origin.getTenant() + "/" + origin.getNamespacePortion() + + "/persistent/" + Codec.encode(expectedLocalName); + Assert.assertEquals(store.getManagedLedger().get().getName(), expectedMlName); + + closePendingAckStoreWithRetry(store); + } + /** * Overridden cases: * 1. Batched write and replay with batched feature. From 9deda7a1ed85c83013944c9f9b8ed24bbbb4d371 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 9 May 2026 14:32:09 +0800 Subject: [PATCH 051/201] [fix][client]Broker-side producer handle leak if closes a producer which state is regitering schema (#25725) (cherry picked from commit 4f0766708ad4d6ce027938414d6fc8e4d764343f) --- .../broker/service/OneWayReplicatorTest.java | 4 ++ .../SchemaCompatibilityCheckTest.java | 68 +++++++++++++++++++ .../pulsar/client/impl/ProducerImpl.java | 2 +- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index 06c4411aae034..ddfef968a3b14 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -40,6 +40,8 @@ import io.netty.util.concurrent.FastThreadLocalThread; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -97,9 +99,11 @@ import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.ProducerBuilderImpl; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.api.proto.CommandSendReceipt; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index b59ae89c0bf5e..6d95ae7458275 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -29,15 +29,20 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; +import org.apache.pulsar.client.api.InjectedClientCnxClientBuilder; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClient; @@ -45,7 +50,11 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.impl.ClientBuilderImpl; +import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; +import org.apache.pulsar.common.api.proto.CommandGetOrCreateSchemaResponse; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; @@ -57,6 +66,7 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.schema.MockExternalJsonSchema; import org.apache.pulsar.schema.Schemas; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -637,6 +647,64 @@ public void testConsumerWithNotCompatibilitySchema(SchemaCompatibilityStrategy s } + @Test + public void testCloseProducerWhenRegisteringNewSchema() throws Exception { + final String ns = BrokerTestUtil.newUniqueName(PUBLIC_TENANT + "/ns"); + final String topic = "persistent://" + BrokerTestUtil.newUniqueName(ns + "/tp"); + admin.namespaces().createNamespace(ns); + admin.namespaces().setSchemaCompatibilityStrategy(ns, SchemaCompatibilityStrategy.ALWAYS_INCOMPATIBLE); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(ns), + SchemaCompatibilityStrategy.ALWAYS_INCOMPATIBLE); + }); + + // Injection: Let the handling response of registering schema delay, then we have enough time to close producer + // when it's state is registering schema. + CountDownLatch handleErrorSignal = new CountDownLatch(1); + ClientBuilderImpl clientBuilder = (ClientBuilderImpl) PulsarClient.builder().serviceUrl(lookupUrl.toString()); + PulsarClient injectedReplClient = InjectedClientCnxClientBuilder.create(clientBuilder, + (conf, eventLoopGroup) -> { + return new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup) { + + @Override + protected void handleGetOrCreateSchemaResponse(CommandGetOrCreateSchemaResponse response) { + if (response.hasErrorCode()) { + try { + handleErrorSignal.await(); + } catch (InterruptedException e) { + // Nothing to do. + } + } + super.handleGetOrCreateSchemaResponse(response); + } + }; + }); + + Producer producer = injectedReplClient.newProducer(Schema.AUTO_PRODUCE_BYTES()).topic(topic).create(); + // Registers a consumer to avoid client to close idle connections. + Consumer consumer = injectedReplClient.newConsumer(Schema.AUTO_CONSUME()).subscriptionName("s1") + .topic(topic).subscribe(); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + assertEquals(persistentTopic.getProducers().size(), 1); + producer.newMessage(Schema.AVRO(Schemas.PersonOne.class)).value(new Schemas.PersonOne(1)).send(); + CompletableFuture send2 = producer.newMessage(Schema.AVRO(Schemas.PersonTwo.class)) + .value(new Schemas.PersonTwo(2, "2")).sendAsync(); + producer.close(); + Awaitility.await().untilAsserted(() -> { + assertTrue(send2.isDone()); + assertTrue(send2.isCompletedExceptionally()); + // Since the producer was closed, the topic should maintain 0 producers. + assertEquals(persistentTopic.getProducers().size(), 0); + }); + handleErrorSignal.countDown(); + + // cleanup. + consumer.close(); + injectedReplClient.close(); + admin.topics().unload(topic); + } + @Test public void testExternalSchemaTypeCompatibility() throws Exception { String namespace = "test-namespace-" + randomName(16); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 4119fc8b6ed55..4ec3c46c85848 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1240,7 +1240,7 @@ public synchronized CompletableFuture closeAsync() { closeProducerTasks(); ClientCnx cnx = cnx(); - if (cnx == null || currentState != State.Ready) { + if (cnx == null || (currentState != State.Ready && currentState != State.RegisteringSchema)) { log.info("[{}] [{}] Closed Producer (not connected)", topic, producerName); closeAndClearPendingMessages(); return CompletableFuture.completedFuture(null); From 44cc0133a75cd91688cfa0ac25cd1fbd6c950884 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Mon, 27 Apr 2026 20:10:37 +0800 Subject: [PATCH 052/201] [improve][cli] Add client side looping in "pulsar-admin topics analyze-backlog" cli to avoid potential HTTP call timeout (#25126) --- .../pulsar/admin/cli/PulsarAdminToolTest.java | 35 ++++ .../apache/pulsar/admin/cli/CliCommand.java | 16 +- .../apache/pulsar/admin/cli/CmdTopics.java | 41 ++++- .../cli/topic/AnalyzeBacklogTest.java | 151 ++++++++++++++++++ .../src/test/resources/pulsar-cli.xml | 1 + 5 files changed, 237 insertions(+), 7 deletions(-) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/topic/AnalyzeBacklogTest.java diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index a06f5d44665cf..bee1527b95565 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -120,6 +120,7 @@ import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.protocol.schema.PostSchemaPayload; +import org.apache.pulsar.common.stats.AnalyzeSubscriptionBacklogResult; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; @@ -2317,6 +2318,40 @@ public void nonPersistentTopics() throws Exception { verify(mockNonPersistentTopics).getListInBundle("myprop/ns1", "0x23d70a30_0x26666658"); } + @Test + public void topicsAnalyzeBacklogParameterParsing() throws Exception { + PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + Topics mockTopics = mock(Topics.class); + when(admin.topics()).thenReturn(mockTopics); + + AnalyzeSubscriptionBacklogResult backlogResult = new AnalyzeSubscriptionBacklogResult(); + doReturn(backlogResult).when(mockTopics) + .analyzeSubscriptionBacklog(eq("persistent://myprop/ns1/ds1"), eq("sub1"), Mockito.any()); + doReturn(backlogResult).when(mockTopics) + .analyzeSubscriptionBacklog(eq("persistent://myprop/ns1/ds1"), eq("sub1"), Mockito.any(), + Mockito.any()); + + CmdTopics cmdTopics = new CmdTopics(() -> admin); + cmdTopics.run(split("analyze-backlog persistent://myprop/ns1/ds1 -s sub1 --position 1:1")); + verify(mockTopics).analyzeSubscriptionBacklog(eq("persistent://myprop/ns1/ds1"), eq("sub1"), + eq(Optional.of(new MessageIdImpl(1, 1, -1)))); + + cmdTopics = new CmdTopics(() -> admin); + cmdTopics.run(split("analyze-backlog persistent://myprop/ns1/ds1 -s sub1 -b 100 --plain --quiet")); + verify(mockTopics).analyzeSubscriptionBacklog(eq("persistent://myprop/ns1/ds1"), eq("sub1"), + eq(Optional.empty()), Mockito.any()); + } + + @Test + public void topicsAnalyzeBacklogRejectsNonPositiveBacklogScanMaxEntries() { + PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + Topics mockTopics = mock(Topics.class); + when(admin.topics()).thenReturn(mockTopics); + + CmdTopics cmdTopics = new CmdTopics(() -> admin); + assertFalse(cmdTopics.run(split("analyze-backlog persistent://myprop/ns1/ds1 -s sub1 -b 0"))); + } + @Test public void bookies() throws Exception { PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java index 41593eb9e236e..be31438634959 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java @@ -110,11 +110,17 @@ void print(Map items) { } void print(T item) { + print(item, true); + } + + void print(T item, boolean prettyPrint) { try { if (item instanceof String) { commandSpec.commandLine().getOut().println(item); - } else { + } else if (prettyPrint) { prettyPrint(item); + } else { + plainPrint(item); } } catch (Exception e) { throw new RuntimeException(e); @@ -129,6 +135,14 @@ void prettyPrint(T item) { } } + void plainPrint(T item) { + try { + commandSpec.commandLine().getOut().println(MAPPER.writeValueAsString(item)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private static final ObjectMapper MAPPER = ObjectMapperFactory.create(); private static final ObjectWriter WRITER = MAPPER.writerWithDefaultPrettyPrinter(); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index d0bdb4ddaea37..8dc7da0febe3c 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -83,6 +83,7 @@ import org.apache.pulsar.common.policies.data.PublishRate; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SubscribeRate; +import org.apache.pulsar.common.stats.AnalyzeSubscriptionBacklogResult; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.ObjectMapperFactory; import picocli.CommandLine.Command; @@ -3001,24 +3002,52 @@ private class AnalyzeBacklog extends CliCommand { @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") private String topicName; - @Option(names = { "-s", "--subscription" }, description = "Subscription to be analyzed", required = true) + @Option(names = {"-s", "--subscription"}, description = "Subscription to be analyzed", required = true) private String subName; - @Option(names = { "--position", - "-p" }, description = "message position to start the scan from (ledgerId:entryId)", required = false) + @Option(names = {"--position", + "-p"}, description = "Message position to start the scan from (ledgerId:entryId)", required = false) private String messagePosition; + @Option(names = {"--backlog-scan-max-entries", + "-b"}, description = "The maximum number of backlog entries the client will scan before terminating " + + "its loop", required = false) + private Long backlogScanMaxEntries; + + @Option(names = {"--quiet", "-q"}, description = "Disable analyze-backlog progress reporting", required = false) + private boolean quiet = false; + + @Option(names = {"--plain"}, description = "Plain(Non-pretty) print backlog results as NDJSON", + required = false) + private boolean plainPrint = false; + @Override - void run() throws PulsarAdminException { + void run() throws Exception { String persistentTopic = validatePersistentTopic(topicName); Optional startPosition = Optional.empty(); + int partitionIndex = TopicName.get(persistentTopic).getPartitionIndex(); if (isNotBlank(messagePosition)) { - int partitionIndex = TopicName.get(persistentTopic).getPartitionIndex(); MessageId messageId = validateMessageIdString(messagePosition, partitionIndex); startPosition = Optional.of(messageId); } - print(getTopics().analyzeSubscriptionBacklog(persistentTopic, subName, startPosition)); + AnalyzeSubscriptionBacklogResult backlogResult; + if (backlogScanMaxEntries == null) { + backlogResult = getTopics().analyzeSubscriptionBacklog(persistentTopic, subName, startPosition); + } else { + if (backlogScanMaxEntries <= 0) { + throw new ParameterException("--backlog-scan-max-entries must be greater than 0"); + } + backlogResult = getTopics().analyzeSubscriptionBacklog(persistentTopic, subName, startPosition, + result -> { + boolean terminate = result.getEntries() >= backlogScanMaxEntries; + if (!quiet && !terminate) { + print(result, !plainPrint); + } + return terminate; + }); + } + print(backlogResult, !plainPrint); } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/topic/AnalyzeBacklogTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/topic/AnalyzeBacklogTest.java new file mode 100644 index 0000000000000..55220a6355e66 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/topic/AnalyzeBacklogTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.cli.topic; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import lombok.Cleanup; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.stats.AnalyzeSubscriptionBacklogResult; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.suites.PulsarTestSuite; +import org.testng.annotations.Test; + +public class AnalyzeBacklogTest extends PulsarTestSuite { + + private static final String PREFIX = "PULSAR_PREFIX_"; + private static final String ANALYZE_BACKLOG_TOPIC_NAME = "public/default/analyze-backlog-topic"; + private static final String ANALYZE_BACKLOG_SUBSCRIPTION_NAME = "sub1"; + private static final int SUBSCRIPTION_BACKLOG_SCAN_MAX_ENTRIES = 10; + private static final String LINE_SEPARATOR_REGEX = "\\r?\\n"; + private static final String TOPICS_CMD = "topics"; + + @Override + public void setupCluster() throws Exception { + brokerEnvs.put(PREFIX + "subscriptionBacklogScanMaxEntries", + String.valueOf(SUBSCRIPTION_BACKLOG_SCAN_MAX_ENTRIES)); + super.setupCluster(); + } + + @Test + public void testAnalyzeBacklogUsingDefaultConfig() throws Exception { + prepareSubscriptionBacklog(SUBSCRIPTION_BACKLOG_SCAN_MAX_ENTRIES + 1); + + ContainerExecResult result = + pulsarCluster.runAdminCommandOnAnyBroker(TOPICS_CMD, "analyze-backlog", ANALYZE_BACKLOG_TOPIC_NAME, + "-s", ANALYZE_BACKLOG_SUBSCRIPTION_NAME); + + String stdout = result.getStdout(); + AnalyzeSubscriptionBacklogResult backlogResult = + jsonMapper().readValue(stdout, AnalyzeSubscriptionBacklogResult.class); + assertEquals(SUBSCRIPTION_BACKLOG_SCAN_MAX_ENTRIES, backlogResult.getEntries()); + + String[] lines = stdout.split(LINE_SEPARATOR_REGEX); + assertTrue(lines.length > 1); + } + + @Test + public void testAnalyzeBacklogUsingPlainPrint() throws Exception { + prepareSubscriptionBacklog(SUBSCRIPTION_BACKLOG_SCAN_MAX_ENTRIES + 1); + + ContainerExecResult result = + pulsarCluster.runAdminCommandOnAnyBroker(TOPICS_CMD, "analyze-backlog", ANALYZE_BACKLOG_TOPIC_NAME, + "-s", ANALYZE_BACKLOG_SUBSCRIPTION_NAME, "--plain"); + + String stdout = result.getStdout(); + AnalyzeSubscriptionBacklogResult backlogResult = + jsonMapper().readValue(stdout, AnalyzeSubscriptionBacklogResult.class); + assertEquals(SUBSCRIPTION_BACKLOG_SCAN_MAX_ENTRIES, backlogResult.getEntries()); + + String[] lines = stdout.split(LINE_SEPARATOR_REGEX); + assertEquals(1, lines.length); + } + + @Test + public void testAnalyzeBacklogClientSideLoopUsingPlainPrint() throws Exception { + int backlogNum = 50; + prepareSubscriptionBacklog(backlogNum); + + int backlogScanMaxEntries = 40; + ContainerExecResult result = + pulsarCluster.runAdminCommandOnAnyBroker(TOPICS_CMD, "analyze-backlog", ANALYZE_BACKLOG_TOPIC_NAME, + "-s", ANALYZE_BACKLOG_SUBSCRIPTION_NAME, "-b", String.valueOf(backlogScanMaxEntries), + "--plain"); + + int expectedResultLines = 4; + String stdout = result.getStdout(); + String[] lines = stdout.split(LINE_SEPARATOR_REGEX); + assertEquals(expectedResultLines, lines.length); + + for (int i = 0; i < expectedResultLines; i++) { + AnalyzeSubscriptionBacklogResult backlogResult = + jsonMapper().readValue(lines[i], AnalyzeSubscriptionBacklogResult.class); + assertEquals((long) SUBSCRIPTION_BACKLOG_SCAN_MAX_ENTRIES * (i + 1), backlogResult.getEntries()); + } + } + + @Test + public void testAnalyzeBacklogClientSideLoopUsingQuietPlainPrint() throws Exception { + int backlogNum = 50; + prepareSubscriptionBacklog(backlogNum); + + int backlogScanMaxEntries = 35; + ContainerExecResult result = + pulsarCluster.runAdminCommandOnAnyBroker(TOPICS_CMD, "analyze-backlog", ANALYZE_BACKLOG_TOPIC_NAME, + "-s", ANALYZE_BACKLOG_SUBSCRIPTION_NAME, "-b", String.valueOf(backlogScanMaxEntries), "-q", + "--plain"); + + String stdout = result.getStdout(); + String[] lines = stdout.split(LINE_SEPARATOR_REGEX); + assertEquals(1, lines.length); + + int expectedEntries = 40; + AnalyzeSubscriptionBacklogResult backlogResult = + jsonMapper().readValue(stdout, AnalyzeSubscriptionBacklogResult.class); + assertEquals(expectedEntries, backlogResult.getEntries()); + } + + private void prepareSubscriptionBacklog(int backlogNum) throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder().serviceUrl(pulsarCluster.getPlainTextServiceUrl()).build(); + @Cleanup + Producer producer = + client.newProducer().topic(ANALYZE_BACKLOG_TOPIC_NAME).enableBatching(false).create(); + @Cleanup + Consumer consumer = client.newConsumer().topic(ANALYZE_BACKLOG_TOPIC_NAME) + .subscriptionName(ANALYZE_BACKLOG_SUBSCRIPTION_NAME).subscribe(); + + List> futures = new ArrayList<>(backlogNum); + for (int i = 0; i < backlogNum; i++) { + byte[] msgBytes = ("test" + i).getBytes(StandardCharsets.UTF_8); + CompletableFuture future = producer.sendAsync(msgBytes); + futures.add(future); + } + FutureUtil.waitForAll(futures).get(); + } + +} diff --git a/tests/integration/src/test/resources/pulsar-cli.xml b/tests/integration/src/test/resources/pulsar-cli.xml index af55aca8a0098..e32c4649d248f 100644 --- a/tests/integration/src/test/resources/pulsar-cli.xml +++ b/tests/integration/src/test/resources/pulsar-cli.xml @@ -34,6 +34,7 @@ + From ef327a849169469b2b2c4d8b63fab4a60d87e208 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 27 Apr 2026 20:50:37 +0800 Subject: [PATCH 053/201] [fix][broker] Decrement unacked counter when removeAllUpTo removes pending acks (#25581) --- .../pulsar/broker/service/Consumer.java | 29 +++ .../pulsar/broker/service/PendingAcksMap.java | 22 ++- ...PersistentDispatcherMultipleConsumers.java | 4 +- .../broker/service/PendingAcksMapTest.java | 31 +++- .../broker/stats/ConsumerStatsTest.java | 166 ++++++++++++++++++ .../client/api/ProducerConsumerBase.java | 3 +- 6 files changed, 243 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 3edf6e2d68140..a7ca3f8339f47 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -1113,6 +1113,35 @@ public PendingAcksMap getPendingAcks() { return pendingAcks; } + /** + * Remove all pending acks up to the given mark-delete position and decrement the consumer's unacked message + * counter by the remaining unacked count for each removed entry. + * + *

This is used when the cursor's mark-delete position advances past entries that are still in the consumer's + * pending acks. The remaining unacked count accounts for batch index level acknowledgments — only the truly + * unacked batch indexes are decremented. + * + * @param markDeleteLedgerId the ledger ID up to which to remove pending acks + * @param markDeleteEntryId the entry ID up to which to remove pending acks + */ + public void removePendingAcksUpToPositionAndDecrementUnacked(long markDeleteLedgerId, long markDeleteEntryId) { + if (pendingAcks == null) { + return; + } + + MutableInt mutableTotalUnacked = new MutableInt(0); + pendingAcks.removeAllUpTo(markDeleteLedgerId, markDeleteEntryId, + (ledgerId, entryId, batchSize, stickyKeyHash) -> { + mutableTotalUnacked.add((int) getUnAckedCountForBatchIndexLevelEnabled( + PositionFactory.create(ledgerId, entryId), batchSize)); + }); + int totalUnacked = mutableTotalUnacked.intValue(); + if (totalUnacked > 0) { + addAndGetUnAckedMsgs(this, -totalUnacked); + updateBlockedConsumerOnUnackedMsgs(this); + } + } + public int getPriorityLevel() { return priorityLevel; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java index 7a728a037dc62..0f7802d8294e0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java @@ -326,13 +326,16 @@ public boolean remove(long ledgerId, long entryId) { } /** - * Remove all pending acks up to the given ledger ID and entry ID. + * Remove all pending acks up to the given ledger ID and entry ID, invoking a callback for each removed entry. * * @param markDeleteLedgerId the ledger ID up to which to remove pending acks * @param markDeleteEntryId the entry ID up to which to remove pending acks + * @param removedEntryCallback optional callback invoked for each removed entry (within the write lock), + * receiving ledgerId, entryId, batchSize, and stickyKeyHash */ - public void removeAllUpTo(long markDeleteLedgerId, long markDeleteEntryId) { - internalRemoveAllUpTo(markDeleteLedgerId, markDeleteEntryId, false); + public void removeAllUpTo(long markDeleteLedgerId, long markDeleteEntryId, + PendingAcksConsumer removedEntryCallback) { + internalRemoveAllUpTo(markDeleteLedgerId, markDeleteEntryId, false, removedEntryCallback); } /** @@ -345,8 +348,10 @@ public void removeAllUpTo(long markDeleteLedgerId, long markDeleteEntryId) { * @param markDeleteLedgerId the ledger ID up to which to remove pending acks * @param markDeleteEntryId the entry ID up to which to remove pending acks * @param useWriteLock true if the method should use a write lock, false otherwise + * @param removedEntryCallback optional callback invoked for each removed entry (within the write lock) */ - private void internalRemoveAllUpTo(long markDeleteLedgerId, long markDeleteEntryId, boolean useWriteLock) { + private void internalRemoveAllUpTo(long markDeleteLedgerId, long markDeleteEntryId, boolean useWriteLock, + PendingAcksConsumer removedEntryCallback) { PendingAcksRemoveHandler pendingAcksRemoveHandler = pendingAcksRemoveHandlerSupplier.get(); // track if the write lock was acquired boolean acquiredWriteLock = false; @@ -382,14 +387,19 @@ private void internalRemoveAllUpTo(long markDeleteLedgerId, long markDeleteEntry retryWithWriteLock = true; return; } + IntIntPair value = intIntPairEntry.getValue(); + int batchSize = value.leftInt(); + int stickyKeyHash = value.rightInt(); if (pendingAcksRemoveHandler != null) { if (!batchStarted) { pendingAcksRemoveHandler.startBatch(); batchStarted = true; } - int stickyKeyHash = intIntPairEntry.getValue().rightInt(); pendingAcksRemoveHandler.handleRemoving(consumer, ledgerId, entryId, stickyKeyHash, closed); } + if (removedEntryCallback != null) { + removedEntryCallback.accept(ledgerId, entryId, batchSize, stickyKeyHash); + } entryMapIterator.remove(); } if (ledgerMap.isEmpty()) { @@ -409,7 +419,7 @@ private void internalRemoveAllUpTo(long markDeleteLedgerId, long markDeleteEntry } else { readLock.unlock(); if (retryWithWriteLock) { - internalRemoveAllUpTo(markDeleteLedgerId, markDeleteEntryId, true); + internalRemoveAllUpTo(markDeleteLedgerId, markDeleteEntryId, true, removedEntryCallback); } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index a88fbf863eba3..220b9e23a3e0e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -363,8 +363,8 @@ public synchronized void readMoreEntries() { if (lastMarkDeletePositionBeforeReadMoreEntries != markDeletePosition) { redeliveryMessages.removeAllUpTo(markDeletePosition.getLedgerId(), markDeletePosition.getEntryId()); for (Consumer consumer : consumerList) { - consumer.getPendingAcks() - .removeAllUpTo(markDeletePosition.getLedgerId(), markDeletePosition.getEntryId()); + consumer.removePendingAcksUpToPositionAndDecrementUnacked( + markDeletePosition.getLedgerId(), markDeletePosition.getEntryId()); } lastMarkDeletePositionBeforeReadMoreEntries = markDeletePosition; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java index 42f5935ca88ff..8db0e3a0f73e6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java @@ -117,7 +117,8 @@ public void removeAllUpTo_RemovesAllPendingAcksUpToSpecifiedEntry() { pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 1, 124); pendingAcksMap.addPendingAckIfAllowed(2L, 1L, 1, 125); - pendingAcksMap.removeAllUpTo(1L, 2L); + pendingAcksMap.removeAllUpTo(1L, 2L, (ledgerId, entryId, batchSize, stickyKeyHash) -> { + }); assertFalse(pendingAcksMap.contains(1L, 1L)); assertFalse(pendingAcksMap.contains(1L, 2L)); @@ -134,7 +135,8 @@ public void removeAllUpTo_RemovesAllPendingAcksUpToSpecifiedEntryAcrossMultipleL pendingAcksMap.addPendingAckIfAllowed(2L, 2L, 1, 126); pendingAcksMap.addPendingAckIfAllowed(3L, 1L, 1, 127); - pendingAcksMap.removeAllUpTo(2L, 1L); + pendingAcksMap.removeAllUpTo(2L, 1L, (ledgerId, entryId, batchSize, stickyKeyHash) -> { + }); assertFalse(pendingAcksMap.contains(1L, 1L)); assertFalse(pendingAcksMap.contains(1L, 2L)); @@ -176,13 +178,36 @@ public void removeAllUpTo_InvokesRemoveHandlerForEachEntry() { pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 1, 124); pendingAcksMap.addPendingAckIfAllowed(2L, 1L, 1, 125); - pendingAcksMap.removeAllUpTo(1L, 2L); + pendingAcksMap.removeAllUpTo(1L, 2L, (ledgerId, entryId, batchSize, stickyKeyHash) -> { + }); verify(removeHandler).handleRemoving(consumer, 1L, 1L, 123, false); verify(removeHandler).handleRemoving(consumer, 1L, 2L, 124, false); verify(removeHandler, never()).handleRemoving(consumer, 2L, 1L, 125, false); } + @Test + public void removeAllUpToWithCallback_InvokesCallbackForEachRemovedEntry() { + Consumer consumer = createMockConsumer("consumer1"); + PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null); + pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 3, 123); + pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 5, 124); + pendingAcksMap.addPendingAckIfAllowed(2L, 1L, 7, 125); + + List callbackInvocations = new ArrayList<>(); + pendingAcksMap.removeAllUpTo(1L, 2L, + (ledgerId, entryId, batchSize, stickyKeyHash) -> { + callbackInvocations.add(new int[]{(int) ledgerId, (int) entryId, batchSize, stickyKeyHash}); + }); + + assertEquals(callbackInvocations.size(), 2); + assertEquals(callbackInvocations.get(0), new int[]{1, 1, 3, 123}); + assertEquals(callbackInvocations.get(1), new int[]{1, 2, 5, 124}); + assertFalse(pendingAcksMap.contains(1L, 1L)); + assertFalse(pendingAcksMap.contains(1L, 2L)); + assertTrue(pendingAcksMap.contains(2L, 1L)); + } + @Test public void size_ReturnsCorrectSize() { Consumer consumer = createMockConsumer("consumer1"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java index 568a3cb08506b..30b8310b28d9b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java @@ -59,6 +59,7 @@ import org.apache.pulsar.broker.service.StickyKeyDispatcher; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.AbstractPersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.plugin.EntryFilter; import org.apache.pulsar.broker.service.plugin.EntryFilterProducerTest; @@ -106,6 +107,7 @@ protected void setup() throws Exception { @Override protected ServiceConfiguration getDefaultConf() { ServiceConfiguration conf = super.getDefaultConf(); + conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); conf.setMaxUnackedMessagesPerConsumer(0); // wait for shutdown of the broker, this prevents flakiness which could be caused by metrics being // unregistered asynchronously. This impacts the execution of the next test method if this would be happening. @@ -729,6 +731,170 @@ public void testKeySharedDrainingHashesConsumerStats() throws Exception { } + @DataProvider(name = "subscriptionTypes") + public Object[][] subscriptionTypes() { + return new Object[][]{ + {SubscriptionType.Shared}, + {SubscriptionType.Key_Shared} + }; + } + + /** + * Verify unacked count is correctly decremented when removeAllUpTo removes non-batch + * entries from pendingAcks after mark-delete advances via message expiry. + */ + @Test(dataProvider = "subscriptionTypes") + public void testUnackedCountNonBatchAfterExpire(SubscriptionType subType) throws Exception { + String topic = newTopicName(); + String sub = "sub"; + int numMessages = 10; + + @Cleanup Producer producer = pulsarClient.newProducer() + .topic(topic).enableBatching(false).create(); + @Cleanup Consumer consumer = pulsarClient.newConsumer() + .topic(topic).subscriptionName(sub) + .subscriptionType(subType) + .subscribe(); + + for (int i = 0; i < numMessages; i++) { + producer.send(("msg-" + i).getBytes()); + } + + org.apache.pulsar.broker.service.Consumer svcConsumer = + getTheUniqueServiceConsumer(topic, sub); + for (int i = 0; i < numMessages; i++) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(msg, "Expected to receive message " + i); + } + + Awaitility.await().untilAsserted(() -> + assertEquals(numMessages, svcConsumer.getUnackedMessages())); + + expireAndVerifyUnackedDrained(topic, sub, producer, consumer, svcConsumer); + } + + /** + * Verify unacked count is correctly decremented when removeAllUpTo removes batch + * entries from pendingAcks after mark-delete advances via message expiry. + */ + @Test(dataProvider = "subscriptionTypes") + public void testUnackedCountBatchAfterExpire(SubscriptionType subType) throws Exception { + String topic = newTopicName(); + String sub = "sub"; + int numMessages = 10; + + @Cleanup Producer producer = pulsarClient.newProducer() + .topic(topic) + .batchingMaxMessages(20) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .enableBatching(true) + .create(); + @Cleanup Consumer consumer = pulsarClient.newConsumer() + .topic(topic).subscriptionName(sub) + .subscriptionType(subType) + .subscribe(); + + for (int i = 0; i < numMessages; i++) { + producer.newMessage().value(("batch-" + i).getBytes()).sendAsync(); + } + producer.flush(); + + for (int i = 0; i < numMessages; i++) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(msg, "Expected to receive message " + i); + } + + org.apache.pulsar.broker.service.Consumer svcConsumer = + getTheUniqueServiceConsumer(topic, sub); + + Awaitility.await().untilAsserted(() -> + assertEquals(numMessages, svcConsumer.getUnackedMessages())); + + expireAndVerifyUnackedDrained(topic, sub, producer, consumer, svcConsumer); + } + + /** + * Verify unacked count is correctly decremented when removeAllUpTo removes a partially-acked + * batch entry from pendingAcks after mark-delete advances via message expiry. + * + *

Flow: produce batch(batchSize=10) → consume all → ack 5 of 10 → expire → unacked should be 0. + */ + @Test(dataProvider = "subscriptionTypes") + public void testUnackedCountBatchPartialAckAfterExpire(SubscriptionType subType) throws Exception { + String topic = newTopicName(); + String sub = "sub"; + int numMessages = 10; + int ackCount = 5; + + @Cleanup Producer producer = pulsarClient.newProducer() + .topic(topic) + .batchingMaxMessages(20) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .enableBatching(true) + .create(); + @Cleanup Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(sub) + .enableBatchIndexAcknowledgment(true) + .subscriptionType(subType) + .subscribe(); + + for (int i = 0; i < numMessages; i++) { + producer.newMessage().value(("batch-" + i).getBytes()).sendAsync(); + } + producer.flush(); + + List> messages = new ArrayList<>(); + for (int i = 0; i < numMessages; i++) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(msg, "Expected to receive message " + i); + messages.add(msg); + } + + org.apache.pulsar.broker.service.Consumer svcConsumer = + getTheUniqueServiceConsumer(topic, sub); + + Awaitility.await().untilAsserted(() -> + assertEquals(numMessages, svcConsumer.getUnackedMessages())); + + // Partially ack — ack 5 of 10 batch indexes + for (int i = 0; i < ackCount; i++) { + consumer.acknowledge(messages.get(i)); + } + Awaitility.await().untilAsserted(() -> + assertEquals(numMessages - ackCount, svcConsumer.getUnackedMessages())); + + expireAndVerifyUnackedDrained(topic, sub, producer, consumer, svcConsumer); + } + + private void expireAndVerifyUnackedDrained(String topic, String sub, + Producer producer, Consumer consumer, + org.apache.pulsar.broker.service.Consumer svcConsumer) + throws Exception { + PersistentTopic pTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicReference(topic).get(); + + Thread.sleep(1100); + pTopic.getSubscription(sub).expireMessagesAsync(1).get(); + + // Trigger readMoreEntries to invoke removeAllUpTo + producer.send("trigger".getBytes()); + Message triggerMsg = consumer.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(triggerMsg); + consumer.acknowledge(triggerMsg); + + Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> + assertEquals(0, svcConsumer.getUnackedMessages())); + } + + private org.apache.pulsar.broker.service.Consumer getTheUniqueServiceConsumer(String topic, String sub) { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + AbstractPersistentDispatcherMultipleConsumers dispatcher = + (AbstractPersistentDispatcherMultipleConsumers) persistentTopic.getSubscription(sub).getDispatcher(); + return dispatcher.getConsumers().iterator().next(); + } + private String findConsumerNameForHash(SubscriptionStats subscriptionStats, int hash) { return findConsumerForHash(subscriptionStats, hash).map(ConsumerStats::getConsumerName).orElse(null); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java index 90babc00d2adb..638f5aec484f4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java @@ -30,6 +30,7 @@ import java.util.function.BiFunction; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.testng.Assert; @@ -71,7 +72,7 @@ protected void testMessageOrderAndDuplicates(Set messagesReceived, T rece private static final Random random = new Random(); protected String newTopicName() { - return "my-property/my-ns/topic-" + Long.toHexString(random.nextLong()); + return TopicName.get("my-property/my-ns/topic-" + Long.toHexString(random.nextLong())).toString(); } protected ReceivedMessages receiveAndAckMessages( From e2ca1dd23c8222f43c71b751dea16705405d16ce Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Mon, 11 May 2026 17:40:08 +0800 Subject: [PATCH 054/201] [fix][client] Make ClientBuilder serializable (#25730) (#25739) --- .../auth/oauth2/ClientCredentialsFlow.java | 2 +- .../client/impl/auth/oauth2/FlowBase.java | 23 +++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java index 4975af3b7967e..d841010add81d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java @@ -138,7 +138,7 @@ public void initialize() throws PulsarClientException { assert this.metadata != null; URL tokenUrl = this.metadata.getTokenEndpoint(); - this.exchanger = new TokenClient(tokenUrl, httpClient); + this.exchanger = new TokenClient(tokenUrl, getHttpClient()); initialized = true; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java index 9649d17903148..63248e8605108 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java @@ -55,16 +55,22 @@ abstract class FlowBase implements Flow { private static final long serialVersionUID = 1L; protected final URL issuerUrl; - protected final AsyncHttpClient httpClient; + private final Duration connectTimeout; + private final Duration readTimeout; + private final String trustCertsFilePath; protected final String wellKnownMetadataPath; protected transient Metadata metadata; + private transient AsyncHttpClient httpClient; protected FlowBase(URL issuerUrl, Duration connectTimeout, Duration readTimeout, String trustCertsFilePath, String wellKnownMetadataPath) { this.issuerUrl = issuerUrl; - this.httpClient = defaultHttpClient(readTimeout, connectTimeout, trustCertsFilePath); + this.connectTimeout = connectTimeout; + this.readTimeout = readTimeout; + this.trustCertsFilePath = trustCertsFilePath; this.wellKnownMetadataPath = wellKnownMetadataPath; + getHttpClient(); } private AsyncHttpClient defaultHttpClient(Duration readTimeout, Duration connectTimeout, @@ -91,6 +97,13 @@ private AsyncHttpClient defaultHttpClient(Duration readTimeout, Duration connect return new DefaultAsyncHttpClient(confBuilder.build()); } + protected synchronized AsyncHttpClient getHttpClient() { + if (httpClient == null) { + httpClient = defaultHttpClient(readTimeout, connectTimeout, trustCertsFilePath); + } + return httpClient; + } + private int getParameterDurationToMillis(String name, Duration value, Duration defaultValue) { Duration duration; if (value == null) { @@ -118,7 +131,7 @@ public void initialize() throws PulsarClientException { } protected MetadataResolver createMetadataResolver() { - return DefaultMetadataResolver.fromIssuerUrl(issuerUrl, httpClient, wellKnownMetadataPath); + return DefaultMetadataResolver.fromIssuerUrl(issuerUrl, getHttpClient(), wellKnownMetadataPath); } static String parseParameterString(Map params, String name) { @@ -155,6 +168,8 @@ static Duration parseParameterDuration(Map params, String name) @Override public void close() throws Exception { - httpClient.close(); + if (httpClient != null) { + httpClient.close(); + } } } From 431516d1b4d5787d73d8b92290abe18aa2e06b57 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 11 May 2026 13:35:14 +0300 Subject: [PATCH 055/201] [fix][build][branch-4.2] Fix checkstyle issue after a previous cherry-pick --- .../apache/pulsar/broker/service/OneWayReplicatorTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index ddfef968a3b14..06c4411aae034 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -40,8 +40,6 @@ import io.netty.util.concurrent.FastThreadLocalThread; import java.lang.reflect.Field; import java.lang.reflect.Method; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -99,11 +97,9 @@ import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.client.impl.ClientCnx; -import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.ProducerBuilderImpl; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; -import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.api.proto.CommandSendReceipt; From d5ba0d86ce67ad03313c1696ce85ca9326d1de8f Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 11 May 2026 11:36:14 +0300 Subject: [PATCH 056/201] [fix][sec] Upgrade vert.x to 4.5.25 to address CVE-2026-6860 (#25737) (cherry picked from commit 90f83d465020bbb2f703fa1dc54b2d271b2ec8ae) --- distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++----- pom.xml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 1144d8caac87a..18a5c3f59cdc7 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -509,11 +509,11 @@ The Apache Software License, Version 2.0 * JCTools - Java Concurrency Tools for the JVM - org.jctools-jctools-core-4.0.5.jar * Vertx - - io.vertx-vertx-auth-common-4.5.24.jar - - io.vertx-vertx-bridge-common-4.5.24.jar - - io.vertx-vertx-core-4.5.24.jar - - io.vertx-vertx-web-4.5.24.jar - - io.vertx-vertx-web-common-4.5.24.jar + - io.vertx-vertx-auth-common-4.5.25.jar + - io.vertx-vertx-bridge-common-4.5.25.jar + - io.vertx-vertx-core-4.5.25.jar + - io.vertx-vertx-web-4.5.25.jar + - io.vertx-vertx-web-common-4.5.25.jar * Apache ZooKeeper - org.apache.zookeeper-zookeeper-jute-3.9.5.jar * Snappy Java diff --git a/pom.xml b/pom.xml index 0b648acb6227a..38302335e7a36 100644 --- a/pom.xml +++ b/pom.xml @@ -198,7 +198,7 @@ flexible messaging model and an intuitive client API. 2.42 1.10.62 0.16.0 - 4.5.24 + 4.5.25 7.9.2 2.0.17 4.5.0 From 7d40597c40efdbdbcec5a888cc353d4666d3ead4 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 6 May 2026 11:05:26 +0300 Subject: [PATCH 057/201] [fix][broker] Skip backlog-quota eviction on fenced/closing topics (#25684) --- .../pulsar/broker/service/BacklogQuotaManager.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java index f04570c5441cd..4e3dbe1228875 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java @@ -85,6 +85,17 @@ public BacklogQuotaImpl getBacklogQuota(NamespaceName namespace, BacklogQuotaTyp */ public void handleExceededBacklogQuota(PersistentTopic persistentTopic, BacklogQuotaType backlogQuotaType, boolean preciseTimeBasedBacklogQuotaCheck) { + if (persistentTopic.isFenced() || persistentTopic.isClosingOrDeleting()) { + // Skip eviction work on a topic that is being torn down or transiently fenced. + // Mutating cursors here (skipEntries / markDeletePosition) contends with the + // delete path and can keep namespace force-delete from completing in time; + // the entries are about to be discarded anyway. + log.debug() + .attr("topic", persistentTopic.getName()) + .attr("backlogQuotaType", backlogQuotaType) + .log("Skipping backlog-quota eviction on fenced/closing topic"); + return; + } BacklogQuota quota = persistentTopic.getBacklogQuota(backlogQuotaType); BacklogQuotaMetrics topicBacklogQuotaMetrics = persistentTopic.getPersistentTopicMetrics().getBacklogQuotaMetrics(); From c9723462b1199bd1288fc15961562ea0cd9f92c1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 11 May 2026 14:22:46 +0300 Subject: [PATCH 058/201] [fix][build][branch-4.2] Fix compilation issue after backporting --- .../apache/pulsar/broker/service/BacklogQuotaManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java index 4e3dbe1228875..47ff3588ab258 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java @@ -90,10 +90,10 @@ public void handleExceededBacklogQuota(PersistentTopic persistentTopic, BacklogQ // Mutating cursors here (skipEntries / markDeletePosition) contends with the // delete path and can keep namespace force-delete from completing in time; // the entries are about to be discarded anyway. - log.debug() - .attr("topic", persistentTopic.getName()) - .attr("backlogQuotaType", backlogQuotaType) - .log("Skipping backlog-quota eviction on fenced/closing topic"); + if (log.isDebugEnabled()) { + log.debug("Skipping backlog-quota eviction on fenced/closing topic {}, backlog quota type {}", + persistentTopic.getName(), backlogQuotaType); + } return; } BacklogQuota quota = persistentTopic.getBacklogQuota(backlogQuotaType); From e953228fe3331a11a7b0c6c500d8c9c1044bde9f Mon Sep 17 00:00:00 2001 From: Chris Hamons Date: Fri, 8 May 2026 15:41:23 -0500 Subject: [PATCH 059/201] [fix][broker] Correct two race conditions in the tracker code and logic bug in InMemoryDelayedDeliveryTracker that failed with NoSuchElementException (#25681) (cherry picked from commit 6cbf4d2ad0d1514bfd2885015084102c2896ea67) --- .../InMemoryDelayedDeliveryTracker.java | 14 ++- ...PersistentDispatcherMultipleConsumers.java | 10 +- ...entDispatcherMultipleConsumersClassic.java | 10 +- .../delayed/InMemoryDeliveryTrackerTest.java | 10 ++ ...ispatcherMultipleConsumersClassicTest.java | 99 +++++++++++++++++++ ...istentDispatcherMultipleConsumersTest.java | 99 +++++++++++++++++++ 6 files changed, 228 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index ad5ab25fbbf6b..1ded2cc428d32 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -126,10 +126,16 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { } long timestamp = trimLowerBit(deliverAt, timestampPrecisionBitCnt); - delayedMessageMap.computeIfAbsent(timestamp, k -> new Long2ObjectRBTreeMap<>()) - .computeIfAbsent(ledgerId, k -> new Roaring64Bitmap()) - .add(entryId); - delayedMessagesCount.incrementAndGet(); + Roaring64Bitmap bitmap = delayedMessageMap.computeIfAbsent(timestamp, k -> new Long2ObjectRBTreeMap<>()) + .computeIfAbsent(ledgerId, k -> new Roaring64Bitmap()); + // Roaring64Bitmap does not store duplicates, so track if it a new element + // so we can keep delayedMessagesCount in sync + boolean isNew = !bitmap.contains(entryId); + + if (isNew) { + bitmap.add(entryId); + delayedMessagesCount.incrementAndGet(); + } updateTimer(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 220b9e23a3e0e..cfabc9333adaf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -442,7 +442,7 @@ public synchronized void readMoreEntries() { } } - protected Predicate createReadEntriesSkipConditionForNormalRead() { + protected synchronized Predicate createReadEntriesSkipConditionForNormalRead() { Predicate skipCondition = null; // Filter out and skip read delayed messages exist in DelayedDeliveryTracker if (delayedDeliveryTracker.isPresent()) { @@ -1368,12 +1368,12 @@ protected synchronized boolean shouldPauseDeliveryForDelayTracker() { } @Override - public long getNumberOfDelayedMessages() { + public synchronized long getNumberOfDelayedMessages() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getNumberOfDelayedMessages).orElse(0L); } @Override - public CompletableFuture clearDelayedMessages() { + public synchronized CompletableFuture clearDelayedMessages() { if (!topic.isDelayedDeliveryEnabled()) { return CompletableFuture.completedFuture(null); } @@ -1452,11 +1452,11 @@ public PersistentTopic getTopic() { } - public long getDelayedTrackerMemoryUsage() { + public synchronized long getDelayedTrackerMemoryUsage() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getBufferMemoryUsage).orElse(0L); } - public Map getBucketDelayedIndexStats() { + public synchronized Map getBucketDelayedIndexStats() { if (delayedDeliveryTracker.isEmpty()) { return Collections.emptyMap(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java index d828f3381323d..28dd52f89d818 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java @@ -1138,7 +1138,7 @@ public boolean initializeDispatchRateLimiterIfNeeded() { } @Override - public boolean trackDelayedDelivery(long ledgerId, long entryId, MessageMetadata msgMetadata) { + public synchronized boolean trackDelayedDelivery(long ledgerId, long entryId, MessageMetadata msgMetadata) { if (!topic.isDelayedDeliveryEnabled()) { // If broker has the feature disabled, always deliver messages immediately return false; @@ -1204,12 +1204,12 @@ protected synchronized boolean shouldPauseDeliveryForDelayTracker() { } @Override - public long getNumberOfDelayedMessages() { + public synchronized long getNumberOfDelayedMessages() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getNumberOfDelayedMessages).orElse(0L); } @Override - public CompletableFuture clearDelayedMessages() { + public synchronized CompletableFuture clearDelayedMessages() { if (!topic.isDelayedDeliveryEnabled()) { return CompletableFuture.completedFuture(null); } @@ -1283,11 +1283,11 @@ public PersistentTopic getTopic() { } - public long getDelayedTrackerMemoryUsage() { + public synchronized long getDelayedTrackerMemoryUsage() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getBufferMemoryUsage).orElse(0L); } - public Map getBucketDelayedIndexStats() { + public synchronized Map getBucketDelayedIndexStats() { if (delayedDeliveryTracker.isEmpty()) { return Collections.emptyMap(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index c5a564d1b664b..2affb7fef22e2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -274,4 +274,14 @@ public void testDelaySequence(InMemoryDelayedDeliveryTracker tracker) throws Exc tracker.close(); } + @Test(dataProvider = "delayedTracker") + public void testAddMultipleMessagesSameWindow(InMemoryDelayedDeliveryTracker tracker) throws Exception { + tracker.addMessage(1, 1, 50); + tracker.addMessage(1, 1, 50); + tracker.addMessage(1, 1, 50); + + clockTime.set(60); + + tracker.getScheduledMessages(10); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassicTest.java index 52b3649339464..879cbecd61e31 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassicTest.java @@ -20,7 +20,12 @@ import com.carrotsearch.hppc.ObjectSet; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -34,6 +39,7 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.api.proto.MessageMetadata; import org.awaitility.reflect.WhiteboxImpl; import org.mockito.Mockito; import org.testng.Assert; @@ -152,4 +158,97 @@ public void testSkipReadEntriesFromCloseCursor() throws Exception { // Verify: the topic can be deleted successfully. admin.topics().delete(topicName, false); } + + @Test + public void testRaceConditionInTrackDelayedDelivery() throws Exception { + final int numThreads = 16; + final int operationsPerThread = 2000; + final CountDownLatch startLatch = new CountDownLatch(1); + final CountDownLatch doneLatch = new CountDownLatch(numThreads); + final AtomicInteger errors = new AtomicInteger(0); + final AtomicReference firstException = new AtomicReference<>(); + + final String topicName = newTopicName(); + final String subscription = "s1"; + + // Needed to create the topic + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName).subscriptionName(subscription) + .subscriptionType(SubscriptionType.Shared).subscribe(); + + PersistentTopic topic = (PersistentTopic) getTopic(topicName, false).join().get(); + + ManagedCursor cursor = Mockito.mock(ManagedCursorImpl.class); + Mockito.doReturn(subscription).when(cursor).getName(); + + Subscription sub = Mockito.mock(PersistentSubscription.class); + Mockito.doReturn(topic).when(sub).getTopic(); + + PersistentDispatcherMultipleConsumersClassic dispatcher = + new PersistentDispatcherMultipleConsumersClassic(topic, cursor, sub); + + // Align all writes to the same bucket + // This is the key which triggers the race condition + long deliverAt = System.currentTimeMillis() + 5000; + + MessageMetadata messageMetadata = new MessageMetadata() + .setSequenceId(1) + .setProducerName("testProducer") + .setPartitionKeyB64Encoded(false) + .setPublishTime(System.currentTimeMillis()) + .setDeliverAtTime(deliverAt); + + @Cleanup("shutdown") + ExecutorService executorService = Executors.newFixedThreadPool(32); + + // Start clear message thread + for (int i = 0; i < numThreads / 2; i++) { + executorService.submit(() -> { + try { + startLatch.await(); + for (int j = 0; j < operationsPerThread; j++) { + dispatcher.clearDelayedMessages(); + Thread.sleep(1); + } + } catch (Exception e) { + errors.incrementAndGet(); + firstException.compareAndSet(null, e); + e.printStackTrace(); + } finally { + doneLatch.countDown(); + } + }); + } + + // Start track delayed delivery thread + for (int i = numThreads / 2; i < numThreads; i++) { + executorService.submit(() -> { + try { + startLatch.await(); + for (int j = 0; j < operationsPerThread; j++) { + dispatcher.trackDelayedDelivery(1, 1, messageMetadata); + Thread.sleep(1); + } + } catch (Exception e) { + errors.incrementAndGet(); + firstException.compareAndSet(null, e); + e.printStackTrace(); + } finally { + doneLatch.countDown(); + } + }); + } + + startLatch.countDown(); + Assert.assertTrue(doneLatch.await(30, TimeUnit.SECONDS), "Test should complete within 30 seconds"); + + if (errors.get() > 0) { + Exception exception = firstException.get(); + if (exception != null) { + System.err.println("First exception caught: " + exception.getMessage()); + exception.printStackTrace(); + } + } + Assert.assertEquals(errors.get(), 0, "No exceptions should occur during concurrent operations"); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java index 4cba12fc3dfa5..79fd3b03d1fc7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java @@ -20,7 +20,12 @@ import com.carrotsearch.hppc.ObjectSet; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -34,6 +39,7 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.api.proto.MessageMetadata; import org.awaitility.reflect.WhiteboxImpl; import org.mockito.Mockito; import org.testng.Assert; @@ -152,4 +158,97 @@ public void testSkipReadEntriesFromCloseCursor() throws Exception { // Verify: the topic can be deleted successfully. admin.topics().delete(topicName, false); } + + @Test + public void testRaceConditionInTrackDelayedDelivery() throws Exception { + final int numThreads = 16; + final int operationsPerThread = 2000; + final CountDownLatch startLatch = new CountDownLatch(1); + final CountDownLatch doneLatch = new CountDownLatch(numThreads); + final AtomicInteger errors = new AtomicInteger(0); + final AtomicReference firstException = new AtomicReference<>(); + + final String topicName = newTopicName(); + final String subscription = "s1"; + + // Needed to create the topic + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName).subscriptionName(subscription) + .subscriptionType(SubscriptionType.Shared).subscribe(); + + PersistentTopic topic = (PersistentTopic) getTopic(topicName, false).join().get(); + + ManagedCursor cursor = Mockito.mock(ManagedCursorImpl.class); + Mockito.doReturn(subscription).when(cursor).getName(); + + Subscription sub = Mockito.mock(PersistentSubscription.class); + Mockito.doReturn(topic).when(sub).getTopic(); + + PersistentDispatcherMultipleConsumers dispatcher = + new PersistentDispatcherMultipleConsumers(topic, cursor, sub); + + // Align all writes to the same bucket + // This is the key which triggers the race condition + long deliverAt = System.currentTimeMillis() + 5000; + + MessageMetadata messageMetadata = new MessageMetadata() + .setSequenceId(1) + .setProducerName("testProducer") + .setPartitionKeyB64Encoded(false) + .setPublishTime(System.currentTimeMillis()) + .setDeliverAtTime(deliverAt); + + @Cleanup("shutdown") + ExecutorService executorService = Executors.newFixedThreadPool(32); + + // Start clear message thread + for (int i = 0; i < numThreads / 2; i++) { + executorService.submit(() -> { + try { + startLatch.await(); + for (int j = 0; j < operationsPerThread; j++) { + dispatcher.clearDelayedMessages(); + Thread.sleep(1); + } + } catch (Exception e) { + errors.incrementAndGet(); + firstException.compareAndSet(null, e); + e.printStackTrace(); + } finally { + doneLatch.countDown(); + } + }); + } + + // Start track delayed delivery thread + for (int i = numThreads / 2; i < numThreads; i++) { + executorService.submit(() -> { + try { + startLatch.await(); + for (int j = 0; j < operationsPerThread; j++) { + dispatcher.trackDelayedDelivery(1, 1, messageMetadata); + Thread.sleep(1); + } + } catch (Exception e) { + errors.incrementAndGet(); + firstException.compareAndSet(null, e); + e.printStackTrace(); + } finally { + doneLatch.countDown(); + } + }); + } + + startLatch.countDown(); + Assert.assertTrue(doneLatch.await(30, TimeUnit.SECONDS), "Test should complete within 30 seconds"); + + if (errors.get() > 0) { + Exception exception = firstException.get(); + if (exception != null) { + System.err.println("First exception caught: " + exception.getMessage()); + exception.printStackTrace(); + } + } + Assert.assertEquals(errors.get(), 0, "No exceptions should occur during concurrent operations"); + } } From 5747c09cc3d1cd76a8416dab94da0b85bd2497aa Mon Sep 17 00:00:00 2001 From: zhou zhuohan <843520313@qq.com> Date: Thu, 23 Apr 2026 20:50:39 +0800 Subject: [PATCH 060/201] [fix][broker] Clean up orphan ledger on concurrent initial schema creation in BookkeeperSchemaStorage (#25514) --- .../mledger/impl/LedgerMetadataUtils.java | 30 ++++----- .../schema/BookkeeperSchemaStorage.java | 44 +++++++++++-- .../org/apache/pulsar/schema/SchemaTest.java | 65 +++++++++++++++++++ .../client/PulsarMockBookKeeper.java | 2 +- .../client/PulsarMockLedgerHandle.java | 22 +++++-- 5 files changed, 136 insertions(+), 27 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java index 4107949de51b9..9e1529d068388 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java @@ -29,28 +29,28 @@ */ public final class LedgerMetadataUtils { - private static final String METADATA_PROPERTY_APPLICATION = "application"; - private static final byte[] METADATA_PROPERTY_APPLICATION_PULSAR = "pulsar".getBytes(StandardCharsets.UTF_8); + public static final String METADATA_PROPERTY_APPLICATION = "application"; + public static final byte[] METADATA_PROPERTY_APPLICATION_PULSAR = "pulsar".getBytes(StandardCharsets.UTF_8); - private static final String METADATA_PROPERTY_COMPONENT = "component"; - private static final byte[] METADATA_PROPERTY_COMPONENT_MANAGED_LEDGER = + public static final String METADATA_PROPERTY_COMPONENT = "component"; + public static final byte[] METADATA_PROPERTY_COMPONENT_MANAGED_LEDGER = "managed-ledger".getBytes(StandardCharsets.UTF_8); - private static final byte[] METADATA_PROPERTY_COMPONENT_COMPACTED_LEDGER = + public static final byte[] METADATA_PROPERTY_COMPONENT_COMPACTED_LEDGER = "compacted-ledger".getBytes(StandardCharsets.UTF_8); - private static final byte[] METADATA_PROPERTY_COMPONENT_SCHEMA = "schema".getBytes(StandardCharsets.UTF_8); + public static final byte[] METADATA_PROPERTY_COMPONENT_SCHEMA = "schema".getBytes(StandardCharsets.UTF_8); - private static final byte[] METADATA_PROPERTY_COMPONENT_DELAYED_INDEX_BUCKET = + public static final byte[] METADATA_PROPERTY_COMPONENT_DELAYED_INDEX_BUCKET = "delayed-index-bucket".getBytes(StandardCharsets.UTF_8); - private static final String METADATA_PROPERTY_MANAGED_LEDGER_NAME = "pulsar/managed-ledger"; - private static final String METADATA_PROPERTY_CURSOR_NAME = "pulsar/cursor"; - private static final String METADATA_PROPERTY_COMPACTEDTOPIC = "pulsar/compactedTopic"; - private static final String METADATA_PROPERTY_COMPACTEDTO = "pulsar/compactedTo"; - private static final String METADATA_PROPERTY_SCHEMAID = "pulsar/schemaId"; + public static final String METADATA_PROPERTY_MANAGED_LEDGER_NAME = "pulsar/managed-ledger"; + public static final String METADATA_PROPERTY_CURSOR_NAME = "pulsar/cursor"; + public static final String METADATA_PROPERTY_COMPACTEDTOPIC = "pulsar/compactedTopic"; + public static final String METADATA_PROPERTY_COMPACTEDTO = "pulsar/compactedTo"; + public static final String METADATA_PROPERTY_SCHEMAID = "pulsar/schemaId"; - private static final String METADATA_PROPERTY_DELAYED_INDEX_BUCKET_KEY = "pulsar/delayedIndexBucketKey"; - private static final String METADATA_PROPERTY_DELAYED_INDEX_TOPIC = "pulsar/delayedIndexTopic"; - private static final String METADATA_PROPERTY_DELAYED_INDEX_CURSOR = "pulsar/delayedIndexCursor"; + public static final String METADATA_PROPERTY_DELAYED_INDEX_BUCKET_KEY = "pulsar/delayedIndexBucketKey"; + public static final String METADATA_PROPERTY_DELAYED_INDEX_TOPIC = "pulsar/delayedIndexTopic"; + public static final String METADATA_PROPERTY_DELAYED_INDEX_CURSOR = "pulsar/delayedIndexCursor"; /** * Build base metadata for every ManagedLedger. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index 74b41c1be8414..3d5ee246c22e9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -355,23 +355,51 @@ private CompletableFuture putSchema(String schemaId, byte[] data, byte[] h } private CompletableFuture createNewSchema(String schemaId, byte[] data, byte[] hash) { + // Step 1: Store the schema data into a new BookKeeper ledger IndexEntry emptyIndex = new IndexEntry(); emptyIndex.setVersion(0); emptyIndex.setHash(hash); emptyIndex.setPosition().setEntryId(-1L).setLedgerId(-1L); + CompletableFuture stored = addNewSchemaEntryToStore( + schemaId, Collections.singletonList(emptyIndex), data + ); - return addNewSchemaEntryToStore(schemaId, Collections.singletonList(emptyIndex), data).thenCompose(position -> { - // The schema was stored in the ledger, now update the z-node with the pointer to it + return stored.thenCompose(position -> { + // Step 2: Create the schema locator z-node pointing to the ledger IndexEntry info = new IndexEntry(); info.setVersion(0); info.setPosition().copyFrom(position); info.setHash(hash); - SchemaLocator locator = new SchemaLocator(); locator.setInfo().copyFrom(info); locator.addIndex().copyFrom(info); - return createSchemaLocator(getSchemaPath(schemaId), locator) - .thenApply(ignore -> 0L); + CompletableFuture created = createSchemaLocator( + getSchemaPath(schemaId), locator).thenApply(ignore -> 0L); + + // Step 3: Handle failure by cleaning up the orphan ledger + // if concurrent schema creation caused a CAS conflict + return created.whenComplete((__, ex) -> { + if (ex == null) { + return; + } + Throwable cause = FutureUtil.unwrapCompletionException(ex); + log.warn() + .attr("schemaId", schemaId) + .attr("ledgerId", position.getLedgerId()) + .exception(cause) + .log("Failed to create schema locator with position"); + if (cause instanceof AlreadyExistsException || cause instanceof BadVersionException) { + bookKeeper.asyncDeleteLedger(position.getLedgerId(), (rc, ctx) -> { + if (rc != BKException.Code.OK) { + log.warn() + .attr("schemaId", schemaId) + .attr("ledgerId", position.getLedgerId()) + .attr("rc", rc) + .log("Failed to delete orphan ledger after schema locator creation failed"); + } + }, null); + } + }); }); } @@ -484,7 +512,11 @@ private CompletableFuture updateSchemaLocator( ).thenApply(ignore -> nextVersion).whenComplete((__, ex) -> { if (ex != null) { Throwable cause = FutureUtil.unwrapCompletionException(ex); - log.warn("[{}] Failed to update schema locator with position {}", schemaId, position, cause); + log.warn() + .attr("schemaId", schemaId) + .attr("ledgerId", position.getLedgerId()) + .exception(cause) + .log("Failed to update schema locator with position"); if (cause instanceof AlreadyExistsException || cause instanceof BadVersionException) { bookKeeper.asyncDeleteLedger(position.getLedgerId(), new AsyncCallback.DeleteCallback() { @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index 393ed0a84b17a..23a61636072cf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -55,6 +55,7 @@ import org.apache.avro.Schema.Parser; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.mledger.impl.LedgerMetadataUtils; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; @@ -1372,6 +1373,70 @@ public void testCreateSchemaInParallel() throws Exception { producers2.clear(); } + /** + * Test that when multiple producers concurrently create schema for a brand-new topic, + * the orphan BookKeeper ledgers created by the losing requests are properly cleaned up. + */ + @Test + public void testConcurrentCreateSchemaNoOrphanLedger() throws Exception { + final String namespace = "test-namespace-" + randomName(16); + String ns = PUBLIC_TENANT + "/" + namespace; + admin.namespaces().createNamespace(ns, Sets.newHashSet(CLUSTER_NAME)); + + final String topic = getTopicName(ns, "testConcurrentCreateSchemaNoOrphanLedger"); + final String schemaName = TopicName.get(topic).getSchemaName(); + + org.apache.bookkeeper.client.PulsarMockBookKeeper mockBk = + (org.apache.bookkeeper.client.PulsarMockBookKeeper) pulsar.getBookKeeperClient(); + + // Concurrently create producers with the same schema on a brand-new topic + int concurrency = 16; + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newFixedThreadPool(concurrency); + List>> producers = + Collections.synchronizedList(new ArrayList<>(concurrency)); + CountDownLatch latch = new CountDownLatch(concurrency); + for (int i = 0; i < concurrency; i++) { + executor.execute(() -> { + try { + producers.add(pulsarClient.newProducer(Schema.AVRO(Schemas.PersonOne.class)) + .topic(topic).createAsync()); + } finally { + latch.countDown(); + } + }); + } + latch.await(); + FutureUtil.waitForAll(producers).join(); + + // Verify only 1 schema version exists + assertEquals(admin.schemas().getAllSchemas(topic).size(), 1); + + // Count surviving BK ledgers whose customMetadata "pulsar/schemaId" matches this topic's schemaName. + // If orphan ledgers were not cleaned up, there would be more than 1. + int schemaLedgerCount = 0; + for (org.apache.bookkeeper.client.PulsarMockLedgerHandle lh : mockBk.getLedgerMap().values()) { + Map metadata = lh.getLedgerMetadata().getCustomMetadata(); + byte[] schemaIdBytes = metadata.get(LedgerMetadataUtils.METADATA_PROPERTY_SCHEMAID); + if (schemaIdBytes != null + && schemaName.equals(new String(schemaIdBytes, java.nio.charset.StandardCharsets.UTF_8))) { + schemaLedgerCount++; + } + } + assertEquals(schemaLedgerCount, 1, + "Expected exactly 1 schema ledger for the topic, but found " + + schemaLedgerCount + ". Orphan ledgers were not cleaned up."); + + // Cleanup + producers.forEach(p -> { + try { + p.join().close(); + } catch (Exception ignore) { + } + }); + producers.clear(); + } + @EqualsAndHashCode static class User implements Serializable { private String name; diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java index 58fddb07ff982..1cc7765fb79bd 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java @@ -137,7 +137,7 @@ public void asyncCreateLedger(int ensSize, int writeQuorumSize, int ackQuorumSiz long id = sequence.getAndIncrement(); log.info("Creating ledger {}", id); PulsarMockLedgerHandle lh = - new PulsarMockLedgerHandle(PulsarMockBookKeeper.this, id, digestType, passwd); + new PulsarMockLedgerHandle(PulsarMockBookKeeper.this, id, digestType, passwd, properties); ledgers.put(id, lh); return FutureUtils.value(lh); } catch (Throwable t) { diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java index 74dab3ebbfeb3..a540377413346 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java @@ -28,6 +28,7 @@ import java.util.Collections; import java.util.Enumeration; import java.util.List; +import java.util.Map; import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; @@ -69,7 +70,14 @@ public class PulsarMockLedgerHandle extends LedgerHandle { public PulsarMockLedgerHandle(PulsarMockBookKeeper bk, long id, DigestType digest, byte[] passwd) throws GeneralSecurityException { - super(bk.getClientCtx(), id, new Versioned<>(createMetadata(id, digest, passwd), new LongVersion(0L)), + this(bk, id, digest, passwd, Collections.emptyMap()); + } + + public PulsarMockLedgerHandle(PulsarMockBookKeeper bk, long id, + DigestType digest, byte[] passwd, + Map customMetadata) throws GeneralSecurityException { + super(bk.getClientCtx(), id, + new Versioned<>(createMetadata(id, digest, passwd, customMetadata), new LongVersion(0L)), digest, passwd, WriteFlag.NONE); this.bk = bk; this.id = id; @@ -274,14 +282,18 @@ public CompletableFuture readLastAddConfirmedAndEntryAsyn return readHandle.readLastAddConfirmedAndEntryAsync(entryId, timeOutInMillis, parallel); } - private static LedgerMetadata createMetadata(long id, DigestType digest, byte[] passwd) { + private static LedgerMetadata createMetadata(long id, DigestType digest, byte[] passwd, + Map customMetadata) { List ensemble = new ArrayList<>(PulsarMockBookKeeper.getMockEnsemble()); - return LedgerMetadataBuilder.create() + LedgerMetadataBuilder builder = LedgerMetadataBuilder.create() .withDigestType(digest.toApiDigestType()) .withPassword(passwd) .withId(id) - .newEnsembleEntry(0L, ensemble) - .build(); + .newEnsembleEntry(0L, ensemble); + if (customMetadata != null && !customMetadata.isEmpty()) { + builder.withCustomMetadata(customMetadata); + } + return builder.build(); } private static final Logger log = LoggerFactory.getLogger(PulsarMockLedgerHandle.class); From 0517f4822118f1ae728db699c9b9f0da4e5667a8 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 12 May 2026 09:45:49 +0800 Subject: [PATCH 061/201] fix log --- .../schema/BookkeeperSchemaStorage.java | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index 3d5ee246c22e9..40cc59c6a15d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -383,19 +383,13 @@ private CompletableFuture createNewSchema(String schemaId, byte[] data, by return; } Throwable cause = FutureUtil.unwrapCompletionException(ex); - log.warn() - .attr("schemaId", schemaId) - .attr("ledgerId", position.getLedgerId()) - .exception(cause) - .log("Failed to create schema locator with position"); + log.warn("Failed to create schema locator with position, schemaId {}, ledgerId {}", + schemaId, position.getLedgerId(), ex); if (cause instanceof AlreadyExistsException || cause instanceof BadVersionException) { bookKeeper.asyncDeleteLedger(position.getLedgerId(), (rc, ctx) -> { if (rc != BKException.Code.OK) { - log.warn() - .attr("schemaId", schemaId) - .attr("ledgerId", position.getLedgerId()) - .attr("rc", rc) - .log("Failed to delete orphan ledger after schema locator creation failed"); + log.warn("Failed to delete orphan ledger after schema locator creation" + + " failed, schemaId {}, ledgerId {}, rc: {}", schemaId, position.getLedgerId(), rc); } }, null); } @@ -512,11 +506,8 @@ private CompletableFuture updateSchemaLocator( ).thenApply(ignore -> nextVersion).whenComplete((__, ex) -> { if (ex != null) { Throwable cause = FutureUtil.unwrapCompletionException(ex); - log.warn() - .attr("schemaId", schemaId) - .attr("ledgerId", position.getLedgerId()) - .exception(cause) - .log("Failed to update schema locator with position"); + log.warn("Failed to update schema locator with position, schemaId {}, ledgerId {}", + schemaId, position.getLedgerId(), ex); if (cause instanceof AlreadyExistsException || cause instanceof BadVersionException) { bookKeeper.asyncDeleteLedger(position.getLedgerId(), new AsyncCallback.DeleteCallback() { @Override From 44f6bbccc4c628ee59ab11ec6550b62067927b55 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 12 May 2026 09:46:46 +0800 Subject: [PATCH 062/201] fix checkstyle --- .../pulsar/broker/service/schema/BookkeeperSchemaStorage.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index 40cc59c6a15d8..9b71daff91f4a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -389,7 +389,8 @@ private CompletableFuture createNewSchema(String schemaId, byte[] data, by bookKeeper.asyncDeleteLedger(position.getLedgerId(), (rc, ctx) -> { if (rc != BKException.Code.OK) { log.warn("Failed to delete orphan ledger after schema locator creation" - + " failed, schemaId {}, ledgerId {}, rc: {}", schemaId, position.getLedgerId(), rc); + + " failed, schemaId {}, ledgerId {}, rc: {}", + schemaId, position.getLedgerId(), rc); } }, null); } From e09461b37a3bef0bca39f9a2811687b8da92bdef Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 11 May 2026 23:36:17 +0300 Subject: [PATCH 063/201] [fix][sec] Upgrade vertx to 4.5.27 to address CVE-2026-6860 (#25745) (cherry picked from commit 2964ab59af0fb65406b9cfe3a632f4de5c628b9a) --- distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++----- pom.xml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 18a5c3f59cdc7..175c4c317a2d9 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -509,11 +509,11 @@ The Apache Software License, Version 2.0 * JCTools - Java Concurrency Tools for the JVM - org.jctools-jctools-core-4.0.5.jar * Vertx - - io.vertx-vertx-auth-common-4.5.25.jar - - io.vertx-vertx-bridge-common-4.5.25.jar - - io.vertx-vertx-core-4.5.25.jar - - io.vertx-vertx-web-4.5.25.jar - - io.vertx-vertx-web-common-4.5.25.jar + - io.vertx-vertx-auth-common-4.5.27.jar + - io.vertx-vertx-bridge-common-4.5.27.jar + - io.vertx-vertx-core-4.5.27.jar + - io.vertx-vertx-web-4.5.27.jar + - io.vertx-vertx-web-common-4.5.27.jar * Apache ZooKeeper - org.apache.zookeeper-zookeeper-jute-3.9.5.jar * Snappy Java diff --git a/pom.xml b/pom.xml index 38302335e7a36..0682e77f0b745 100644 --- a/pom.xml +++ b/pom.xml @@ -198,7 +198,7 @@ flexible messaging model and an intuitive client API. 2.42 1.10.62 0.16.0 - 4.5.25 + 4.5.27 7.9.2 2.0.17 4.5.0 From 7dd424b65c32030f04d19eedfb016c17f3bbe49c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 11 May 2026 23:36:05 +0300 Subject: [PATCH 064/201] [fix][sec] Upgrade thrift to 0.23.0 to address CVE-2026-43869 (#25744) (cherry picked from commit 6c2fdef849acf2c66bddc8631bc86909adfe0d8e) --- .../server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 175c4c317a2d9..fcb664ade0582 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -432,7 +432,7 @@ The Apache Software License, Version 2.0 * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.45.0.jar - * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar + * Apache Thrift - org.apache.thrift-libthrift-0.23.0.jar * OkHttp3 - com.squareup.okhttp3-logging-interceptor-5.3.1.jar - com.squareup.okhttp3-okhttp-5.3.1.jar diff --git a/pom.xml b/pom.xml index 0682e77f0b745..5e0f310c8eb40 100644 --- a/pom.xml +++ b/pom.xml @@ -304,6 +304,7 @@ flexible messaging model and an intuitive client API. 1.10.12 5.5.0 3.4.3 + 0.23.0 @@ -1346,6 +1347,45 @@ flexible messaging model and an intuitive client API. + + + org.apache.thrift + libthrift + ${thrift.version} + + + + org.apache.tomcat.embed + tomcat-embed-core + + + javax.annotation + javax.annotation-api + + + jakarta.annotation + jakarta.annotation-api + + + jakarta.servlet + jakarta.servlet-api + + + org.apache.httpcomponents.client5 + httpclient5 + + + org.apache.httpcomponents.core5 + httpcore5 + + + org.apache.httpcomponents.core5 + httpcore5-h2 + + + org.apache.commons From ca3e8a40730057ad374a3dfc595037333c7aca1d Mon Sep 17 00:00:00 2001 From: sinan liu Date: Mon, 27 Apr 2026 11:30:24 +0800 Subject: [PATCH 065/201] [fix][broker] Wait for orphan schema ledger cleanup before retry (#25579) --- .../schema/BookkeeperSchemaStorage.java | 88 +++++++++++-------- .../org/apache/pulsar/schema/SchemaTest.java | 88 +++++++++++++++---- 2 files changed, 121 insertions(+), 55 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index 9b71daff91f4a..d92bd8a2bc724 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -39,7 +39,11 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.function.Function; +<<<<<<< HEAD import org.apache.bookkeeper.client.AsyncCallback; +======= +import lombok.CustomLog; +>>>>>>> f07decea04 ([fix][broker] Wait for orphan schema ledger cleanup before retry (#25579)) import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; @@ -373,28 +377,21 @@ private CompletableFuture createNewSchema(String schemaId, byte[] data, by SchemaLocator locator = new SchemaLocator(); locator.setInfo().copyFrom(info); locator.addIndex().copyFrom(info); - CompletableFuture created = createSchemaLocator( - getSchemaPath(schemaId), locator).thenApply(ignore -> 0L); - // Step 3: Handle failure by cleaning up the orphan ledger // if concurrent schema creation caused a CAS conflict - return created.whenComplete((__, ex) -> { - if (ex == null) { - return; - } - Throwable cause = FutureUtil.unwrapCompletionException(ex); - log.warn("Failed to create schema locator with position, schemaId {}, ledgerId {}", - schemaId, position.getLedgerId(), ex); - if (cause instanceof AlreadyExistsException || cause instanceof BadVersionException) { - bookKeeper.asyncDeleteLedger(position.getLedgerId(), (rc, ctx) -> { - if (rc != BKException.Code.OK) { - log.warn("Failed to delete orphan ledger after schema locator creation" - + " failed, schemaId {}, ledgerId {}, rc: {}", - schemaId, position.getLedgerId(), rc); + return createSchemaLocator(getSchemaPath(schemaId), locator) + .thenApply(ignore -> 0L) + .exceptionallyCompose(ex -> { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + log.warn("Failed to create schema locator with position, schemaId {}, ledgerId {}", + schemaId, position.getLedgerId(), cause); + if (cause instanceof AlreadyExistsException || cause instanceof BadVersionException) { + return deleteLedgerAsync(schemaId, position.getLedgerId(), + "schema locator creation failed") + .thenCompose(__ -> FutureUtil.failedFuture(cause)); } - }, null); - } - }); + return FutureUtil.failedFuture(cause); + }); }); } @@ -504,26 +501,45 @@ private CompletableFuture updateSchemaLocator( return updateSchemaLocator(getSchemaPath(schemaId), newLocator , locatorEntry.version - ).thenApply(ignore -> nextVersion).whenComplete((__, ex) -> { - if (ex != null) { - Throwable cause = FutureUtil.unwrapCompletionException(ex); - log.warn("Failed to update schema locator with position, schemaId {}, ledgerId {}", - schemaId, position.getLedgerId(), ex); - if (cause instanceof AlreadyExistsException || cause instanceof BadVersionException) { - bookKeeper.asyncDeleteLedger(position.getLedgerId(), new AsyncCallback.DeleteCallback() { - @Override - public void deleteComplete(int rc, Object ctx) { - if (rc != BKException.Code.OK) { - log.warn("[{}] Failed to delete ledger {} after updating schema locator failed, rc: {}", - schemaId, position.getLedgerId(), rc); - } - } - }, null); - } + ).thenApply(ignore -> nextVersion).exceptionallyCompose(ex -> { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + log.warn("Failed to update schema locator with position, schemaId {}, ledgerId {}", + schemaId, position.getLedgerId(), cause); + if (cause instanceof AlreadyExistsException || cause instanceof BadVersionException) { + return deleteLedgerAsync(schemaId, position.getLedgerId(), + "schema locator update failed") + .thenCompose(__ -> FutureUtil.failedFuture(cause)); } + return FutureUtil.failedFuture(cause); }); } + private CompletableFuture deleteLedgerAsync(String schemaId, long ledgerId, String reason) { + CompletableFuture future = new CompletableFuture<>(); + try { + bookKeeper.asyncDeleteLedger(ledgerId, (rc, ctx) -> { + if (rc != BKException.Code.OK) { + log.warn() + .attr("schemaId", schemaId) + .attr("ledgerId", ledgerId) + .attr("rc", rc) + .attr("reason", reason) + .log("Failed to delete orphan schema ledger"); + } + future.complete(null); + }, null); + } catch (Throwable t) { + log.warn() + .attr("schemaId", schemaId) + .attr("ledgerId", ledgerId) + .attr("reason", reason) + .exception(t) + .log("Failed to trigger orphan schema ledger deletion"); + future.complete(null); + } + return future; + } + @NonNull private CompletableFuture findSchemaEntryByVersion( List index, @@ -809,4 +825,4 @@ public static CompletableFuture ignoreUnrecoverableBKException(Completabl throw t instanceof CompletionException ? (CompletionException) t : new CompletionException(t); }); } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index 23a61636072cf..fa0d3bdc9b202 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -55,6 +55,8 @@ import org.apache.avro.Schema.Parser; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.PulsarMockBookKeeper; +import org.apache.bookkeeper.client.PulsarMockLedgerHandle; import org.apache.bookkeeper.mledger.impl.LedgerMetadataUtils; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; @@ -1386,48 +1388,96 @@ public void testConcurrentCreateSchemaNoOrphanLedger() throws Exception { final String topic = getTopicName(ns, "testConcurrentCreateSchemaNoOrphanLedger"); final String schemaName = TopicName.get(topic).getSchemaName(); - org.apache.bookkeeper.client.PulsarMockBookKeeper mockBk = - (org.apache.bookkeeper.client.PulsarMockBookKeeper) pulsar.getBookKeeperClient(); + PulsarMockBookKeeper mockBk = (PulsarMockBookKeeper) pulsar.getBookKeeperClient(); // Concurrently create producers with the same schema on a brand-new topic int concurrency = 16; + List>> producers = createProducersInParallel( + topic, Schema.AVRO(Schemas.PersonOne.class), concurrency); + try { + FutureUtil.waitForAll(producers).join(); + + // Verify only 1 schema version exists + assertEquals(admin.schemas().getAllSchemas(topic).size(), 1); + + int schemaLedgerCount = countSchemaLedgers(mockBk, schemaName); + assertEquals(schemaLedgerCount, 1, + "Expected exactly 1 schema ledger for the topic, but found " + + schemaLedgerCount + ". Orphan ledgers were not cleaned up."); + } finally { + closeProducers(producers); + } + } + + /** + * Test that concurrent compatible schema updates clean up ledgers created by requests + * that lose the schema locator CAS race. + */ + @Test + public void testConcurrentUpdateSchemaNoOrphanLedger() throws Exception { + final String namespace = "test-namespace-" + randomName(16); + String ns = PUBLIC_TENANT + "/" + namespace; + admin.namespaces().createNamespace(ns, Sets.newHashSet(CLUSTER_NAME)); + + final String topic = getTopicName(ns, "testConcurrentUpdateSchemaNoOrphanLedger"); + final String schemaName = TopicName.get(topic).getSchemaName(); + PulsarMockBookKeeper mockBk = (PulsarMockBookKeeper) pulsar.getBookKeeperClient(); + + @Cleanup + Producer initialProducer = pulsarClient + .newProducer(Schema.AVRO(Schemas.PersonOne.class)) + .topic(topic) + .create(); + assertEquals(admin.schemas().getAllSchemas(topic).size(), 1); + + int concurrency = 16; + List>> producers = createProducersInParallel( + topic, Schema.AVRO(Schemas.PersonThree.class), concurrency); + try { + FutureUtil.waitForAll(producers).join(); + + assertEquals(admin.schemas().getAllSchemas(topic).size(), 2); + int schemaLedgerCount = countSchemaLedgers(mockBk, schemaName); + assertEquals(schemaLedgerCount, 2, + "Expected exactly 2 schema ledgers for the topic, but found " + + schemaLedgerCount + ". Orphan ledgers were not cleaned up."); + } finally { + closeProducers(producers); + } + } + + private List>> createProducersInParallel( + String topic, Schema schema, int concurrency) throws InterruptedException { @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(concurrency); - List>> producers = - Collections.synchronizedList(new ArrayList<>(concurrency)); + List>> producers = Collections.synchronizedList(new ArrayList<>(concurrency)); CountDownLatch latch = new CountDownLatch(concurrency); for (int i = 0; i < concurrency; i++) { executor.execute(() -> { try { - producers.add(pulsarClient.newProducer(Schema.AVRO(Schemas.PersonOne.class)) - .topic(topic).createAsync()); + producers.add(pulsarClient.newProducer(schema).topic(topic).createAsync()); } finally { latch.countDown(); } }); } latch.await(); - FutureUtil.waitForAll(producers).join(); - - // Verify only 1 schema version exists - assertEquals(admin.schemas().getAllSchemas(topic).size(), 1); + return producers; + } - // Count surviving BK ledgers whose customMetadata "pulsar/schemaId" matches this topic's schemaName. - // If orphan ledgers were not cleaned up, there would be more than 1. + private int countSchemaLedgers(PulsarMockBookKeeper mockBk, String schemaName) { int schemaLedgerCount = 0; - for (org.apache.bookkeeper.client.PulsarMockLedgerHandle lh : mockBk.getLedgerMap().values()) { + for (PulsarMockLedgerHandle lh : mockBk.getLedgerMap().values()) { Map metadata = lh.getLedgerMetadata().getCustomMetadata(); byte[] schemaIdBytes = metadata.get(LedgerMetadataUtils.METADATA_PROPERTY_SCHEMAID); - if (schemaIdBytes != null - && schemaName.equals(new String(schemaIdBytes, java.nio.charset.StandardCharsets.UTF_8))) { + if (schemaIdBytes != null && schemaName.equals(new String(schemaIdBytes, StandardCharsets.UTF_8))) { schemaLedgerCount++; } } - assertEquals(schemaLedgerCount, 1, - "Expected exactly 1 schema ledger for the topic, but found " - + schemaLedgerCount + ". Orphan ledgers were not cleaned up."); + return schemaLedgerCount; + } - // Cleanup + private void closeProducers(List>> producers) { producers.forEach(p -> { try { p.join().close(); From 62609ed293c3d34aa634ff36cf2922c73519aeaa Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 12 May 2026 17:42:48 +0800 Subject: [PATCH 066/201] fix checkstyle --- .../schema/BookkeeperSchemaStorage.java | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index d92bd8a2bc724..104c5bd0667d7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -39,11 +39,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.function.Function; -<<<<<<< HEAD -import org.apache.bookkeeper.client.AsyncCallback; -======= -import lombok.CustomLog; ->>>>>>> f07decea04 ([fix][broker] Wait for orphan schema ledger cleanup before retry (#25579)) import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; @@ -519,22 +514,14 @@ private CompletableFuture deleteLedgerAsync(String schemaId, long ledgerId try { bookKeeper.asyncDeleteLedger(ledgerId, (rc, ctx) -> { if (rc != BKException.Code.OK) { - log.warn() - .attr("schemaId", schemaId) - .attr("ledgerId", ledgerId) - .attr("rc", rc) - .attr("reason", reason) - .log("Failed to delete orphan schema ledger"); + log.warn("Failed to delete orphan schema ledger, schemaId {}, ledgerId {}, reason {}, rc {}", + schemaId, ledgerId, reason, rc); } future.complete(null); }, null); } catch (Throwable t) { - log.warn() - .attr("schemaId", schemaId) - .attr("ledgerId", ledgerId) - .attr("reason", reason) - .exception(t) - .log("Failed to trigger orphan schema ledger deletion"); + log.warn("Failed to trigger orphan schema ledger deletion, schemaId {}, ledgerId {}, reason {}", + schemaId, ledgerId, reason, t); future.complete(null); } return future; From bcd64102e0f3b7e483dfcebc87420ff210634f62 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 12 May 2026 18:30:06 +0800 Subject: [PATCH 067/201] [fix][broker] Fix race in pending acks removal in redeliverUnacknowledgedMessages (#25589) --- .../pulsar/broker/service/Consumer.java | 25 ++---- .../pulsar/broker/service/PendingAcksMap.java | 49 ++++++++++- .../broker/service/PendingAcksMapTest.java | 81 +++++++++++++++++++ 3 files changed, 136 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index a7ca3f8339f47..d37cecc276770 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -1148,8 +1148,6 @@ public int getPriorityLevel() { public void redeliverUnacknowledgedMessages(long consumerEpoch) { // cleanup unackedMessage bucket and redeliver those unack-msgs again - clearUnAckedMsgs(); - blockedConsumerOnUnackedMsgs = false; if (log.isDebugEnabled()) { log.debug("[{}-{}] consumer {} received redelivery", topicName, subscription, consumerId); } @@ -1157,7 +1155,7 @@ public void redeliverUnacknowledgedMessages(long consumerEpoch) { if (pendingAcks != null) { List pendingPositions = new ArrayList<>((int) pendingAcks.size()); MutableInt totalRedeliveryMessages = new MutableInt(0); - pendingAcks.forEach((ledgerId, entryId, batchSize, stickyKeyHash) -> { + pendingAcks.forEachAndClear((ledgerId, entryId, batchSize, stickyKeyHash) -> { int unAckedCount = (int) getUnAckedCountForBatchIndexLevelEnabled(PositionFactory.create(ledgerId, entryId), batchSize); @@ -1165,15 +1163,18 @@ public void redeliverUnacknowledgedMessages(long consumerEpoch) { pendingPositions.add(PositionFactory.create(ledgerId, entryId)); }); - for (Position p : pendingPositions) { - pendingAcks.remove(p.getLedgerId(), p.getEntryId()); + if (totalRedeliveryMessages.intValue() > 0) { + addAndGetUnAckedMsgs(this, -totalRedeliveryMessages.intValue()); } + blockedConsumerOnUnackedMsgs = false; msgRedeliver.recordMultipleEvents(totalRedeliveryMessages.intValue(), totalRedeliveryMessages.intValue()); msgRedeliverCounter.add(totalRedeliveryMessages.intValue()); subscription.redeliverUnacknowledgedMessages(this, pendingPositions); } else { + clearUnAckedMsgs(); + blockedConsumerOnUnackedMsgs = false; subscription.redeliverUnacknowledgedMessages(this, consumerEpoch); } @@ -1185,7 +1186,7 @@ public void redeliverUnacknowledgedMessages(List messageIds) { List pendingPositions = new ArrayList<>(); for (MessageIdData msg : messageIds) { Position position = PositionFactory.create(msg.getLedgerId(), msg.getEntryId()); - IntIntPair pendingAck = pendingAcks.get(position.getLedgerId(), position.getEntryId()); + IntIntPair pendingAck = pendingAcks.removeAndGet(position.getLedgerId(), position.getEntryId()); if (pendingAck != null) { int unAckedCount = (int) getUnAckedCountForBatchIndexLevelEnabled(position, pendingAck.leftInt()); pendingAcks.remove(position.getLedgerId(), position.getEntryId()); @@ -1206,17 +1207,7 @@ public void redeliverUnacknowledgedMessages(List messageIds) { msgRedeliver.recordMultipleEvents(totalRedeliveryMessages, totalRedeliveryMessages); msgRedeliverCounter.add(totalRedeliveryMessages); - int numberOfBlockedPermits = PERMITS_RECEIVED_WHILE_CONSUMER_BLOCKED_UPDATER.getAndSet(this, 0); - - // if permitsReceivedWhileConsumerBlocked has been accumulated then pass it to Dispatcher to flow messages - if (numberOfBlockedPermits > 0) { - MESSAGE_PERMITS_UPDATER.getAndAdd(this, numberOfBlockedPermits); - if (log.isDebugEnabled()) { - log.debug("[{}-{}] Added {} blockedPermits to broker.service.Consumer's messagePermits for consumer {}", - topicName, subscription, numberOfBlockedPermits, consumerId); - } - subscription.consumerFlow(this, numberOfBlockedPermits); - } + flowConsumerBlockedPermits(this); } public Subscription getSubscription() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java index 0f7802d8294e0..549aef36cec00 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java @@ -204,9 +204,26 @@ private void processPendingAcks(PendingAcksConsumer processor) { * @param processor the processor to handle each pending ack */ public void forEachAndClose(PendingAcksConsumer processor) { + internalForEachAndClear(processor, true); + } + + /** + * Iterate over all the pending acks and clear the map. + * Unlike {@link #forEachAndClose(PendingAcksConsumer)}, this method does not close the map, + * so new entries can still be added after this method returns. + * + * @param processor the processor to handle each pending ack + */ + public void forEachAndClear(PendingAcksConsumer processor) { + internalForEachAndClear(processor, false); + } + + private void internalForEachAndClear(PendingAcksConsumer processor, boolean close) { try { writeLock.lock(); - closed = true; + if (close) { + closed = true; + } PendingAcksRemoveHandler pendingAcksRemoveHandler = pendingAcksRemoveHandlerSupplier.get(); if (pendingAcksRemoveHandler != null) { try { @@ -226,7 +243,6 @@ public void forEachAndClose(PendingAcksConsumer processor) { writeLock.unlock(); } } - /** * Check if the map contains a pending ack for the given ledger ID and entry ID. * @@ -325,6 +341,35 @@ public boolean remove(long ledgerId, long entryId) { } } + /** + * Atomically remove and return the pending ack for the given ledger ID and entry ID. + * Unlike {@link #remove(long, long)}, this method returns the removed entry so the caller + * can access the batch size and sticky key hash without a separate get operation. + * + * @param ledgerId the ledger ID + * @param entryId the entry ID + * @return the removed entry as an IntIntPair (batchSize, stickyKeyHash), or null if not found + */ + public IntIntPair removeAndGet(long ledgerId, long entryId) { + try { + writeLock.lock(); + Long2ObjectSortedMap ledgerMap = pendingAcks.get(ledgerId); + if (ledgerMap == null) { + return null; + } + IntIntPair removedEntry = ledgerMap.remove(entryId); + if (removedEntry != null) { + handleRemovePendingAck(ledgerId, entryId, removedEntry.rightInt()); + } + if (removedEntry != null && ledgerMap.isEmpty()) { + pendingAcks.remove(ledgerId); + } + return removedEntry; + } finally { + writeLock.unlock(); + } + } + /** * Remove all pending acks up to the given ledger ID and entry ID, invoking a callback for each removed entry. * diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java index 8db0e3a0f73e6..ca32a1aee5f14 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PendingAcksMapTest.java @@ -29,6 +29,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import it.unimi.dsi.fastutil.ints.IntIntPair; import java.util.ArrayList; import java.util.List; import org.testng.annotations.Test; @@ -218,4 +219,84 @@ public void size_ReturnsCorrectSize() { assertEquals(pendingAcksMap.size(), 3); } + + @Test + public void forEachAndClear_ProcessesAndClearsAllPendingAcks() { + Consumer consumer = createMockConsumer("consumer1"); + PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null); + pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123); + pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 1, 124); + + List processedEntries = new ArrayList<>(); + pendingAcksMap.forEachAndClear((ledgerId, entryId, batchSize, stickyKeyHash) -> processedEntries.add(entryId)); + + assertEquals(processedEntries, List.of(1L, 2L)); + assertEquals(pendingAcksMap.size(), 0); + } + + @Test + public void forEachAndClear_AllowsAddingAfterClear() { + Consumer consumer = createMockConsumer("consumer1"); + PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null); + pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123); + + pendingAcksMap.forEachAndClear((ledgerId, entryId, batchSize, stickyKeyHash) -> {}); + + // Unlike forEachAndClose, forEachAndClear should allow new additions + boolean result = pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 1, 124); + assertTrue(result); + assertTrue(pendingAcksMap.contains(1L, 2L)); + } + + @Test + public void forEachAndClear_InvokesRemoveHandler() { + Consumer consumer = createMockConsumer("consumer1"); + PendingAcksMap.PendingAcksRemoveHandler removeHandler = mock(PendingAcksMap.PendingAcksRemoveHandler.class); + PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> removeHandler); + pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 1, 123); + pendingAcksMap.addPendingAckIfAllowed(1L, 2L, 1, 124); + + pendingAcksMap.forEachAndClear((ledgerId, entryId, batchSize, stickyKeyHash) -> {}); + + verify(removeHandler).startBatch(); + verify(removeHandler).handleRemoving(consumer, 1L, 1L, 123, false); + verify(removeHandler).handleRemoving(consumer, 1L, 2L, 124, false); + verify(removeHandler).endBatch(); + } + + @Test + public void removeAndGet_RemovesAndReturnsEntry() { + Consumer consumer = createMockConsumer("consumer1"); + PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null); + pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 5, 123); + + IntIntPair result = pendingAcksMap.removeAndGet(1L, 1L); + + assertTrue(result != null); + assertEquals(result.leftInt(), 5); + assertEquals(result.rightInt(), 123); + assertFalse(pendingAcksMap.contains(1L, 1L)); + } + + @Test + public void removeAndGet_ReturnsNullWhenNotFound() { + Consumer consumer = createMockConsumer("consumer1"); + PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> null); + + IntIntPair result = pendingAcksMap.removeAndGet(1L, 1L); + + assertTrue(result == null); + } + + @Test + public void removeAndGet_InvokesRemoveHandler() { + Consumer consumer = createMockConsumer("consumer1"); + PendingAcksMap.PendingAcksRemoveHandler removeHandler = mock(PendingAcksMap.PendingAcksRemoveHandler.class); + PendingAcksMap pendingAcksMap = new PendingAcksMap(consumer, () -> null, () -> removeHandler); + pendingAcksMap.addPendingAckIfAllowed(1L, 1L, 5, 123); + + pendingAcksMap.removeAndGet(1L, 1L); + + verify(removeHandler).handleRemoving(consumer, 1L, 1L, 123, false); + } } \ No newline at end of file From c08b1e9502c27b5044f7b243d40f3660be9e34ee Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 12 May 2026 18:52:00 +0800 Subject: [PATCH 068/201] [fix][broker] Move pending acks cleanup to selected mark-delete callbacks (#25592) --- .../pulsar/broker/service/Consumer.java | 132 ++++++++++-------- .../pulsar/broker/service/Dispatcher.java | 9 ++ .../pulsar/broker/service/PendingAcksMap.java | 35 ++++- ...PersistentDispatcherMultipleConsumers.java | 24 ++-- .../PersistentMessageExpiryMonitor.java | 10 +- ...tStickyKeyDispatcherMultipleConsumers.java | 1 + ...KeyDispatcherMultipleConsumersClassic.java | 4 +- .../persistent/PersistentSubscription.java | 6 +- .../client/impl/TransactionEndToEndTest.java | 8 -- 9 files changed, 140 insertions(+), 89 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index d37cecc276770..8d897c80062f7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -253,6 +253,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo OPEN_TELEMETRY_ATTRIBUTES_FIELD_UPDATER.set(this, null); } + @VisibleForTesting Consumer(String consumerName, int availablePermits) { this.subscription = null; @@ -378,9 +379,18 @@ public Future sendMessages(final List entries, } else { stickyKeyHash = stickyKeyHashes.get(i); } - boolean sendingAllowed = - pendingAcks.addPendingAckIfAllowed(entry.getLedgerId(), entry.getEntryId(), batchSize, - stickyKeyHash); + boolean sendingAllowed; + long[] ackSet = batchIndexesAcks == null ? null : batchIndexesAcks.getAckSet(i); + int remainingUnacked; + if (ackSet != null) { + remainingUnacked = BitSet.valueOf(ackSet).cardinality(); + unackedMessages -= (batchSize - remainingUnacked); + } else { + remainingUnacked = batchSize; + } + sendingAllowed = + pendingAcks.addPendingAckIfAllowed(entry.getLedgerId(), entry.getEntryId(), + remainingUnacked, stickyKeyHash); if (!sendingAllowed) { // sending isn't allowed when pending acks doesn't accept adding the entry // this happens when Key_Shared draining hashes contains the stickyKeyHash @@ -395,10 +405,6 @@ public Future sendMessages(final List entries, consumerId); } } else { - long[] ackSet = batchIndexesAcks == null ? null : batchIndexesAcks.getAckSet(i); - if (ackSet != null) { - unackedMessages -= (batchSize - BitSet.valueOf(ackSet).cardinality()); - } if (log.isDebugEnabled()) { log.debug("[{}-{}] Added {}:{} ledger entry with batchSize of {} to pendingAcks in" + " broker.service.Consumer for consumerId: {}", @@ -578,7 +584,7 @@ private CompletableFuture individualAckNormal(CommandAck ack, Map ackOwnerConsumerAndBatchSize = getAckOwnerConsumerAndBatchSize(msgId.getLedgerId(), msgId.getEntryId()); Consumer ackOwnerConsumer = ackOwnerConsumerAndBatchSize.left(); - long ackedCount; + long ackedCount = 0; int batchSize = ackOwnerConsumerAndBatchSize.rightInt(); if (msgId.getAckSetsCount() > 0) { long[] ackSets = new long[msgId.getAckSetsCount()]; @@ -594,11 +600,19 @@ private CompletableFuture individualAckNormal(CommandAck ack, Map 0) { + boolean updated = ackOwnerConsumer.updateRemainingUnacked( + position.getLedgerId(), position.getEntryId(), (int) ackedCount); + if (updated) { + addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount); + } + } } else { position = PositionFactory.create(msgId.getLedgerId(), msgId.getEntryId()); - ackedCount = getAckedCountForMsgIdNoAckSets(batchSize, position, ackOwnerConsumer); - if (checkCanRemovePendingAcksAndHandle(ackOwnerConsumer, position, msgId)) { + IntIntPair removed = ackOwnerConsumer.removePendingAckAndGet( + position.getLedgerId(), position.getEntryId()); + if (removed != null) { + ackedCount = removed.leftInt(); addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount); updateBlockedConsumerOnUnackedMsgs(ackOwnerConsumer); } @@ -676,12 +690,22 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { } AckSetStateUtil.getAckSetState(position).setAckSet(ackSets); ackedCount = getAckedCountForTransactionAck(batchSize, ackSets); + if (ackedCount > 0) { + boolean updated = ackOwnerConsumer.updateRemainingUnacked( + position.getLedgerId(), position.getEntryId(), (int) ackedCount); + if (updated) { + addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount); + } + } + } else { + IntIntPair removed = ackOwnerConsumer.removePendingAckAndGet( + position.getLedgerId(), position.getEntryId()); + if (removed != null) { + addAndGetUnAckedMsgs(ackOwnerConsumer, -removed.leftInt()); + updateBlockedConsumerOnUnackedMsgs(ackOwnerConsumer); + } } - addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount); - - checkCanRemovePendingAcksAndHandle(ackOwnerConsumer, position, msgId); - checkAckValidationError(ack, position); totalAckCount.add(ackedCount); @@ -705,16 +729,6 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { return completableFuture.thenApply(__ -> totalAckCount.sum()); } - private long getAckedCountForMsgIdNoAckSets(int batchSize, Position position, Consumer consumer) { - if (isAcknowledgmentAtBatchIndexLevelEnabled && Subscription.isIndividualAckMode(subType)) { - long[] cursorAckSet = getCursorAckSet(position); - if (cursorAckSet != null) { - return getAckedCountForBatchIndexLevelEnabled(position, batchSize, EMPTY_ACK_SET, consumer); - } - } - return batchSize; - } - private long getAckedCountForBatchIndexLevelEnabled(Position position, int batchSize, long[] ackSets, Consumer consumer) { long ackedCount = 0; @@ -744,19 +758,6 @@ private long getAckedCountForTransactionAck(int batchSize, long[] ackSets) { return ackedCount; } - private long getUnAckedCountForBatchIndexLevelEnabled(Position position, int batchSize) { - long unAckedCount = batchSize; - if (isAcknowledgmentAtBatchIndexLevelEnabled) { - long[] cursorAckSet = getCursorAckSet(position); - if (cursorAckSet != null) { - BitSetRecyclable cursorBitSet = BitSetRecyclable.create().resetWords(cursorAckSet); - unAckedCount = cursorBitSet.cardinality(); - cursorBitSet.recycle(); - } - } - return unAckedCount; - } - private void checkAckValidationError(CommandAck ack, Position position) { if (ack.hasValidationError()) { log.warn("[{}] [{}] Received ack for corrupted message at {} - Reason: {}", subscription, @@ -764,14 +765,6 @@ private void checkAckValidationError(CommandAck ack, Position position) { } } - private boolean checkCanRemovePendingAcksAndHandle(Consumer ackOwnedConsumer, - Position position, MessageIdData msgId) { - if (Subscription.isIndividualAckMode(subType) && msgId.getAckSetsCount() == 0) { - return removePendingAcks(ackOwnedConsumer, position); - } - return false; - } - /** * Retrieves the acknowledgment owner consumer and batch size for the specified ledgerId and entryId. * @@ -1113,6 +1106,37 @@ public PendingAcksMap getPendingAcks() { return pendingAcks; } + /** + * Atomically decrement the remaining unacked count for the specified position + * by the given acknowledged delta. + * + *

No-op if {@code pendingAcks} is not initialized. + * + * @return {@code true} if the update succeeds or pendingAcks is null; + * {@code false} otherwise + */ + public boolean updateRemainingUnacked(long ledgerId, long entryId, int ackedDelta) { + if (pendingAcks != null) { + return pendingAcks.updateRemainingUnacked(ledgerId, entryId, ackedDelta); + } + return true; + } + + /** + * Atomically remove the pending ack entry and return its stored values. + * + *

No-op if {@code pendingAcks} is not initialized. + * + * @return the removed {@link IntIntPair#leftInt() remainingUnacked} and + * {@link IntIntPair#rightInt() stickyKeyHash}, or {@code null} if not found + */ + public IntIntPair removePendingAckAndGet(long ledgerId, long entryId) { + if (pendingAcks != null) { + return pendingAcks.removeAndGet(ledgerId, entryId); + } + return null; + } + /** * Remove all pending acks up to the given mark-delete position and decrement the consumer's unacked message * counter by the remaining unacked count for each removed entry. @@ -1131,9 +1155,8 @@ public void removePendingAcksUpToPositionAndDecrementUnacked(long markDeleteLedg MutableInt mutableTotalUnacked = new MutableInt(0); pendingAcks.removeAllUpTo(markDeleteLedgerId, markDeleteEntryId, - (ledgerId, entryId, batchSize, stickyKeyHash) -> { - mutableTotalUnacked.add((int) getUnAckedCountForBatchIndexLevelEnabled( - PositionFactory.create(ledgerId, entryId), batchSize)); + (ledgerId, entryId, remainingUnacked, stickyKeyHash) -> { + mutableTotalUnacked.add(remainingUnacked); }); int totalUnacked = mutableTotalUnacked.intValue(); if (totalUnacked > 0) { @@ -1155,11 +1178,8 @@ public void redeliverUnacknowledgedMessages(long consumerEpoch) { if (pendingAcks != null) { List pendingPositions = new ArrayList<>((int) pendingAcks.size()); MutableInt totalRedeliveryMessages = new MutableInt(0); - pendingAcks.forEachAndClear((ledgerId, entryId, batchSize, stickyKeyHash) -> { - int unAckedCount = - (int) getUnAckedCountForBatchIndexLevelEnabled(PositionFactory.create(ledgerId, entryId), - batchSize); - totalRedeliveryMessages.add(unAckedCount); + pendingAcks.forEachAndClear((ledgerId, entryId, remainingUnacked, stickyKeyHash) -> { + totalRedeliveryMessages.add(remainingUnacked); pendingPositions.add(PositionFactory.create(ledgerId, entryId)); }); @@ -1188,9 +1208,7 @@ public void redeliverUnacknowledgedMessages(List messageIds) { Position position = PositionFactory.create(msg.getLedgerId(), msg.getEntryId()); IntIntPair pendingAck = pendingAcks.removeAndGet(position.getLedgerId(), position.getEntryId()); if (pendingAck != null) { - int unAckedCount = (int) getUnAckedCountForBatchIndexLevelEnabled(position, pendingAck.leftInt()); - pendingAcks.remove(position.getLedgerId(), position.getEntryId()); - totalRedeliveryMessages += unAckedCount; + totalRedeliveryMessages += pendingAck.leftInt(); pendingPositions.add(position); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java index e19deb34e31b9..9f59d4cd175b9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java @@ -131,6 +131,15 @@ default void cursorIsReset() { //No-op } + /** + * This hook is invoked after cursor mark-delete operations triggered by + * message removal flows such as expiry, skip, or clear backlog, but not for + * regular ack-driven mark-delete operations due to their higher frequency. + * + *

Since the cursor ack set may no longer be available after mark-delete, + * the cleanup logic relies on the remaining unacked count stored in + * {@code PendingAcksMap} entries. + */ default void markDeletePositionMoveForward() { // No-op } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java index 549aef36cec00..a0e48e5a2df5e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PendingAcksMap.java @@ -124,11 +124,12 @@ public interface PendingAcksConsumer { * * @param ledgerId the ledger ID * @param entryId the entry ID - * @param batchSize the batch size + * @param remainingUnacked the number of remaining unacked messages in this entry + * (for batch entries with some indexes already acked, this may be less than batchSize) * @param stickyKeyHash the sticky key hash * @return true if the pending ack was added, and it's allowed to send a message, false otherwise */ - public boolean addPendingAckIfAllowed(long ledgerId, long entryId, int batchSize, int stickyKeyHash) { + public boolean addPendingAckIfAllowed(long ledgerId, long entryId, int remainingUnacked, int stickyKeyHash) { try { writeLock.lock(); // prevent adding sticky hash to pending acks if the PendingAcksMap has already been closed @@ -145,7 +146,7 @@ public boolean addPendingAckIfAllowed(long ledgerId, long entryId, int batchSize } Long2ObjectSortedMap ledgerPendingAcks = pendingAcks.computeIfAbsent(ledgerId, k -> new Long2ObjectRBTreeMap<>()); - ledgerPendingAcks.put(entryId, IntIntPair.of(batchSize, stickyKeyHash)); + ledgerPendingAcks.put(entryId, IntIntPair.of(remainingUnacked, stickyKeyHash)); return true; } finally { writeLock.unlock(); @@ -312,6 +313,34 @@ public boolean remove(long ledgerId, long entryId, int batchSize, int stickyKeyH } } + /** + * Atomically update the remaining unacked count for a pending ack entry by subtracting the given delta. + * Called from the ack handler after computing the number of batch indexes acknowledged in a partial ack. + * + * @param ledgerId the ledger ID + * @param entryId the entry ID + * @param ackedDelta the number of batch indexes that were just acknowledged + * @return true if the entry was found and updated, false otherwise + */ + public boolean updateRemainingUnacked(long ledgerId, long entryId, int ackedDelta) { + try { + writeLock.lock(); + Long2ObjectSortedMap ledgerMap = pendingAcks.get(ledgerId); + if (ledgerMap == null) { + return false; + } + IntIntPair current = ledgerMap.get(entryId); + if (current == null) { + return false; + } + int newRemaining = current.leftInt() - ackedDelta; + ledgerMap.put(entryId, IntIntPair.of(newRemaining, current.rightInt())); + return true; + } finally { + writeLock.unlock(); + } + } + /** * Remove the pending ack for the given ledger ID and entry ID. * diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index cfabc9333adaf..ce56ea3689d76 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -142,7 +142,6 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis protected enum ReadType { Normal, Replay } - private Position lastMarkDeletePositionBeforeReadMoreEntries; private volatile long readMoreEntriesCallCount; public PersistentDispatcherMultipleConsumers(PersistentTopic topic, ManagedCursor cursor, @@ -358,17 +357,6 @@ public synchronized void readMoreEntries() { // increment the counter for readMoreEntries calls, to track the number of times readMoreEntries is called readMoreEntriesCallCount++; - // remove possible expired messages from redelivery tracker and pending acks - Position markDeletePosition = cursor.getMarkDeletedPosition(); - if (lastMarkDeletePositionBeforeReadMoreEntries != markDeletePosition) { - redeliveryMessages.removeAllUpTo(markDeletePosition.getLedgerId(), markDeletePosition.getEntryId()); - for (Consumer consumer : consumerList) { - consumer.removePendingAcksUpToPositionAndDecrementUnacked( - markDeletePosition.getLedgerId(), markDeletePosition.getEntryId()); - } - lastMarkDeletePositionBeforeReadMoreEntries = markDeletePosition; - } - // totalAvailablePermits may be updated by other threads int firstAvailableConsumerPermits = getFirstAvailableConsumerPermits(); int currentTotalAvailablePermits = Math.max(totalAvailablePermits, firstAvailableConsumerPermits); @@ -605,6 +593,18 @@ public CopyOnWriteArrayList getConsumers() { return consumerList; } + @Override + public void markDeletePositionMoveForward() { + Position markDeletePosition = cursor.getMarkDeletedPosition(); + if (markDeletePosition != null) { + redeliveryMessages.removeAllUpTo(markDeletePosition.getLedgerId(), markDeletePosition.getEntryId()); + for (Consumer consumer : consumerList) { + consumer.removePendingAcksUpToPositionAndDecrementUnacked( + markDeletePosition.getLedgerId(), markDeletePosition.getEntryId()); + } + } + } + @Override public synchronized boolean canUnsubscribe(Consumer consumer) { return consumerList.size() == 1 && consumerSet.contains(consumer); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index 929261783dfff..0ad8e8cfe7867 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -37,9 +37,9 @@ import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.proto.ManagedLedgerInfo; +import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.client.impl.MessageImpl; -import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.stats.Rate; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; @@ -210,9 +210,11 @@ public void markDeleteComplete(Object ctx) { long numMessagesExpired = (long) ctx - cursor.getNumberOfEntriesInBacklog(false); msgExpired.recordMultipleEvents(numMessagesExpired, 0 /* no value stats */); totalMsgExpired.add(numMessagesExpired); - // If the subscription is a Key_Shared subscription, we should to trigger message dispatch. - if (subscription != null && subscription.getType() == SubType.Key_Shared) { - subscription.getDispatcher().markDeletePositionMoveForward(); + if (subscription != null) { + Dispatcher dispatcher = subscription.getDispatcher(); + if (dispatcher != null) { + dispatcher.markDeletePositionMoveForward(); + } } expirationCheckInProgress = FALSE; if (log.isDebugEnabled()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index c2afc35c619e9..86c4329521b3b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -622,6 +622,7 @@ protected int getStickyKeyHash(Entry entry) { @Override public void markDeletePositionMoveForward() { + super.markDeletePositionMoveForward(); // reschedule a read with a backoff after moving the mark-delete position forward since there might have // been consumers that were blocked by hash and couldn't make progress reScheduleReadWithKeySharedUnblockingInterval(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java index 1f29cb7362685..748b57af5c204 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java @@ -481,11 +481,13 @@ private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List { synchronized (PersistentStickyKeyDispatcherMultipleConsumersClassic.this) { + super.markDeletePositionMoveForward(); if (recentlyJoinedConsumers != null && !recentlyJoinedConsumers.isEmpty() && removeConsumersFromRecentJoinedConsumers()) { // After we process acks, we need to check whether the mark-delete position was advanced and we diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index a5cd832e99f2c..ce7795a917adc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -553,10 +553,6 @@ private void notifyTheMarkDeletePositionChanged(Position oldPosition) { if (newMD.compareTo(oldPosition) != 0) { updateLastMarkDeleteAdvancedTimestamp(); handleReplicatedSubscriptionsUpdate(newMD); - - if (dispatcher != null) { - dispatcher.markDeletePositionMoveForward(); - } } } @@ -778,6 +774,7 @@ public void clearBacklogComplete(Object ctx) { future.complete(null); } }); + dispatcher.markDeletePositionMoveForward(); dispatcher.afterAckMessages(null, ctx); } else { future.complete(null); @@ -815,6 +812,7 @@ public void skipEntriesComplete(Object ctx) { } future.complete(null); if (dispatcher != null) { + dispatcher.markDeletePositionMoveForward(); dispatcher.afterAckMessages(null, ctx); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 6741cb5c7868e..f1bbe6415dbaf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -1518,14 +1518,6 @@ public void testAckWithTransactionReduceUnackCountNotInPendingAcks() throws Exce messageIds.add(consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS).getMessageId()); } - MessageIdImpl messageId = (MessageIdImpl) messageIds.get(0); - - - // remove the message from the pendingAcks, in fact redeliver will remove the messageId from the pendingAck - getPulsarServiceList().get(0).getBrokerService().getTopic(topic, false) - .get().get().getSubscription(subName).getConsumers().get(0).getPendingAcks() - .remove(messageId.ledgerId, messageId.entryId); - Transaction txn = getTxn(); consumer.acknowledgeAsync(messageIds.get(1), txn).get(); From 2537ffc87440d344092d167f65d2c4602224dfcc Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 12 May 2026 19:05:41 +0800 Subject: [PATCH 069/201] [fix][broker] Close pending acks cleanup gap in BacklogQuotaManager (#25624) --- .../broker/service/BacklogQuotaManager.java | 68 +++++++-- .../service/BacklogQuotaManagerTest.java | 129 ++++++++++++++++++ 2 files changed, 188 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java index 47ff3588ab258..b5a0d5877fed3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import static java.util.concurrent.TimeUnit.SECONDS; +import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -33,12 +34,14 @@ import org.apache.bookkeeper.mledger.proto.ManagedLedgerInfo; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.resources.NamespaceResources; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.persistent.PersistentTopicMetrics.BacklogQuotaMetrics; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; import org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl; +import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -137,9 +140,7 @@ public void handleExceededBacklogQuota(PersistentTopic persistentTopic, BacklogQ * Backlog quota set for the topic */ private void dropBacklogForSizeLimit(PersistentTopic persistentTopic, BacklogQuota quota) { - // Set the reduction factor to 90%. The aim is to drop down the backlog to 90% of the quota limit. - double reductionFactor = 0.9; - double targetSize = reductionFactor * quota.getLimitSize(); + long targetSize = computeEvictionTarget(quota.getLimitSize()); // Get estimated unconsumed size for the managed ledger associated with this topic. Estimated size is more // useful than the actual storage size. Actual storage size gets updated only when managed ledger is trimmed. @@ -148,7 +149,7 @@ private void dropBacklogForSizeLimit(PersistentTopic persistentTopic, BacklogQuo if (log.isDebugEnabled()) { log.debug("[{}] target size is [{}] for quota limit [{}], backlog size is [{}]", persistentTopic.getName(), - targetSize, targetSize / reductionFactor, backlogSize); + targetSize, quota.getLimitSize(), backlogSize); } ManagedCursor previousSlowestConsumer = null; while (backlogSize > targetSize) { @@ -165,13 +166,17 @@ private void dropBacklogForSizeLimit(PersistentTopic persistentTopic, BacklogQuo if (slowestConsumer == previousSlowestConsumer) { log.info("[{}] Cursors not progressing, target size is [{}] for quota limit [{}], backlog size is [{}]", - persistentTopic.getName(), targetSize, targetSize / reductionFactor, backlogSize); + persistentTopic.getName(), targetSize, quota.getLimitSize(), backlogSize); break; } // Calculate number of messages to be skipped using the current backlog and the skip factor. long entriesInBacklog = slowestConsumer.getNumberOfEntriesInBacklog(false); - int messagesToSkip = (int) (messageSkipFactor * entriesInBacklog); + + int messagesToSkip = computeEntriesToEvict( + backlogSize, + quota.getLimitSize(), + entriesInBacklog); try { // If there are no messages to skip, break out of the loop if (messagesToSkip == 0) { @@ -186,6 +191,7 @@ private void dropBacklogForSizeLimit(PersistentTopic persistentTopic, BacklogQuo persistentTopic.getName(), messagesToSkip, slowestConsumer.getName(), entriesInBacklog); } slowestConsumer.skipEntries(messagesToSkip, IndividualDeletedEntries.Include); + markDeletePositionMoveForward(persistentTopic, slowestConsumer); } catch (Exception e) { log.error("[{}] Error skipping [{}] messages from slowest consumer [{}]", persistentTopic.getName(), messagesToSkip, slowestConsumer.getName(), e); @@ -213,9 +219,7 @@ private void dropBacklogForTimeLimit(PersistentTopic persistentTopic, BacklogQuo boolean preciseTimeBasedBacklogQuotaCheck) { // If enabled precise time based backlog quota check, will expire message based on the timeBaseQuota if (preciseTimeBasedBacklogQuotaCheck) { - // Set the reduction factor to 90%. The aim is to drop down the backlog to 90% of the quota limit. - double reductionFactor = 0.9; - int target = (int) (reductionFactor * quota.getLimitTime()); + int target = (int) computeEvictionTarget(quota.getLimitTime()); if (log.isDebugEnabled()) { log.debug("[{}] target backlog expire time is [{}]", persistentTopic.getName(), target); } @@ -244,6 +248,7 @@ private void dropBacklogForTimeLimit(PersistentTopic persistentTopic, BacklogQuo long ledgerId = mLedger.getLedgersInfo().ceilingKey(oldestPosition.getLedgerId() + 1); Position nextPosition = PositionFactory.create(ledgerId, -1); slowestConsumer.markDelete(nextPosition); + markDeletePositionMoveForward(persistentTopic, slowestConsumer); continue; } // Timestamp only > 0 if ledger has been closed @@ -254,6 +259,7 @@ private void dropBacklogForTimeLimit(PersistentTopic persistentTopic, BacklogQuo Position nextPosition = PositionFactory.create(ledgerId, -1); if (!nextPosition.equals(oldestPosition)) { slowestConsumer.markDelete(nextPosition); + markDeletePositionMoveForward(persistentTopic, slowestConsumer); continue; } } @@ -314,4 +320,48 @@ private boolean advanceSlowestSystemCursor(PersistentTopic persistentTopic) { // We may need to check other system cursors here : replicator, compaction return false; } + + /** + * Invoke {@link Dispatcher#markDeletePositionMoveForward()} for the subscription that owns the given cursor. + * This ensures pending acks and redelivery state are cleaned up when the cursor is advanced by + * backlog quota eviction (bypassing the subscription-level wrappers that normally fire this hook). + * + * @param persistentTopic the topic + * @param cursor the cursor that was advanced + */ + private void markDeletePositionMoveForward(PersistentTopic persistentTopic, ManagedCursor cursor) { + PersistentSubscription subscription = + persistentTopic.getSubscriptions().get(Codec.decode(cursor.getName())); + if (subscription != null && subscription.getDispatcher() != null) { + subscription.getDispatcher().markDeletePositionMoveForward(); + } + } + + + /** + * Compute the target value after backlog eviction. + * + * @param quotaLimit configured quota limit + * @return target value after eviction + */ + private static long computeEvictionTarget(long quotaLimit) { + double factor = 0.9; + return (long) (factor * quotaLimit); + } + + /** + * Compute the number of entries to evict in a single eviction iteration. + * + * @param currentValue current backlog value + * @param quotaLimit configured quota limit + * @param totalEntries total entries in backlog + * @return entries to evict + */ + @VisibleForTesting + static int computeEntriesToEvict( + long currentValue, long quotaLimit, long totalEntries) { + long evictionTarget = computeEvictionTarget(quotaLimit); + return (int) ((currentValue - evictionTarget) + * (double) totalEntries / currentValue); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java index 13562fafdf830..a693364ec8fa7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -21,6 +21,7 @@ import static java.util.Map.entry; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.pulsar.broker.service.BacklogQuotaManager.computeEntriesToEvict; import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongGaugeValue; import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; import static org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType.destination_storage; @@ -55,6 +56,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil; import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; @@ -71,6 +73,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.ClusterData; @@ -2174,4 +2177,130 @@ public void testBacklogQuotaInGB(boolean backlogQuotaSizeGB) throws Exception { assertTrue(stats.getBacklogSize() < 10 * 1024, "Storage size is [" + stats.getStorageSize() + "]"); } private static final Logger LOG = LoggerFactory.getLogger(BacklogQuotaManagerTest.class); + + private void assertPendingAcks(org.apache.pulsar.broker.service.Consumer consumer, int expected) { + PendingAcksMap pendingAcks = consumer.getPendingAcks(); + assertThat(pendingAcks).isNotNull(); + assertThat(pendingAcks.size()).isEqualTo(expected); + assertThat(consumer.getUnackedMessages()).isEqualTo(expected); + } + + @Test + public void testConsumerBacklogEvictionSizeQuotaCleansPendingAcks() throws Exception { + final int msgSize = 1024; + final int quotaSizeLimit = 10 * 1024; + final int numMsgs = 20; + + admin.namespaces().setBacklogQuota("prop/ns-quota", + BacklogQuota.builder() + .limitSize(quotaSizeLimit) + .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction) + .build()); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(adminUrl.toString()) + .build(); + + final String topic = + BrokerTestUtil.newUniqueName("persistent://prop/ns-quota/topic-pending-acks-size"); + final String subName = "key-shared-sub"; + + @Cleanup + Consumer consumer = client.newConsumer() + .topic(topic) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + + @Cleanup + Producer producer = createProducer(client, topic); + + byte[] content = new byte[msgSize]; + for (int i = 0; i < numMsgs; i++) { + producer.send(content); + } + + // Receive all messages but don't ack — pending acks accumulate. + for (int i = 0; i < numMsgs; i++) { + consumer.receive(); + } + + PersistentTopic topicRef = + (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + PersistentSubscription sub = topicRef.getSubscription(subName); + + org.apache.pulsar.broker.service.Consumer brokerConsumer = sub.getDispatcher().getConsumers().get(0); + assertThat(sub).isNotNull(); + assertPendingAcks(brokerConsumer, numMsgs); + + int expectedRemaining = numMsgs - computeEntriesToEvict( + (long) numMsgs * msgSize, + quotaSizeLimit, + numMsgs); + + Awaitility.await() + .pollDelay(TIME_TO_CHECK_BACKLOG_QUOTA + 1, SECONDS) + .pollInterval(1, SECONDS) + .untilAsserted(() -> assertPendingAcks(brokerConsumer, expectedRemaining)); + } + + @Test + public void testConsumerBacklogEvictionTimeQuotaNotPreciseCleansPendingAcks() + throws Exception { + admin.namespaces().setBacklogQuota("prop/ns-quota", + BacklogQuota.builder() + .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) + .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction) + .build(), message_age); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(adminUrl.toString()) + .build(); + + final String topic = + BrokerTestUtil.newUniqueName("persistent://prop/ns-quota/topic-pending-acks-time"); + final String subName = "key-shared-sub-time"; + final int numMsgs = 14; + + @Cleanup + Consumer consumer = client.newConsumer() + .topic(topic) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + + @Cleanup + Producer producer = createProducer(client, topic); + + byte[] content = new byte[1024]; + for (int i = 0; i < numMsgs; i++) { + producer.send(content); + } + + // Receive all messages but don't ack — pending acks accumulate. + for (int i = 0; i < numMsgs; i++) { + consumer.receive(); + } + + PersistentTopic topicRef = + (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + PersistentSubscription sub = topicRef.getSubscription(subName); + + org.apache.pulsar.broker.service.Consumer brokerConsumer = sub.getDispatcher().getConsumers().get(0); + assertThat(sub).isNotNull(); + + assertPendingAcks(brokerConsumer, numMsgs); + + // Non-precise eviction removes whole closed ledgers only. + // With MAX_ENTRIES_PER_LEDGER=5 and 14 entries: + // ledgers are [5, 5, 4]. The last ledger remains open and is not evicted. + int expectedRemaining = numMsgs % MAX_ENTRIES_PER_LEDGER; + + Awaitility.await() + .pollDelay(TIME_TO_CHECK_BACKLOG_QUOTA * 2, SECONDS) + .pollInterval(1, SECONDS) + .untilAsserted(() -> assertPendingAcks(brokerConsumer, expectedRemaining)); + } } From 34354be6a6856b6bd3961f849eefd5d7b40844fe Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 12 May 2026 19:31:40 +0800 Subject: [PATCH 070/201] [improve][broker] optimize namespaceBundle validation to fix single-thread 100% CPU during unloading entire namespaces (#25626) --- .../broker/admin/impl/NamespacesBase.java | 36 ++---- .../broker/admin/impl/ResourceQuotasBase.java | 3 +- .../broker/admin/v2/NonPersistentTopics.java | 5 +- .../pulsar/broker/web/PulsarWebResource.java | 85 ++++++------- .../pulsar/broker/admin/NamespacesTest.java | 120 ++++++++++++++++++ 5 files changed, 174 insertions(+), 75 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 0b8a0591ae12b..1e99aa0716ba5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -596,8 +596,7 @@ protected CompletableFuture internalDeleteNamespaceBundleAsync(String bund } return future .thenCompose(__ -> - validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, - bundleRange, + validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange, authoritative, true)) .thenCompose(bundle -> { return pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName) @@ -1459,9 +1458,8 @@ public CompletableFuture internalUnloadNamespaceBundleAsync(String bundleR } }) .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) - .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) - .thenCompose(policies -> - isBundleOwnedByAnyBroker(namespaceName, policies.bundles, bundleRange) + .thenCompose(__ -> + isBundleOwnedByAnyBroker(namespaceName, bundleRange) .thenCompose(flag -> { if (!flag) { log.info("[{}] Namespace bundle is not owned by any broker {}/{}", clientAppId(), @@ -1469,7 +1467,7 @@ public CompletableFuture internalUnloadNamespaceBundleAsync(String bundleR return CompletableFuture.completedFuture(null); } Optional destinationBrokerOpt = Optional.ofNullable(destinationBroker); - return validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, + return validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange, authoritative, true) .thenCompose(nsBundle -> pulsar().getNamespaceService() .unloadNamespaceBundle(nsBundle, destinationBrokerOpt)); @@ -1509,10 +1507,8 @@ protected CompletableFuture internalSplitNamespaceBundleAsync(String bundl .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) .thenCompose(__ -> getBundleRangeAsync(bundleName)) .thenCompose(bundleRange -> { - return getNamespacePoliciesAsync(namespaceName) - .thenCompose(policies -> - validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, - authoritative, false)) + return validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange, + authoritative, false) .thenCompose(nsBundle -> pulsar().getNamespaceService().splitAndOwnBundle(nsBundle, unload, pulsar().getNamespaceService() .getNamespaceBundleSplitAlgorithmByName(splitAlgorithmName), @@ -1526,9 +1522,8 @@ protected CompletableFuture internalGetTopicHashPositionsAsy log.debug("[{}] Getting hash position for topic list {}, bundle {}", clientAppId(), topics, bundleRange); } return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.PERSISTENCE, PolicyOperation.READ) - .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) - .thenCompose(policies -> { - return validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, + .thenCompose(__ -> { + return validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange, false, true) .thenCompose(nsBundle -> pulsar().getNamespaceService().getOwnedTopicListForNamespaceBundle(nsBundle)) @@ -1859,11 +1854,10 @@ protected CompletableFuture internalClearNamespaceBundleBacklogAsync(Strin // check cluster ownership for a given global namespace: redirect if peer-cluster owns it return validateGlobalNamespaceOwnershipAsync(namespaceName); }) - .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) - .thenCompose(policies -> + .thenCompose(__ -> // Allow acquiring ownership for an unassigned bundle so backlog can be cleared // even if not loaded. - validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, + validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange, authoritative, false)) .thenCompose(bundle -> clearBacklogAsync(bundle, null)) .thenRun(() -> log.info("[{}] Successfully cleared backlog on namespace bundle {}/{}", clientAppId(), @@ -1912,11 +1906,10 @@ protected CompletableFuture internalClearNamespaceBundleBacklogForSubscrip // check cluster ownership for a given global namespace: redirect if peer-cluster owns it return validateGlobalNamespaceOwnershipAsync(namespaceName); }) - .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) - .thenCompose(policies -> + .thenCompose(__ -> // Allow acquiring ownership for an unassigned bundle so backlog can be cleared // even if not loaded. - validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, + validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange, authoritative, false)) .thenCompose(bundle -> clearBacklogAsync(bundle, subscription)) .thenRun(() -> log.info( @@ -1960,9 +1953,8 @@ protected CompletableFuture internalUnsubscribeNamespaceBundleAsync(String return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.UNSUBSCRIBE) .thenCompose(__ -> validateGlobalNamespaceOwnershipAsync(namespaceName)) - .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) - .thenCompose(policies -> - validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, + .thenCompose(__ -> + validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange, authoritative, false)) .thenCompose(bundle -> unsubscribeAsync(bundle, subscription)) .thenRun(() -> log.info("[{}] Successfully unsubscribed {} on namespace bundle {}/{}", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceQuotasBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceQuotasBase.java index a1a3ce6558392..8e1956e6d0d71 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceQuotasBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceQuotasBase.java @@ -71,7 +71,6 @@ private CompletableFuture getNamespaceBundleRangeAsync(String b } }); return ret - .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) - .thenApply(policies -> validateNamespaceBundleRange(namespaceName, policies.bundles, bundleRange)); + .thenCompose(__ -> validateNamespaceBundleRangeAsync(namespaceName, bundleRange)); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java index 16cbc7104d4d8..cf06b84c365c4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java @@ -463,18 +463,17 @@ public void getListFromBundle( } validateNamespaceOperation(namespaceName, NamespaceOperation.GET_BUNDLE); - Policies policies = getNamespacePolicies(namespaceName); // check cluster ownership for a given global namespace: redirect if peer-cluster owns it validateGlobalNamespaceOwnership(namespaceName); - isBundleOwnedByAnyBroker(namespaceName, policies.bundles, bundleRange).thenAccept(flag -> { + isBundleOwnedByAnyBroker(namespaceName, bundleRange).thenAccept(flag -> { if (!flag) { log.info("[{}] Namespace bundle is not owned by any broker {}/{}", clientAppId(), namespaceName, bundleRange); asyncResponse.resume(Response.noContent().build()); } else { - validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, true, true) + validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange, true, true) .thenAccept(nsBundle -> { final var bundleTopics = pulsar().getBrokerService().getMultiLayerTopicsMap() .get(namespaceName.toString()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 3fe51df90f72e..41c1e709aee25 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -79,7 +79,6 @@ import org.apache.pulsar.common.naming.NamespaceBundles; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ClusterDataImpl; import org.apache.pulsar.common.policies.data.NamespaceOperation; @@ -563,7 +562,7 @@ static boolean isValidCluster(PulsarService pulsarService, String cluster) { return !pulsarService.getConfiguration().isAuthorizationEnabled(); } - protected NamespaceBundle validateNamespaceBundleRange(NamespaceName fqnn, BundlesData bundles, + protected CompletableFuture validateNamespaceBundleRangeAsync(NamespaceName fqnn, String bundleRange) { try { checkArgument(bundleRange.contains("_"), "Invalid bundle range: " + bundleRange); @@ -574,68 +573,58 @@ protected NamespaceBundle validateNamespaceBundleRange(NamespaceName fqnn, Bundl (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN); NamespaceBundle nsBundle = pulsar().getNamespaceService().getNamespaceBundleFactory().getBundle(fqnn, hashRange); - NamespaceBundles nsBundles = pulsar().getNamespaceService().getNamespaceBundleFactory().getBundles(fqnn, - bundles); - nsBundles.validateBundle(nsBundle); - return nsBundle; + return pulsar().getNamespaceService().getNamespaceBundleFactory().getBundlesAsync(fqnn) + .thenApply(nsBundles -> { + try { + nsBundles.validateBundle(nsBundle); + return nsBundle; + } catch (IllegalArgumentException e) { + log.error("Invalid bundle range, namespace={}, bundleRange={}", + fqnn.toString(), bundleRange, e); + throw new RestException(Response.Status.PRECONDITION_FAILED, e.getMessage()); + } catch (Exception e) { + log.error("Failed to validate namespace bundle, namespace={}, bundleRange={}", + fqnn.toString(), bundleRange, e); + throw new RestException(e); + } + }); } catch (IllegalArgumentException e) { log.error("[{}] Invalid bundle range {}/{}, {}", clientAppId(), fqnn.toString(), bundleRange, e.getMessage()); - throw new RestException(Response.Status.PRECONDITION_FAILED, e.getMessage()); + return CompletableFuture.failedFuture( + new RestException(Response.Status.PRECONDITION_FAILED, e.getMessage())); } catch (Exception e) { log.error("[{}] Failed to validate namespace bundle {}/{}", clientAppId(), fqnn.toString(), bundleRange, e); - throw new RestException(e); + return CompletableFuture.failedFuture(new RestException(e)); } } /** * Checks whether a given bundle is currently loaded by any broker. */ - protected CompletableFuture isBundleOwnedByAnyBroker(NamespaceName fqnn, BundlesData bundles, - String bundleRange) { - NamespaceBundle nsBundle = validateNamespaceBundleRange(fqnn, bundles, bundleRange); - NamespaceService nsService = pulsar().getNamespaceService(); - - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { - return nsService.checkOwnershipPresentAsync(nsBundle); - } - - LookupOptions options = LookupOptions.builder() - .authoritative(false) - .requestHttps(isRequestHttps()) - .readOnly(true) - .loadTopicsInBundle(false).build(); - - return nsService.getWebServiceUrlAsync(nsBundle, options).thenApply(Optional::isPresent); - } - - protected NamespaceBundle validateNamespaceBundleOwnership(NamespaceName fqnn, BundlesData bundles, - String bundleRange, boolean authoritative, boolean readOnly) { - try { - NamespaceBundle nsBundle = validateNamespaceBundleRange(fqnn, bundles, bundleRange); - validateBundleOwnership(nsBundle, authoritative, readOnly); - return nsBundle; - } catch (WebApplicationException wae) { - throw wae; - } catch (Exception e) { - log.error("[{}] Failed to validate namespace bundle {}/{}", clientAppId(), - fqnn.toString(), bundleRange, e); - throw new RestException(e); - } + protected CompletableFuture isBundleOwnedByAnyBroker(NamespaceName fqnn, String bundleRange) { + return validateNamespaceBundleRangeAsync(fqnn, bundleRange) + .thenCompose(nsBundle -> { + NamespaceService nsService = pulsar().getNamespaceService(); + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { + return nsService.checkOwnershipPresentAsync(nsBundle); + } + LookupOptions options = LookupOptions.builder() + .authoritative(false) + .requestHttps(isRequestHttps()) + .readOnly(true) + .loadTopicsInBundle(false).build(); + return nsService.getWebServiceUrlAsync(nsBundle, options).thenApply(Optional::isPresent); + }); } protected CompletableFuture validateNamespaceBundleOwnershipAsync( - NamespaceName fqnn, BundlesData bundles, String bundleRange, + NamespaceName fqnn, String bundleRange, boolean authoritative, boolean readOnly) { - NamespaceBundle nsBundle; - try { - nsBundle = validateNamespaceBundleRange(fqnn, bundles, bundleRange); - } catch (WebApplicationException wae) { - return CompletableFuture.failedFuture(wae); - } - return validateBundleOwnershipAsync(nsBundle, authoritative, readOnly) - .thenApply(__ -> nsBundle); + return validateNamespaceBundleRangeAsync(fqnn, bundleRange) + .thenCompose(nsBundle -> validateBundleOwnershipAsync(nsBundle, authoritative, readOnly) + .thenApply(__ -> nsBundle)); } public void validateBundleOwnership(NamespaceBundle bundle, boolean authoritative, boolean readOnly) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java index c40e8a84c5efc..6ff2d8a8216b2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java @@ -2559,4 +2559,124 @@ public void testGetClusterAntiAffinityNamespaces() throws Exception { assertEquals(namespacesResp, namespacesWithFullPath); } + @Test + public void testBundleValidationWithNonExistentNamespace() throws Exception { + String nonExistentNs = "non-existent-namespace"; + String bundleRange = "0x00000000_0x80000000"; + + // Test unload on non-existent namespace - should return 404 + // The error is thrown by validateGlobalNamespaceOwnershipAsync before reaching bundle validation + AsyncResponse unloadResponse = mock(AsyncResponse.class); + namespaces.unloadNamespaceBundle(unloadResponse, testTenant, nonExistentNs, + bundleRange, false, null); + ArgumentCaptor unloadCaptor = ArgumentCaptor.forClass(RestException.class); + verify(unloadResponse, timeout(5000).times(1)).resume(unloadCaptor.capture()); + assertEquals(unloadCaptor.getValue().getResponse().getStatus(), + Response.Status.NOT_FOUND.getStatusCode(), + "Non-existent namespace should return 404"); + + // Test split on non-existent namespace - should return 404 + AsyncResponse splitResponse = mock(AsyncResponse.class); + namespaces.splitNamespaceBundle(splitResponse, testTenant, nonExistentNs, + bundleRange, false, true, null, null); + ArgumentCaptor splitCaptor = ArgumentCaptor.forClass(RestException.class); + verify(splitResponse, timeout(5000).times(1)).resume(splitCaptor.capture()); + assertEquals(splitCaptor.getValue().getResponse().getStatus(), + Response.Status.NOT_FOUND.getStatusCode(), + "Non-existent namespace should return 404"); + + // Test clear backlog on non-existent namespace - should return 404 + AsyncResponse clearResponse = mock(AsyncResponse.class); + namespaces.clearNamespaceBundleBacklog(clearResponse, testTenant, nonExistentNs, + bundleRange, false); + ArgumentCaptor clearCaptor = ArgumentCaptor.forClass(RestException.class); + verify(clearResponse, timeout(5000).times(1)).resume(clearCaptor.capture()); + assertEquals(clearCaptor.getValue().getResponse().getStatus(), + Response.Status.NOT_FOUND.getStatusCode(), + "Non-existent namespace should return 404"); + + // Test clear backlog for subscription on non-existent namespace - should return 404 + AsyncResponse clearSubResponse = mock(AsyncResponse.class); + namespaces.clearNamespaceBundleBacklogForSubscription(clearSubResponse, testTenant, nonExistentNs, + bundleRange, "test-sub", false); + ArgumentCaptor clearSubCaptor = ArgumentCaptor.forClass(RestException.class); + verify(clearSubResponse, timeout(5000).times(1)).resume(clearSubCaptor.capture()); + assertEquals(clearSubCaptor.getValue().getResponse().getStatus(), + Response.Status.NOT_FOUND.getStatusCode(), + "Non-existent namespace should return 404"); + } + + @Test + public void testBundleValidationAfterSplit() throws Exception { + URL localWebServiceUrl = new URL(pulsar.getSafeWebServiceAddress()); + String bundledNsLocal = "test-bundle-validation-after-split"; + List boundaries = List.of("0x00000000", "0xffffffff"); + BundlesData bundleData = BundlesData.builder() + .boundaries(boundaries) + .numBundles(boundaries.size() - 1) + .build(); + createBundledTestNamespaces(this.testTenant, bundledNsLocal, bundleData); + final NamespaceName testNs = NamespaceName.get(this.testTenant, bundledNsLocal); + + OwnershipCache mockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache()); + doReturn(CompletableFuture.completedFuture(null)).when(mockOwnershipCache) + .disableOwnership(any(NamespaceBundle.class)); + Field ownership = NamespaceService.class.getDeclaredField("ownershipCache"); + ownership.setAccessible(true); + ownership.set(pulsar.getNamespaceService(), mockOwnershipCache); + mockWebUrl(localWebServiceUrl, testNs); + + // Split the bundle + AsyncResponse splitResponse = mock(AsyncResponse.class); + namespaces.splitNamespaceBundle(splitResponse, testTenant, bundledNsLocal, + "0x00000000_0xffffffff", + false, true, null, null); + ArgumentCaptor splitCaptor = ArgumentCaptor.forClass(Response.class); + verify(splitResponse, timeout(5000).times(1)).resume(splitCaptor.capture()); + + // Verify split was successful + BundlesData bundlesDataAfterSplit = (BundlesData) asyncRequests(ctx -> namespaces.getBundlesData(ctx, + testTenant, bundledNsLocal)); + assertNotNull(bundlesDataAfterSplit); + assertEquals(bundlesDataAfterSplit.getBoundaries().size(), 3); + assertEquals(bundlesDataAfterSplit.getBoundaries().get(0), "0x00000000"); + assertEquals(bundlesDataAfterSplit.getBoundaries().get(1), "0x7fffffff"); + assertEquals(bundlesDataAfterSplit.getBoundaries().get(2), "0xffffffff"); + + // Now test bundle validation with the old (invalid) bundle range - should return 412 + AsyncResponse unloadOldBundleResponse = mock(AsyncResponse.class); + namespaces.unloadNamespaceBundle(unloadOldBundleResponse, testTenant, bundledNsLocal, + "0x00000000_0xffffffff", false, null); + ArgumentCaptor unloadOldCaptor = ArgumentCaptor.forClass(RestException.class); + verify(unloadOldBundleResponse, timeout(5000).times(1)).resume(unloadOldCaptor.capture()); + assertEquals(unloadOldCaptor.getValue().getResponse().getStatus(), + Response.Status.PRECONDITION_FAILED.getStatusCode(), + "Old bundle range after split should return 412"); + + // Test bundle validation with new valid bundle ranges - should succeed + doReturn(true).when(nsSvc) + .isServiceUnitOwned(Mockito.argThat(bundle -> bundle.getNamespaceObject().equals(testNs))); + doReturn(CompletableFuture.completedFuture(null)).when(nsSvc) + .unloadNamespaceBundle(any(NamespaceBundle.class)); + + AsyncResponse unloadNewBundle1Response = mock(AsyncResponse.class); + namespaces.unloadNamespaceBundle(unloadNewBundle1Response, testTenant, bundledNsLocal, + "0x00000000_0x7fffffff", false, null); + ArgumentCaptor newBundle1Captor = ArgumentCaptor.forClass(Response.class); + verify(unloadNewBundle1Response, timeout(5000).times(1)).resume(newBundle1Captor.capture()); + assertEquals(newBundle1Captor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode(), + "New bundle range should be valid"); + + AsyncResponse unloadNewBundle2Response = mock(AsyncResponse.class); + namespaces.unloadNamespaceBundle(unloadNewBundle2Response, testTenant, bundledNsLocal, + "0x7fffffff_0xffffffff", false, null); + ArgumentCaptor newBundle2Captor = ArgumentCaptor.forClass(Response.class); + verify(unloadNewBundle2Response, timeout(5000).times(1)).resume(newBundle2Captor.capture()); + assertEquals(newBundle2Captor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode(), + "New bundle range should be valid"); + + // cleanup + resetBroker(); + } + } From c0957c805916de1ab8b7599c76b5c8d4a3b13cc6 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Tue, 12 May 2026 10:47:16 +0800 Subject: [PATCH 071/201] [fix][broker] Merge broker offload extra configurations (#25736) --- .../policies/data/OffloadPoliciesImpl.java | 24 ++++++--- .../policies/data/OffloadPoliciesTest.java | 49 +++++++++++++++++++ 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java index d42824081765a..93f090c9ddf83 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java @@ -265,13 +265,10 @@ public static OffloadPoliciesImpl create(Properties properties) { } } - Map extraConfigurations = properties.entrySet().stream() - .filter(entry -> entry.getKey().toString().startsWith(EXTRA_CONFIG_PREFIX)) - .collect(Collectors.toMap( - entry -> entry.getKey().toString().replaceFirst(EXTRA_CONFIG_PREFIX, ""), - entry -> entry.getValue().toString())); - - data.getManagedLedgerExtraConfigurations().putAll(extraConfigurations); + Map extraConfigurations = getExtraConfigurations(properties); + if (extraConfigurations != null) { + data.getManagedLedgerExtraConfigurations().putAll(extraConfigurations); + } data.compatibleWithBrokerConfigFile(properties); return data; @@ -469,7 +466,9 @@ public static OffloadPoliciesImpl mergeConfiguration(OffloadPoliciesImpl topicLe */ private static Object getCompatibleValue(Properties properties, Field field) { Object object; - if (field.getName().equals("managedLedgerOffloadThresholdInBytes")) { + if (field.getName().equals("managedLedgerExtraConfigurations")) { + return getExtraConfigurations(properties); + } else if (field.getName().equals("managedLedgerOffloadThresholdInBytes")) { object = properties.getProperty("managedLedgerOffloadThresholdInBytes", properties.getProperty(OFFLOAD_THRESHOLD_NAME_IN_CONF_FILE)); } else if (field.getName().equals("managedLedgerOffloadDeletionLagInMillis")) { @@ -484,6 +483,15 @@ private static Object getCompatibleValue(Properties properties, Field field) { return value((String) object, field); } + private static Map getExtraConfigurations(Properties properties) { + Map extraConfigurations = properties.entrySet().stream() + .filter(entry -> entry.getKey().toString().startsWith(EXTRA_CONFIG_PREFIX)) + .collect(Collectors.toMap( + entry -> entry.getKey().toString().replaceFirst(EXTRA_CONFIG_PREFIX, ""), + entry -> entry.getValue().toString())); + return extraConfigurations.isEmpty() ? null : extraConfigurations; + } + public static class OffloadPoliciesImplBuilder implements OffloadPolicies.Builder { private OffloadPoliciesImpl impl = new OffloadPoliciesImpl(); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java index 70bfa5c377de1..0662be3f05435 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java @@ -333,6 +333,55 @@ public void mergeTest() { Assert.assertNull(offloadPolicies.getS3ManagedLedgerOffloadRegion()); } + @Test + public void brokerExtraConfigMergeTest() { + final String bucketPrefix = "o-123/c-456"; + Properties brokerProperties = new Properties(); + brokerProperties.setProperty("managedLedgerOffloadDriver", "aws-s3"); + brokerProperties.setProperty(EXTRA_CONFIG_PREFIX + "tieredStorageBucketPrefix", bucketPrefix); + + OffloadPoliciesImpl offloadPolicies = + OffloadPoliciesImpl.mergeConfiguration(null, null, brokerProperties); + + Assert.assertNotNull(offloadPolicies); + assertEquals(offloadPolicies.getManagedLedgerExtraConfigurations(), + Map.of("tieredStorageBucketPrefix", bucketPrefix)); + } + + @Test + public void higherLevelExtraConfigOverridesBrokerExtraConfigMergeTest() { + Properties brokerProperties = new Properties(); + brokerProperties.setProperty("managedLedgerOffloadDriver", "aws-s3"); + brokerProperties.setProperty(EXTRA_CONFIG_PREFIX + "tieredStorageBucketPrefix", "broker-prefix"); + + OffloadPoliciesImpl topicLevelPolicies = new OffloadPoliciesImpl(); + topicLevelPolicies.getManagedLedgerExtraConfigurations().put("tieredStorageBucketPrefix", "topic-prefix"); + + OffloadPoliciesImpl offloadPolicies = + OffloadPoliciesImpl.mergeConfiguration(topicLevelPolicies, null, brokerProperties); + + Assert.assertNotNull(offloadPolicies); + assertEquals(offloadPolicies.getManagedLedgerExtraConfigurations(), + Map.of("tieredStorageBucketPrefix", "topic-prefix")); + } + + @Test + public void emptyHigherLevelExtraConfigOverridesBrokerExtraConfigMergeTest() { + Properties brokerProperties = new Properties(); + brokerProperties.setProperty("managedLedgerOffloadDriver", "aws-s3"); + brokerProperties.setProperty(EXTRA_CONFIG_PREFIX + "tieredStorageBucketPrefix", "broker-prefix"); + + OffloadPoliciesImpl topicLevelPolicies = new OffloadPoliciesImpl(); + topicLevelPolicies.getManagedLedgerExtraConfigurations().put("tieredStorageBucketPrefix", ""); + + OffloadPoliciesImpl offloadPolicies = + OffloadPoliciesImpl.mergeConfiguration(topicLevelPolicies, null, brokerProperties); + + Assert.assertNotNull(offloadPolicies); + assertEquals(offloadPolicies.getManagedLedgerExtraConfigurations(), + Map.of("tieredStorageBucketPrefix", "")); + } + @Test public void brokerPropertyCompatibleTest() { From d004ca56e9707a0675da0e36eedfecb6b744089c Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 12 May 2026 20:12:43 +0800 Subject: [PATCH 072/201] [improve][client] Implement tls_client_auth for AuthenticationOAuth2 (#25538) --- .../oauth2/AuthenticationFactoryOAuth2.java | 113 +++++++++++-- .../auth/oauth2/AuthenticationOAuth2.java | 13 +- .../auth/oauth2/ClientCredentialsFlow.java | 14 +- .../client/impl/auth/oauth2/FlowBase.java | 108 +++++++++--- .../impl/auth/oauth2/TlsClientAuthFlow.java | 155 ++++++++++++++++++ .../ClientCredentialsExchangeRequest.java | 3 + .../auth/oauth2/protocol/TokenClient.java | 7 +- .../protocol/TokenEndpointAuthMethod.java | 43 +++++ .../AuthenticationFactoryOAuth2Test.java | 49 +++++- .../auth/oauth2/AuthenticationOAuth2Test.java | 47 +++++- .../auth/oauth2/OAuth2MockHttpClient.java | 49 ++++++ .../auth/oauth2/TlsClientAuthFlowTest.java | 41 +++++ .../auth/oauth2/protocol/TokenClientTest.java | 88 +++++++++- 13 files changed, 687 insertions(+), 43 deletions(-) create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlow.java create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenEndpointAuthMethod.java create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/OAuth2MockHttpClient.java create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlowTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2.java index b035b02437bb9..2b4ee9940b832 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2.java @@ -21,8 +21,10 @@ import java.net.URL; import java.time.Duration; import java.util.concurrent.ScheduledExecutorService; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.impl.auth.oauth2.protocol.DefaultMetadataResolver; +import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenEndpointAuthMethod; /** * Factory class that allows to create {@link Authentication} instances @@ -92,18 +94,36 @@ public static class ClientCredentialsBuilder { private URL issuerUrl; private URL credentialsUrl; + private TokenEndpointAuthMethod tokenEndpointAuthMethod = + TokenEndpointAuthMethod.CLIENT_SECRET_POST; + private String clientId; + private String tlsCertFile; + private String tlsKeyFile; private String audience; private String scope; private Duration connectTimeout; private Duration readTimeout; private String trustCertsFilePath; private String wellKnownMetadataPath; + private Duration autoCertRefreshDuration; private double earlyTokenRefreshPercent = AuthenticationOAuth2.EARLY_TOKEN_REFRESH_PERCENT_DEFAULT; private ScheduledExecutorService scheduler; private ClientCredentialsBuilder() { } + /** + * Optional token endpoint auth method. + * Defaults to {@code client_secret_post}. + * + * @param tokenEndpointAuthMethod the token endpoint auth method + * @return the builder + */ + public ClientCredentialsBuilder tokenEndpointAuthMethod(TokenEndpointAuthMethod tokenEndpointAuthMethod) { + this.tokenEndpointAuthMethod = tokenEndpointAuthMethod; + return this; + } + /** * Required issuer URL. * @@ -126,6 +146,43 @@ public ClientCredentialsBuilder credentialsUrl(URL credentialsUrl) { return this; } + /** + * Optional path to the file for a client certificate. + * Required when {@code tokenEndpointAuthMethod} is {@code tls_client_auth} + * + * @param tlsCertFile the path to the file for a client certificate + * @return the builder + */ + public ClientCredentialsBuilder tlsCertFile(String tlsCertFile) { + this.tlsCertFile = tlsCertFile; + return this; + } + + /** + * Optional path to the file for a client private key. + * Required when {@code tokenEndpointAuthMethod} is {@code tls_client_auth} + * + * @param tlsKeyFile the path to the file for a client private key + * @return the builder + */ + public ClientCredentialsBuilder tlsKeyFile(String tlsKeyFile) { + this.tlsKeyFile = tlsKeyFile; + return this; + } + + /** + * Optional client identifier issued by the authorization server. + * Only used by {@code tls_client_auth}. + * Defaults to {@code pulsar-client} when not provided. + * + * @param clientId the client identifier + * @return the builder + */ + public ClientCredentialsBuilder clientId(String clientId) { + this.clientId = clientId; + return this; + } + /** * Optional audience identifier used by some Identity Providers, like Auth0. * @@ -137,6 +194,17 @@ public ClientCredentialsBuilder audience(String audience) { return this; } + /** + * Optional certificate refresh interval. + * + * @param autoCertRefreshDuration the Certificate refresh interval + * @return the builder + */ + public ClientCredentialsBuilder autoCertRefreshDuration(Duration autoCertRefreshDuration) { + this.autoCertRefreshDuration = autoCertRefreshDuration; + return this; + } + /** * Optional scope expressed as a list of space-delimited, case-sensitive strings. * The strings are defined by the authorization server. @@ -236,16 +304,41 @@ public ClientCredentialsBuilder scheduler(ScheduledExecutorService scheduler) { * @return an Authentication object */ public Authentication build() { - ClientCredentialsFlow flow = ClientCredentialsFlow.builder() - .issuerUrl(issuerUrl) - .privateKey(credentialsUrl == null ? null : credentialsUrl.toExternalForm()) - .audience(audience) - .scope(scope) - .connectTimeout(connectTimeout) - .readTimeout(readTimeout) - .trustCertsFilePath(trustCertsFilePath) - .wellKnownMetadataPath(wellKnownMetadataPath) - .build(); + Flow flow; + if (tokenEndpointAuthMethod == TokenEndpointAuthMethod.CLIENT_SECRET_POST) { + flow = ClientCredentialsFlow.builder() + .issuerUrl(issuerUrl) + .privateKey(credentialsUrl == null ? null : credentialsUrl.toExternalForm()) + .audience(audience) + .scope(scope) + .connectTimeout(connectTimeout) + .readTimeout(readTimeout) + .trustCertsFilePath(trustCertsFilePath) + .certFile(tlsCertFile) + .keyFile(tlsKeyFile) + .autoCertRefreshDuration(autoCertRefreshDuration) + .wellKnownMetadataPath(wellKnownMetadataPath) + .build(); + } else if (tokenEndpointAuthMethod == TokenEndpointAuthMethod.TLS_CLIENT_AUTH) { + if (StringUtils.isBlank(tlsCertFile) || StringUtils.isBlank(tlsKeyFile)) { + throw new IllegalArgumentException("Required configuration parameters: tlsCertFile, tlsKeyFile"); + } + flow = TlsClientAuthFlow.builder() + .issuerUrl(issuerUrl) + .clientId(clientId) + .certFile(tlsCertFile) + .keyFile(tlsKeyFile) + .audience(audience) + .scope(scope) + .connectTimeout(connectTimeout) + .readTimeout(readTimeout) + .trustCertsFilePath(trustCertsFilePath) + .wellKnownMetadataPath(wellKnownMetadataPath) + .autoCertRefreshDuration(autoCertRefreshDuration) + .build(); + } else { + throw new IllegalArgumentException("Unsupported auth method: " + tokenEndpointAuthMethod); + } return new AuthenticationOAuth2(flow, earlyTokenRefreshPercent, scheduler); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2.java index d40785560ba0d..fc4157d7a7d3f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2.java @@ -39,6 +39,7 @@ import org.apache.pulsar.client.api.EncodedAuthenticationParameterSupport; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.AuthenticationUtil; +import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenEndpointAuthMethod; import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenResult; import org.apache.pulsar.common.util.Backoff; @@ -73,6 +74,7 @@ public class AuthenticationOAuth2 implements Authentication, EncodedAuthenticationParameterSupport { public static final String CONFIG_PARAM_TYPE = "type"; + public static final String CONFIG_PARAM_TOKEN_ENDPOINT_AUTH_METHOD = "tokenEndpointAuthMethod"; public static final String CONFIG_PARAM_EARLY_TOKEN_REFRESH_PERCENT = "earlyTokenRefreshPercent"; public static final String TYPE_CLIENT_CREDENTIALS = "client_credentials"; public static final int EARLY_TOKEN_REFRESH_PERCENT_DEFAULT = 1; // feature disabled by default @@ -158,7 +160,16 @@ public void configure(String encodedAuthParamString) { Map params = parseAuthParameters(encodedAuthParamString); String type = params.getOrDefault(CONFIG_PARAM_TYPE, TYPE_CLIENT_CREDENTIALS); if (TYPE_CLIENT_CREDENTIALS.equals(type)) { - this.flow = ClientCredentialsFlow.fromParameters(params); + TokenEndpointAuthMethod authMethod = TokenEndpointAuthMethod.fromValue( + params.getOrDefault(CONFIG_PARAM_TOKEN_ENDPOINT_AUTH_METHOD, + TokenEndpointAuthMethod.CLIENT_SECRET_POST.value())); + if (authMethod == TokenEndpointAuthMethod.CLIENT_SECRET_POST) { + this.flow = ClientCredentialsFlow.fromParameters(params); + } else if (authMethod == TokenEndpointAuthMethod.TLS_CLIENT_AUTH) { + this.flow = TlsClientAuthFlow.fromParameters(params); + } else { + throw new IllegalArgumentException("Unsupported auth method: " + authMethod); + } } else { throw new IllegalArgumentException("Unsupported authentication type: " + type); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java index d841010add81d..6692e86848b73 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java @@ -35,6 +35,7 @@ import org.apache.pulsar.client.impl.auth.oauth2.protocol.ClientCredentialsExchangeRequest; import org.apache.pulsar.client.impl.auth.oauth2.protocol.ClientCredentialsExchanger; import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenClient; +import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenEndpointAuthMethod; import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenExchangeException; import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenResult; @@ -64,8 +65,10 @@ class ClientCredentialsFlow extends FlowBase { @Builder public ClientCredentialsFlow(URL issuerUrl, String audience, String privateKey, String scope, Duration connectTimeout, Duration readTimeout, String trustCertsFilePath, + String certFile, String keyFile, Duration autoCertRefreshDuration, String wellKnownMetadataPath) { - super(issuerUrl, connectTimeout, readTimeout, trustCertsFilePath, wellKnownMetadataPath); + super(issuerUrl, connectTimeout, readTimeout, trustCertsFilePath, certFile, keyFile, autoCertRefreshDuration, + wellKnownMetadataPath); this.audience = audience; this.privateKey = privateKey; this.scope = scope; @@ -87,6 +90,9 @@ public static ClientCredentialsFlow fromParameters(Map params) { Duration connectTimeout = parseParameterDuration(params, CONFIG_PARAM_CONNECT_TIMEOUT); Duration readTimeout = parseParameterDuration(params, CONFIG_PARAM_READ_TIMEOUT); String trustCertsFilePath = params.get(CONFIG_PARAM_TRUST_CERTS_FILE_PATH); + String certFile = params.get(CONFIG_PARAM_CERT_FILE); + String keyFile = params.get(CONFIG_PARAM_TLS_KEY_FILE); + Duration autoCertRefreshDuration = parseParameterDuration(params, CONFIG_PARAM_AUTO_CERT_REFRESH_DURATION); String wellKnownMetadataPath = params.get(CONFIG_PARAM_WELL_KNOWN_METADATA_PATH); return ClientCredentialsFlow.builder() @@ -97,6 +103,9 @@ public static ClientCredentialsFlow fromParameters(Map params) { .connectTimeout(connectTimeout) .readTimeout(readTimeout) .trustCertsFilePath(trustCertsFilePath) + .certFile(certFile) + .keyFile(keyFile) + .autoCertRefreshDuration(autoCertRefreshDuration) .wellKnownMetadataPath(wellKnownMetadataPath) .build(); } @@ -138,7 +147,7 @@ public void initialize() throws PulsarClientException { assert this.metadata != null; URL tokenUrl = this.metadata.getTokenEndpoint(); - this.exchanger = new TokenClient(tokenUrl, getHttpClient()); + this.exchanger = new TokenClient(tokenUrl, httpClient); initialized = true; } @@ -157,6 +166,7 @@ public TokenResult authenticate() throws PulsarClientException { .clientSecret(keyFile.getClientSecret()) .audience(this.audience) .scope(this.scope) + .authMethod(TokenEndpointAuthMethod.CLIENT_SECRET_POST) .build(); TokenResult tr; if (!initialized) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java index 63248e8605108..d1caa82e07c22 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java @@ -26,6 +26,9 @@ import java.time.Duration; import java.time.format.DateTimeParseException; import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -34,9 +37,14 @@ import org.apache.pulsar.client.impl.auth.oauth2.protocol.DefaultMetadataResolver; import org.apache.pulsar.client.impl.auth.oauth2.protocol.Metadata; import org.apache.pulsar.client.impl.auth.oauth2.protocol.MetadataResolver; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.client.util.PulsarHttpAsyncSslEngineFactory; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.DefaultAsyncHttpClient; import org.asynchttpclient.DefaultAsyncHttpClientConfig; +import org.asynchttpclient.SslEngineFactory; /** * An abstract OAuth 2.0 authorization flow. @@ -47,34 +55,37 @@ abstract class FlowBase implements Flow { public static final String CONFIG_PARAM_CONNECT_TIMEOUT = "connectTimeout"; public static final String CONFIG_PARAM_READ_TIMEOUT = "readTimeout"; public static final String CONFIG_PARAM_TRUST_CERTS_FILE_PATH = "trustCertsFilePath"; + public static final String CONFIG_PARAM_CERT_FILE = "tlsCertFile"; + public static final String CONFIG_PARAM_TLS_KEY_FILE = "tlsKeyFile"; + public static final String CONFIG_PARAM_AUTO_CERT_REFRESH_DURATION = "autoCertRefreshDuration"; public static final String CONFIG_PARAM_WELL_KNOWN_METADATA_PATH = "wellKnownMetadataPath"; protected static final Duration DEFAULT_CONNECT_TIMEOUT = Duration.ofSeconds(10); protected static final Duration DEFAULT_READ_TIMEOUT = Duration.ofSeconds(30); + protected static final Duration DEFAULT_AUTO_CERT_REFRESH_DURATION = Duration.ofSeconds(300); private static final long serialVersionUID = 1L; protected final URL issuerUrl; - private final Duration connectTimeout; - private final Duration readTimeout; - private final String trustCertsFilePath; + protected transient AsyncHttpClient httpClient; protected final String wellKnownMetadataPath; - + protected transient PulsarSslFactory sslFactory; + protected transient ScheduledExecutorService sslRefreshScheduler; protected transient Metadata metadata; - private transient AsyncHttpClient httpClient; protected FlowBase(URL issuerUrl, Duration connectTimeout, Duration readTimeout, String trustCertsFilePath, + String certFile, String keyFile, Duration autoCertRefreshDuration, String wellKnownMetadataPath) { this.issuerUrl = issuerUrl; - this.connectTimeout = connectTimeout; - this.readTimeout = readTimeout; - this.trustCertsFilePath = trustCertsFilePath; + this.httpClient = defaultHttpClient(readTimeout, connectTimeout, trustCertsFilePath, certFile, keyFile); + long autoCertRefreshSeconds = getParameterDurationToSeconds(CONFIG_PARAM_AUTO_CERT_REFRESH_DURATION, + autoCertRefreshDuration, DEFAULT_AUTO_CERT_REFRESH_DURATION); + scheduleSslContextRefreshIfEnabled(autoCertRefreshSeconds); this.wellKnownMetadataPath = wellKnownMetadataPath; - getHttpClient(); } private AsyncHttpClient defaultHttpClient(Duration readTimeout, Duration connectTimeout, - String trustCertsFilePath) { + String trustCertsFilePath, String certFile, String keyFile) { DefaultAsyncHttpClientConfig.Builder confBuilder = new DefaultAsyncHttpClientConfig.Builder(); confBuilder.setCookieStore(null); confBuilder.setUseProxyProperties(true); @@ -85,7 +96,31 @@ private AsyncHttpClient defaultHttpClient(Duration readTimeout, Duration connect confBuilder.setReadTimeout( getParameterDurationToMillis(CONFIG_PARAM_READ_TIMEOUT, readTimeout, DEFAULT_READ_TIMEOUT)); confBuilder.setUserAgent(String.format("Pulsar-Java-v%s", PulsarVersion.getVersion())); - if (StringUtils.isNotBlank(trustCertsFilePath)) { + boolean hasCertFile = StringUtils.isNotBlank(certFile); + boolean hasKeyFile = StringUtils.isNotBlank(keyFile); + if (hasCertFile != hasKeyFile) { + throw new IllegalArgumentException("Invalid TLS client certificate configuration: " + CONFIG_PARAM_CERT_FILE + + " and " + CONFIG_PARAM_TLS_KEY_FILE + " must be provided together"); + } + if (hasCertFile && hasKeyFile) { + try { + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(certFile) + .tlsKeyFilePath(keyFile) + .tlsTrustCertsFilePath(trustCertsFilePath) + .allowInsecureConnection(false) + .serverMode(false) + .isHttps(true) + .build(); + sslFactory = new org.apache.pulsar.common.util.DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslEngineFactory sslEngineFactory = new PulsarHttpAsyncSslEngineFactory(sslFactory, null); + confBuilder.setSslEngineFactory(sslEngineFactory); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid TLS client certificate configuration", e); + } + } else if (StringUtils.isNotBlank(trustCertsFilePath)) { try { confBuilder.setSslContext(SslContextBuilder.forClient() .trustManager(new File(trustCertsFilePath)) @@ -97,28 +132,47 @@ private AsyncHttpClient defaultHttpClient(Duration readTimeout, Duration connect return new DefaultAsyncHttpClient(confBuilder.build()); } - protected synchronized AsyncHttpClient getHttpClient() { - if (httpClient == null) { - httpClient = defaultHttpClient(readTimeout, connectTimeout, trustCertsFilePath); + private void scheduleSslContextRefreshIfEnabled(long refreshSeconds) { + if (sslFactory == null || refreshSeconds <= 0 || sslRefreshScheduler != null) { + return; + } + sslRefreshScheduler = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("oauth2-tls-cert-refresher", true)); + sslRefreshScheduler.scheduleWithFixedDelay(this::refreshSslContext, + refreshSeconds, refreshSeconds, TimeUnit.SECONDS); + log.info("Scheduled TLS certificate refresh, refreshSeconds {}", refreshSeconds); + } + + private void refreshSslContext() { + if (this.sslFactory == null) { + return; + } + try { + this.sslFactory.update(); + log.debug("Successfully refreshed SSL context"); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); } - return httpClient; } private int getParameterDurationToMillis(String name, Duration value, Duration defaultValue) { + return (int) getParameterDuration(name, value, defaultValue).toMillis(); + } + + private long getParameterDurationToSeconds(String name, Duration value, Duration defaultValue) { + return getParameterDuration(name, value, defaultValue).getSeconds(); + } + + private Duration getParameterDuration(String name, Duration value, Duration defaultValue) { Duration duration; if (value == null) { - if (log.isDebugEnabled()) { - log.debug("Configuration for [{}] is using the default value: [{}]", name, defaultValue); - } + log.debug("Configuration is using the default value, name {}, defaultValue {}", name, defaultValue); duration = defaultValue; } else { - if (log.isDebugEnabled()) { - log.debug("Configuration for [{}] is: [{}]", name, value); - } + log.debug("Configuration, name {}, value {}", name, value); duration = value; } - - return (int) duration.toMillis(); + return duration; } public void initialize() throws PulsarClientException { @@ -131,7 +185,7 @@ public void initialize() throws PulsarClientException { } protected MetadataResolver createMetadataResolver() { - return DefaultMetadataResolver.fromIssuerUrl(issuerUrl, getHttpClient(), wellKnownMetadataPath); + return DefaultMetadataResolver.fromIssuerUrl(issuerUrl, httpClient, wellKnownMetadataPath); } static String parseParameterString(Map params, String name) { @@ -168,8 +222,14 @@ static Duration parseParameterDuration(Map params, String name) @Override public void close() throws Exception { + if (sslRefreshScheduler != null) { + sslRefreshScheduler.shutdownNow(); + } if (httpClient != null) { httpClient.close(); } + if (sslFactory != null) { + sslFactory.close(); + } } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlow.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlow.java new file mode 100644 index 0000000000000..d61dadef83bfb --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlow.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl.auth.oauth2; + +import com.google.common.annotations.VisibleForTesting; +import java.io.IOException; +import java.net.URL; +import java.time.Duration; +import java.util.Map; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.auth.oauth2.protocol.ClientCredentialsExchangeRequest; +import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenClient; +import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenEndpointAuthMethod; +import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenExchangeException; +import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenResult; + +/** + * Implementation of OAuth 2.0 Client TLS Authentication flow. + * + * @see RFC 8705 - OAuth 2.0 Mutual-TLS Client Authentication + */ +@Slf4j +class TlsClientAuthFlow extends FlowBase { + public static final String CONFIG_PARAM_ISSUER_URL = "issuerUrl"; + public static final String CONFIG_PARAM_CLIENT_ID = "clientId"; + public static final String CONFIG_PARAM_AUDIENCE = "audience"; + public static final String CONFIG_PARAM_SCOPE = "scope"; + public static final String CONFIG_PARAM_AUTO_CERT_REFRESH_DURATION = + FlowBase.CONFIG_PARAM_AUTO_CERT_REFRESH_DURATION; + + private static final String DEFAULT_CLIENT_ID = "pulsar-client"; + + private static final long serialVersionUID = 1L; + + private final String clientId; + private final String audience; + private final String scope; + + private transient TokenClient exchanger; + + private boolean initialized = false; + + @Builder + public TlsClientAuthFlow(URL issuerUrl, String clientId, String certFile, String keyFile, String audience, + String scope, Duration connectTimeout, Duration readTimeout, String trustCertsFilePath, + String wellKnownMetadataPath, Duration autoCertRefreshDuration) { + super(issuerUrl, connectTimeout, readTimeout, trustCertsFilePath, certFile, keyFile, autoCertRefreshDuration, + wellKnownMetadataPath); + this.clientId = StringUtils.defaultIfBlank(clientId, DEFAULT_CLIENT_ID); + this.audience = audience; + this.scope = scope; + } + + /** + * Constructs a {@link TlsClientAuthFlow} from configuration parameters. + * + * @param params Configuration parameters + * @return A new TlsClientAuthFlow instance + */ + public static TlsClientAuthFlow fromParameters(Map params) { + URL issuerUrl = parseParameterUrl(params, CONFIG_PARAM_ISSUER_URL); + // In mTLS-based providers, caller input for client_id can be optional. + // Keep sending client_id in token request for RFC compatibility by applying a default value. + String clientId = params.getOrDefault(CONFIG_PARAM_CLIENT_ID, DEFAULT_CLIENT_ID); + String certFile = parseParameterString(params, CONFIG_PARAM_CERT_FILE); + String keyFile = parseParameterString(params, CONFIG_PARAM_TLS_KEY_FILE); + // These are optional parameters, so we allow null values + String scope = params.get(CONFIG_PARAM_SCOPE); + String audience = params.get(CONFIG_PARAM_AUDIENCE); + Duration connectTimeout = parseParameterDuration(params, CONFIG_PARAM_CONNECT_TIMEOUT); + Duration readTimeout = parseParameterDuration(params, CONFIG_PARAM_READ_TIMEOUT); + String trustCertsFilePath = params.get(CONFIG_PARAM_TRUST_CERTS_FILE_PATH); + String wellKnownMetadataPath = params.get(CONFIG_PARAM_WELL_KNOWN_METADATA_PATH); + Duration autoCertRefreshDuration = parseParameterDuration(params, CONFIG_PARAM_AUTO_CERT_REFRESH_DURATION); + + return TlsClientAuthFlow.builder() + .issuerUrl(issuerUrl) + .clientId(clientId) + .certFile(certFile) + .keyFile(keyFile) + .audience(audience) + .scope(scope) + .connectTimeout(connectTimeout) + .readTimeout(readTimeout) + .trustCertsFilePath(trustCertsFilePath) + .wellKnownMetadataPath(wellKnownMetadataPath) + .autoCertRefreshDuration(autoCertRefreshDuration) + .build(); + } + + @Override + public void initialize() throws PulsarClientException { + super.initialize(); + assert this.metadata != null; + + URL tokenUrl = this.metadata.getTokenEndpoint(); + this.exchanger = new TokenClient(tokenUrl, httpClient); + + initialized = true; + } + + public TokenResult authenticate() throws PulsarClientException { + // request an access token using TLS client authentication + ClientCredentialsExchangeRequest req = ClientCredentialsExchangeRequest.builder() + .clientId(this.clientId) + .audience(this.audience) + .scope(this.scope) + .authMethod(TokenEndpointAuthMethod.TLS_CLIENT_AUTH) + .build(); + TokenResult tr; + if (!initialized) { + initialize(); + } + try { + tr = this.exchanger.exchangeClientCredentials(req); + } catch (TokenExchangeException | IOException e) { + throw new PulsarClientException.AuthenticationException("Unable to obtain an access token: " + + e.getMessage()); + } + + return tr; + } + + @Override + public void close() throws Exception { + super.close(); + if (exchanger != null) { + exchanger.close(); + } + } + + @VisibleForTesting + String getClientId() { + return clientId; + } +} \ No newline at end of file diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/ClientCredentialsExchangeRequest.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/ClientCredentialsExchangeRequest.java index da026bce47eee..de36da5ad5be1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/ClientCredentialsExchangeRequest.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/ClientCredentialsExchangeRequest.java @@ -42,4 +42,7 @@ public class ClientCredentialsExchangeRequest { @JsonProperty("scope") private String scope; + + @JsonProperty("token_endpoint_auth_method") + private TokenEndpointAuthMethod authMethod; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java index cb4c2a551d01e..8beb1d32814e7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClient.java @@ -56,10 +56,15 @@ public void close() throws Exception { * @return Generate the final request body from a map. */ String buildClientCredentialsBody(ClientCredentialsExchangeRequest req) { + TokenEndpointAuthMethod authMethod = req.getAuthMethod() == null + ? TokenEndpointAuthMethod.CLIENT_SECRET_POST + : req.getAuthMethod(); Map bodyMap = new TreeMap<>(); bodyMap.put("grant_type", "client_credentials"); bodyMap.put("client_id", req.getClientId()); - bodyMap.put("client_secret", req.getClientSecret()); + if (authMethod == TokenEndpointAuthMethod.CLIENT_SECRET_POST) { + bodyMap.put("client_secret", req.getClientSecret()); + } // Only set audience and scope if they are non-empty. if (!StringUtils.isBlank(req.getAudience())) { bodyMap.put("audience", req.getAudience()); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenEndpointAuthMethod.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenEndpointAuthMethod.java new file mode 100644 index 0000000000000..104a64bd75630 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenEndpointAuthMethod.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl.auth.oauth2.protocol; + +public enum TokenEndpointAuthMethod { + CLIENT_SECRET_POST("client_secret_post"), + TLS_CLIENT_AUTH("tls_client_auth"); + + private final String value; + + TokenEndpointAuthMethod(String value) { + this.value = value; + } + + public String value() { + return value; + } + + public static TokenEndpointAuthMethod fromValue(String value) { + for (TokenEndpointAuthMethod method : values()) { + if (method.value.equalsIgnoreCase(value)) { + return method; + } + } + throw new IllegalArgumentException("Unsupported token endpoint auth method: " + value); + } +} \ No newline at end of file diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2Test.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2Test.java index f76fee6e10dfa..ea675af997423 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2Test.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationFactoryOAuth2Test.java @@ -18,11 +18,14 @@ */ package org.apache.pulsar.client.impl.auth.oauth2; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import java.io.IOException; import java.net.URL; import java.time.Duration; import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenEndpointAuthMethod; import org.testng.annotations.Test; public class AuthenticationFactoryOAuth2Test { @@ -36,17 +39,56 @@ public void testBuilder() throws IOException { Duration connectTimeout = Duration.parse("PT11S"); Duration readTimeout = Duration.ofSeconds(31); String trustCertsFilePath = null; + String tlsCertFile = ""; + String tlsKeyFile = ""; String wellKnownMetadataPath = "/.well-known/custom-path"; try (Authentication authentication = AuthenticationFactoryOAuth2.clientCredentialsBuilder().issuerUrl(issuerUrl) .credentialsUrl(credentialsUrl).audience(audience).scope(scope) .connectTimeout(connectTimeout).readTimeout(readTimeout) - .trustCertsFilePath(trustCertsFilePath) - .wellKnownMetadataPath(wellKnownMetadataPath).build()) { + .trustCertsFilePath(trustCertsFilePath).tlsCertFile(tlsCertFile) + .tlsKeyFile(tlsKeyFile).wellKnownMetadataPath(wellKnownMetadataPath).build()) { assertTrue(authentication instanceof AuthenticationOAuth2); + assertEquals(((AuthenticationOAuth2) authentication).flow.getClass(), ClientCredentialsFlow.class); } } + @Test + public void testBuilderWithTlsClientAuthFlow() throws Exception { + URL issuerUrl = new URL("http://localhost"); + String clientId = "test-client"; + String tlsCertFile = "/path/to/cert.pem"; + String tlsKeyFile = "/path/to/key.pem"; + String audience = "audience"; + String scope = "scope"; + Duration autoCertRefreshDuration = Duration.ofSeconds(123); + OAuth2MockHttpClient.withMockedSslFactory(() -> { + try (Authentication authentication = + AuthenticationFactoryOAuth2.clientCredentialsBuilder().issuerUrl(issuerUrl) + .tokenEndpointAuthMethod(TokenEndpointAuthMethod.TLS_CLIENT_AUTH) + .clientId(clientId) + .tlsCertFile(tlsCertFile) + .tlsKeyFile(tlsKeyFile) + .audience(audience) + .scope(scope) + .autoCertRefreshDuration(autoCertRefreshDuration) + .build()) { + assertTrue(authentication instanceof AuthenticationOAuth2); + assertEquals(((AuthenticationOAuth2) authentication).flow.getClass(), TlsClientAuthFlow.class); + } + }); + } + + @Test + public void testBuilderWithTlsClientAuthMissingCertOrKey() throws IOException { + URL issuerUrl = new URL("http://localhost"); + assertThrows(IllegalArgumentException.class, () -> + AuthenticationFactoryOAuth2.clientCredentialsBuilder() + .issuerUrl(issuerUrl) + .tokenEndpointAuthMethod(TokenEndpointAuthMethod.TLS_CLIENT_AUTH) + .build()); + } + @Test public void testStandardAuthzServerBuilder() throws IOException { URL issuerUrl = new URL("http://localhost"); @@ -60,6 +102,7 @@ public void testStandardAuthzServerBuilder() throws IOException { } } + @SuppressWarnings("deprecation") @Test public void testClientCredentials() throws IOException { URL issuerUrl = new URL("http://localhost"); @@ -71,4 +114,4 @@ public void testClientCredentials() throws IOException { } } -} +} \ No newline at end of file diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java index 46dbe41b48e41..6b956957d307b 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java @@ -41,7 +41,9 @@ import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.auth.oauth2.protocol.DefaultMetadataResolver; +import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenEndpointAuthMethod; import org.apache.pulsar.client.impl.auth.oauth2.protocol.TokenResult; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -63,6 +65,13 @@ public void before() { this.auth = new AuthenticationOAuth2(flow, this.clock, 1, null); } + @AfterMethod(alwaysRun = true) + public void after() throws Exception { + if (this.auth != null) { + this.auth.close(); + } + } + @Test public void testGetAuthMethodName() { assertEquals(this.auth.getAuthMethodName(), "token"); @@ -110,6 +119,42 @@ public void testConfigureWithoutOptionalParams() throws Exception { assertNotNull(this.auth.flow); } + @Test + public void testConfigureWithTlsClientAuth() throws Exception { + Map params = new HashMap<>(); + params.put("type", "client_credentials"); + params.put(AuthenticationOAuth2.CONFIG_PARAM_TOKEN_ENDPOINT_AUTH_METHOD, + TokenEndpointAuthMethod.TLS_CLIENT_AUTH.value()); + params.put("clientId", "test-client"); + params.put("tlsCertFile", "/path/to/cert.pem"); + params.put("tlsKeyFile", "/path/to/key.pem"); + params.put("issuerUrl", "http://localhost"); + ObjectMapper mapper = new ObjectMapper(); + String authParams = mapper.writeValueAsString(params); + OAuth2MockHttpClient.withMockedSslFactory(() -> { + this.auth.configure(authParams); + assertNotNull(this.auth.flow); + assertEquals(this.auth.flow.getClass(), TlsClientAuthFlow.class); + }); + } + + @Test + public void testConfigureCredentialsWithTlsValues() throws Exception { + Map params = new HashMap<>(); + params.put("type", "client_credentials"); + params.put("privateKey", "data:base64,e30="); + params.put("tlsCertFile", "/path/to/cert.pem"); + params.put("tlsKeyFile", "/path/to/key.pem"); + params.put("issuerUrl", "http://localhost"); + ObjectMapper mapper = new ObjectMapper(); + String authParams = mapper.writeValueAsString(params); + OAuth2MockHttpClient.withMockedSslFactory(() -> { + this.auth.configure(authParams); + assertNotNull(this.auth.flow); + assertEquals(this.auth.flow.getClass(), ClientCredentialsFlow.class); + }); + } + // ----- configure() via default constructor ----- @Test @@ -360,4 +405,4 @@ public void testClose() throws Exception { verify(this.flow).close(); assertThrows(PulsarClientException.AlreadyClosedException.class, () -> this.auth.getAuthData()); } -} +} \ No newline at end of file diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/OAuth2MockHttpClient.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/OAuth2MockHttpClient.java new file mode 100644 index 0000000000000..7a0bb0663e134 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/OAuth2MockHttpClient.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl.auth.oauth2; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mockConstruction; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; +import org.asynchttpclient.DefaultAsyncHttpClient; +import org.mockito.MockedConstruction; + +final class OAuth2MockHttpClient { + + private OAuth2MockHttpClient() { + } + + @FunctionalInterface + interface ThrowingRunnable { + void run() throws Exception; + } + + static void withMockedSslFactory(ThrowingRunnable runnable) throws Exception { + try (MockedConstruction ignoredSslFactory = + mockConstruction(DefaultPulsarSslFactory.class, (mock, context) -> { + doNothing().when(mock).initialize(any()); + doNothing().when(mock).createInternalSslContext(); + }); + MockedConstruction ignoredHttpClient = + mockConstruction(DefaultAsyncHttpClient.class)) { + runnable.run(); + } + } +} \ No newline at end of file diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlowTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlowTest.java new file mode 100644 index 0000000000000..203a0f6f15c0b --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlowTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl.auth.oauth2; + +import static org.testng.Assert.assertEquals; +import java.util.HashMap; +import java.util.Map; +import org.testng.annotations.Test; + +public class TlsClientAuthFlowTest { + + @Test + public void testFromParametersWithoutClientId() throws Exception { + Map params = new HashMap<>(); + params.put("tlsCertFile", "/path/to/cert.pem"); + params.put("tlsKeyFile", "/path/to/key.pem"); + params.put("issuerUrl", "http://localhost"); + params.put("scope", "http://localhost"); + OAuth2MockHttpClient.withMockedSslFactory(() -> { + TlsClientAuthFlow flow = TlsClientAuthFlow.fromParameters(params); + assertEquals(flow.getClientId(), "pulsar-client"); + flow.close(); + }); + } +} \ No newline at end of file diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClientTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClientTest.java index 131427682076e..0992eef040923 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClientTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/protocol/TokenClientTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl.auth.oauth2.protocol; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertNotNull; @@ -48,8 +49,15 @@ public void exchangeClientCredentialsSuccessByScopeTest() throws .clientId("test-client-id") .clientSecret("test-client-secret") .scope("test-scope") + .authMethod(TokenEndpointAuthMethod.CLIENT_SECRET_POST) .build(); String body = tokenClient.buildClientCredentialsBody(request); + assertThat(body) + .contains("grant_type=") + .contains("client_id=") + .contains("client_secret=") + .contains("audience=") + .contains("scope="); BoundRequestBuilder boundRequestBuilder = mock(BoundRequestBuilder.class); Response response = mock(Response.class); ListenableFuture listenableFuture = mock(ListenableFuture.class); @@ -81,6 +89,10 @@ public void exchangeClientCredentialsSuccessWithoutOptionalClientCredentialsTest .clientSecret("test-client-secret") .build(); String body = tokenClient.buildClientCredentialsBody(request); + assertThat(body) + .contains("grant_type=") + .contains("client_id=") + .contains("client_secret="); BoundRequestBuilder boundRequestBuilder = mock(BoundRequestBuilder.class); Response response = mock(Response.class); ListenableFuture listenableFuture = mock(ListenableFuture.class); @@ -99,4 +111,78 @@ public void exchangeClientCredentialsSuccessWithoutOptionalClientCredentialsTest TokenResult tr = tokenClient.exchangeClientCredentials(request); assertNotNull(tr); } -} + + @Test + @SuppressWarnings("unchecked") + public void exchangeTlsClientAuthSuccessTest() throws + IOException, TokenExchangeException, ExecutionException, InterruptedException { + DefaultAsyncHttpClient defaultAsyncHttpClient = mock(DefaultAsyncHttpClient.class); + URL url = new URL("http://localhost"); + TokenClient tokenClient = new TokenClient(url, defaultAsyncHttpClient); + ClientCredentialsExchangeRequest request = ClientCredentialsExchangeRequest.builder() + .clientId("test-client-id") + .audience("test-audience") + .scope("test-scope") + .authMethod(TokenEndpointAuthMethod.TLS_CLIENT_AUTH) + .build(); + String body = tokenClient.buildClientCredentialsBody(request); + assertThat(body) + .contains("grant_type=") + .contains("client_id=") + .contains("audience=") + .contains("scope=") + .doesNotContain("client_secret="); + BoundRequestBuilder boundRequestBuilder = mock(BoundRequestBuilder.class); + Response response = mock(Response.class); + ListenableFuture listenableFuture = mock(ListenableFuture.class); + when(defaultAsyncHttpClient.preparePost(url.toString())).thenReturn(boundRequestBuilder); + when(boundRequestBuilder.setHeader("Accept", "application/json")).thenReturn(boundRequestBuilder); + when(boundRequestBuilder.setHeader("Content-Type", + "application/x-www-form-urlencoded")).thenReturn(boundRequestBuilder); + when(boundRequestBuilder.setBody(body)).thenReturn(boundRequestBuilder); + when(boundRequestBuilder.execute()).thenReturn(listenableFuture); + when(listenableFuture.get()).thenReturn(response); + when(response.getStatusCode()).thenReturn(200); + TokenResult tokenResult = new TokenResult(); + tokenResult.setAccessToken("test-access-token"); + tokenResult.setIdToken("test-id"); + when(response.getResponseBodyAsBytes()).thenReturn(new Gson().toJson(tokenResult).getBytes()); + TokenResult tr = tokenClient.exchangeClientCredentials(request); + assertNotNull(tr); + } + + @Test + @SuppressWarnings("unchecked") + public void exchangeTlsClientAuthSuccessWithoutOptionalParamsTest() throws + IOException, TokenExchangeException, ExecutionException, InterruptedException { + DefaultAsyncHttpClient defaultAsyncHttpClient = mock(DefaultAsyncHttpClient.class); + URL url = new URL("http://localhost"); + TokenClient tokenClient = new TokenClient(url, defaultAsyncHttpClient); + ClientCredentialsExchangeRequest request = ClientCredentialsExchangeRequest.builder() + .clientId("test-client-id") + .authMethod(TokenEndpointAuthMethod.TLS_CLIENT_AUTH) + .build(); + String body = tokenClient.buildClientCredentialsBody(request); + assertThat(body) + .contains("grant_type=") + .contains("client_id=") + .doesNotContain("client_secret="); + BoundRequestBuilder boundRequestBuilder = mock(BoundRequestBuilder.class); + Response response = mock(Response.class); + ListenableFuture listenableFuture = mock(ListenableFuture.class); + when(defaultAsyncHttpClient.preparePost(url.toString())).thenReturn(boundRequestBuilder); + when(boundRequestBuilder.setHeader("Accept", "application/json")).thenReturn(boundRequestBuilder); + when(boundRequestBuilder.setHeader("Content-Type", + "application/x-www-form-urlencoded")).thenReturn(boundRequestBuilder); + when(boundRequestBuilder.setBody(body)).thenReturn(boundRequestBuilder); + when(boundRequestBuilder.execute()).thenReturn(listenableFuture); + when(listenableFuture.get()).thenReturn(response); + when(response.getStatusCode()).thenReturn(200); + TokenResult tokenResult = new TokenResult(); + tokenResult.setAccessToken("test-access-token"); + tokenResult.setIdToken("test-id"); + when(response.getResponseBodyAsBytes()).thenReturn(new Gson().toJson(tokenResult).getBytes()); + TokenResult tr = tokenClient.exchangeClientCredentials(request); + assertNotNull(tr); + } +} \ No newline at end of file From f75ede31c7cd657ae79ccebc7e1e138416a807f4 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 12 May 2026 18:18:27 +0300 Subject: [PATCH 073/201] [improve][misc] Upgrade Jetty to 12.1.9 (#25752) (cherry picked from commit 7658b0cad2b693f88be08882a6b7f3218341993f) --- .../server/src/assemble/LICENSE.bin.txt | 72 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 24 +++---- pom.xml | 2 +- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index fcb664ade0582..7fbc59162167a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -392,43 +392,43 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.14.5.jar - org.asynchttpclient-async-http-client-netty-utils-2.14.5.jar * Jetty - - org.eclipse.jetty-jetty-alpn-client-12.1.8.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.8.jar - - org.eclipse.jetty-jetty-alpn-server-12.1.8.jar - - org.eclipse.jetty-jetty-annotations-12.1.8.jar - - org.eclipse.jetty-jetty-client-12.1.8.jar - - org.eclipse.jetty-jetty-http-12.1.8.jar - - org.eclipse.jetty-jetty-io-12.1.8.jar - - org.eclipse.jetty-jetty-jndi-12.1.8.jar - - org.eclipse.jetty-jetty-plus-12.1.8.jar - - org.eclipse.jetty-jetty-security-12.1.8.jar - - org.eclipse.jetty-jetty-server-12.1.8.jar - - org.eclipse.jetty-jetty-session-12.1.8.jar - - org.eclipse.jetty-jetty-util-12.1.8.jar - - org.eclipse.jetty-jetty-xml-12.1.8.jar - - org.eclipse.jetty.compression-jetty-compression-common-12.1.8.jar - - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.8.jar - - org.eclipse.jetty.compression-jetty-compression-server-12.1.8.jar - - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.8.jar - - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.8.jar - - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.8.jar - - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.8.jar - - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.8.jar - - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.8.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.8.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.8.jar - - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.8.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.8.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.8.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.8.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.8.jar + - org.eclipse.jetty-jetty-alpn-client-12.1.9.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.9.jar + - org.eclipse.jetty-jetty-alpn-server-12.1.9.jar + - org.eclipse.jetty-jetty-annotations-12.1.9.jar + - org.eclipse.jetty-jetty-client-12.1.9.jar + - org.eclipse.jetty-jetty-http-12.1.9.jar + - org.eclipse.jetty-jetty-io-12.1.9.jar + - org.eclipse.jetty-jetty-jndi-12.1.9.jar + - org.eclipse.jetty-jetty-plus-12.1.9.jar + - org.eclipse.jetty-jetty-security-12.1.9.jar + - org.eclipse.jetty-jetty-server-12.1.9.jar + - org.eclipse.jetty-jetty-session-12.1.9.jar + - org.eclipse.jetty-jetty-util-12.1.9.jar + - org.eclipse.jetty-jetty-xml-12.1.9.jar + - org.eclipse.jetty.compression-jetty-compression-common-12.1.9.jar + - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.9.jar + - org.eclipse.jetty.compression-jetty-compression-server-12.1.9.jar + - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.9.jar + - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.9.jar + - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.9.jar + - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.9.jar + - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.9.jar + - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.9.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.9.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.9.jar + - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.9.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.9.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.9.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.9.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.9.jar - org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.8.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.8.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.8.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.8.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.8.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.8.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.9.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.9.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.9.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.9.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.9.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.9.jar * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.45.0.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 089cd99d9d490..bbd7e8892bda6 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -401,18 +401,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.14.5.jar - async-http-client-netty-utils-2.14.5.jar * Jetty - - jetty-alpn-client-12.1.8.jar - - jetty-client-12.1.8.jar - - jetty-compression-common-12.1.8.jar - - jetty-compression-gzip-12.1.8.jar - - jetty-http-12.1.8.jar - - jetty-io-12.1.8.jar - - jetty-util-12.1.8.jar - - jetty-websocket-core-client-12.1.8.jar - - jetty-websocket-core-common-12.1.8.jar - - jetty-websocket-jetty-api-12.1.8.jar - - jetty-websocket-jetty-client-12.1.8.jar - - jetty-websocket-jetty-common-12.1.8.jar + - jetty-alpn-client-12.1.9.jar + - jetty-client-12.1.9.jar + - jetty-compression-common-12.1.9.jar + - jetty-compression-gzip-12.1.9.jar + - jetty-http-12.1.9.jar + - jetty-io-12.1.9.jar + - jetty-util-12.1.9.jar + - jetty-websocket-core-client-12.1.9.jar + - jetty-websocket-core-common-12.1.9.jar + - jetty-websocket-jetty-api-12.1.9.jar + - jetty-websocket-jetty-client-12.1.9.jar + - jetty-websocket-jetty-common-12.1.9.jar * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.45.0.jar * Javassist -- javassist-3.25.0-GA.jar diff --git a/pom.xml b/pom.xml index 5e0f310c8eb40..ac3035737fe95 100644 --- a/pom.xml +++ b/pom.xml @@ -191,7 +191,7 @@ flexible messaging model and an intuitive client API. 5.7.1 4.1.133.Final 0.0.26.Final - 12.1.8 + 12.1.9 9.4.58.v20250814 2.5.2 From c11ddd937b9e6568a6ee90e0a80b828c094f63e7 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 13 May 2026 15:44:45 +0300 Subject: [PATCH 074/201] [fix][client] Apply Avro logical type conversions when decoding schema without classloader (#25759) (cherry picked from commit 30ca23f463412de0fe49007d8d5215252e693c56) --- .../client/impl/schema/reader/AvroReader.java | 24 ++++------- .../client/impl/schema/AvroSchemaTest.java | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/AvroReader.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/AvroReader.java index 3ad43dbd9f2af..0c9b8d37ae2bb 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/AvroReader.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/AvroReader.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl.schema.reader; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.io.InputStream; import java.util.Optional; @@ -39,32 +40,21 @@ public class AvroReader implements SchemaReader { new ThreadLocal<>(); private final Schema schema; + @VisibleForTesting public AvroReader(Schema schema) { - this.reader = new ReflectDatumReader<>(schema); - this.schema = schema; + this(schema, schema, null, false); } public AvroReader(Schema schema, ClassLoader classLoader, boolean jsr310ConversionEnabled) { - this.schema = schema; - if (classLoader != null) { - ReflectData reflectData = new ReflectData(classLoader); - AvroSchema.addLogicalTypeConversions(reflectData, jsr310ConversionEnabled); - this.reader = new ReflectDatumReader<>(schema, schema, reflectData); - } else { - this.reader = new ReflectDatumReader<>(schema); - } + this(schema, schema, classLoader, jsr310ConversionEnabled); } public AvroReader(Schema writerSchema, Schema readerSchema, ClassLoader classLoader, boolean jsr310ConversionEnabled) { this.schema = readerSchema; - if (classLoader != null) { - ReflectData reflectData = new ReflectData(classLoader); - AvroSchema.addLogicalTypeConversions(reflectData, jsr310ConversionEnabled); - this.reader = new ReflectDatumReader<>(writerSchema, readerSchema, reflectData); - } else { - this.reader = new ReflectDatumReader<>(writerSchema, readerSchema); - } + ReflectData reflectData = classLoader != null ? new ReflectData(classLoader) : new ReflectData(); + AvroSchema.addLogicalTypeConversions(reflectData, jsr310ConversionEnabled); + this.reader = new ReflectDatumReader<>(writerSchema, readerSchema, reflectData); } @Override diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index e0a6cdc54fc10..7e7602594d159 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -553,4 +553,44 @@ public void testLocalDateTime() { LocalDateTimePojo pojo = avroSchema.decode(bytes); assertEquals(pojo.getValue().truncatedTo(ChronoUnit.MILLIS), now.truncatedTo(ChronoUnit.MILLIS)); } + + public static class UuidLogicalTypeDto { + public UUID id; + public String name; + + public UuidLogicalTypeDto() { + } + + public UuidLogicalTypeDto(UUID id, String name) { + this.id = id; + this.name = name; + } + } + + @Test + public void testDecodeUuidLogicalTypeWithClass() { + UUID id = UUID.randomUUID(); + String name = "name"; + AvroSchema schema = AvroSchema.of(UuidLogicalTypeDto.class); + byte[] encoded = schema.encode(new UuidLogicalTypeDto(id, name)); + + UuidLogicalTypeDto decoded = schema.decode(encoded); + assertEquals(decoded.id, id); + assertEquals(decoded.name, name); + } + + @Test + public void testDecodeUuidLogicalTypeWithJsonDef() { + UUID id = UUID.randomUUID(); + String name = "name"; + AvroSchema schemaFromClass = AvroSchema.of(UuidLogicalTypeDto.class); + byte[] encoded = schemaFromClass.encode(new UuidLogicalTypeDto(id, name)); + String jsonDef = schemaFromClass.getSchemaInfo().getSchemaDefinition(); + + AvroSchema schemaFromJsonDef = AvroSchema.of( + SchemaDefinition.builder().withJsonDef(jsonDef).build()); + UuidLogicalTypeDto decoded = schemaFromJsonDef.decode(encoded); + assertEquals(decoded.id, id); + assertEquals(decoded.name, name); + } } From 7a7ef8755ad950d3dad35325f2f490d88b927735 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 28 Apr 2026 10:47:45 -0700 Subject: [PATCH 075/201] [fix][test] Fix flaky ExtensibleLoadManagerImplTest.testLoadBalancerServiceUnitTableViewSyncer (#25596) (cherry picked from commit d69c78753c5f3747e8d961ee9ba3a419f9c9d8af) --- .../extensions/ExtensibleLoadManagerImplTest.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index ea417a13c6df7..ef71b542b6301 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -1370,10 +1370,15 @@ public void testLoadBalancerServiceUnitTableViewSyncer() throws Exception { assertTrue(pulsar2.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled())); makeSecondaryAsLeader(); makePrimaryAsLeader(); - Awaitility.await().atMost(30, TimeUnit.SECONDS) + // playLeader()/playFollower() run on a single-threaded loadManagerExecutor, so + // playLeader() on the new leader can be queued behind a still-running playFollower() + // from the prior demotion. Under CI load this serial chain plus the syncer.start() + // work (which opens both table views and runs syncExistingItems/syncTailItems) + // can take longer than 30s, so use a more generous timeout here. + Awaitility.await().atMost(60, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(primaryLoadManager.getServiceUnitStateTableViewSyncer() .isActive())); - Awaitility.await().atMost(30, TimeUnit.SECONDS) + Awaitility.await().atMost(60, TimeUnit.SECONDS) .untilAsserted(() -> assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer() .isActive())); ServiceConfiguration defaultConf = getDefaultConf(); @@ -1582,10 +1587,10 @@ public void testLoadBalancerServiceUnitTableViewSyncer() throws Exception { Awaitility.await().untilAsserted(() -> assertFalse(pulsar2.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled())); makeSecondaryAsLeader(); - Awaitility.await().atMost(30, TimeUnit.SECONDS) + Awaitility.await().atMost(60, TimeUnit.SECONDS) .untilAsserted(() -> assertFalse(primaryLoadManager.getServiceUnitStateTableViewSyncer() .isActive())); - Awaitility.await().atMost(30, TimeUnit.SECONDS) + Awaitility.await().atMost(60, TimeUnit.SECONDS) .untilAsserted(() -> assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer() .isActive())); } From 1259fc1514eea6d685e1b41300e3c75f1fa2b9e3 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 1 May 2026 06:09:07 -0700 Subject: [PATCH 076/201] [fix][test] Reduce flakiness in testLoadBalancerServiceUnitTableViewSyncer (#25638) (cherry picked from commit 2accf437172cef87776973335138c151bb26bd1a) --- .../ExtensibleLoadManagerImplTest.java | 346 ++++++------------ 1 file changed, 113 insertions(+), 233 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index ef71b542b6301..ec76eb7735524 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -1331,268 +1331,148 @@ public void testDeployAndRollbackLoadManager() throws Exception { @Test(priority = 200) public void testLoadBalancerServiceUnitTableViewSyncer() throws Exception { + // Make pulsar1 the leader so primaryLoadManager is the syncer-running broker. + makePrimaryAsLeader(); Pair topicAndBundle = getBundleIsNotOwnByChangeEventTopic("testLoadBalancerServiceUnitTableViewSyncer"); - TopicName topicName = topicAndBundle.getLeft(); NamespaceBundle bundle = topicAndBundle.getRight(); - String topic = topicName.toString(); - - String lookupResultBefore1 = pulsar1.getAdminClient().lookups().lookupTopic(topic); - String lookupResultBefore2 = pulsar2.getAdminClient().lookups().lookupTopic(topic); - assertEquals(lookupResultBefore1, lookupResultBefore2); + String topic = topicAndBundle.getLeft().toString(); LookupOptions options = LookupOptions.builder() .authoritative(false) .requestHttps(false) .readOnly(false) .loadTopicsInBundle(false).build(); - Optional webServiceUrlBefore1 = - pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrlBefore1.isPresent()); - Optional webServiceUrlBefore2 = - pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrlBefore2.isPresent()); - assertEquals(webServiceUrlBefore2.get().toString(), webServiceUrlBefore1.get().toString()); - - - String syncerTyp = serviceUnitStateTableViewClassName.equals(ServiceUnitStateTableViewImpl.class.getName()) - ? "SystemTopicToMetadataStoreSyncer" : "MetadataStoreToSystemTopicSyncer"; + String ownershipBefore = pulsar1.getAdminClient().lookups().lookupTopic(topic); + assertEquals(pulsar2.getAdminClient().lookups().lookupTopic(topic), ownershipBefore); + Optional webUrlBefore = pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webUrlBefore.isPresent()); + + // === Phase 1: enable the syncer and verify it activates on the leader only === + // The syncer is started inside ExtensibleLoadManagerImpl.monitor() when the + // dynamic config is enabled and the broker is the channel owner. Calling + // monitor() directly avoids forcing a leader transition (which serializes + // playLeader() behind playFollower() on the single-threaded load manager + // executor and was the root cause of repeated 30s+ flakes here). + String syncerType = + serviceUnitStateTableViewClassName.equals(ServiceUnitStateTableViewImpl.class.getName()) + ? "SystemTopicToMetadataStoreSyncer" : "MetadataStoreToSystemTopicSyncer"; pulsar.getAdminClient().brokers() - .updateDynamicConfiguration("loadBalancerServiceUnitTableViewSyncer", syncerTyp); - // Wait for the dynamic config to propagate to both brokers before triggering - // leader transitions. Without this, the leader callback may see the old config - // and skip activating the syncer. + .updateDynamicConfiguration("loadBalancerServiceUnitTableViewSyncer", syncerType); Awaitility.await().untilAsserted(() -> assertTrue(pulsar1.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled())); - Awaitility.await().untilAsserted(() -> - assertTrue(pulsar2.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled())); - makeSecondaryAsLeader(); - makePrimaryAsLeader(); - // playLeader()/playFollower() run on a single-threaded loadManagerExecutor, so - // playLeader() on the new leader can be queued behind a still-running playFollower() - // from the prior demotion. Under CI load this serial chain plus the syncer.start() - // work (which opens both table views and runs syncExistingItems/syncTailItems) - // can take longer than 30s, so use a more generous timeout here. - Awaitility.await().atMost(60, TimeUnit.SECONDS) + primaryLoadManager.monitor(); + Awaitility.await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(primaryLoadManager.getServiceUnitStateTableViewSyncer() .isActive())); - Awaitility.await().atMost(60, TimeUnit.SECONDS) - .untilAsserted(() -> assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer() - .isActive())); - ServiceConfiguration defaultConf = getDefaultConf(); - defaultConf.setAllowAutoTopicCreation(true); - defaultConf.setForceDeleteNamespaceAllowed(true); - defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getCanonicalName()); - defaultConf.setLoadBalancerSheddingEnabled(false); - defaultConf.setLoadManagerServiceUnitStateTableViewClassName(ServiceUnitStateTableViewImpl.class.getName()); - try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) { - // start pulsar3 with ServiceUnitStateTableViewImpl - @Cleanup - var pulsar3 = additionalPulsarTestContext.getPulsarService(); - - String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic); - String lookupResult2 = pulsar1.getAdminClient().lookups().lookupTopic(topic); - String lookupResult3 = pulsar2.getAdminClient().lookups().lookupTopic(topic); - assertEquals(lookupResult1, lookupResult2); - assertEquals(lookupResult1, lookupResult3); - assertEquals(lookupResult1, lookupResultBefore1); - - Optional webServiceUrl1 = - pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl1.isPresent()); - - Optional webServiceUrl2 = - pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl2.isPresent()); - assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString()); - - Optional webServiceUrl3 = - pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl3.isPresent()); - assertEquals(webServiceUrl3.get().toString(), webServiceUrl1.get().toString()); - - assertEquals(webServiceUrl3.get().toString(), webServiceUrlBefore1.get().toString()); - - List pulsarServices = List.of(pulsar1, pulsar2, pulsar3); - for (PulsarService pulsarService : pulsarServices) { - // Test lookup heartbeat namespace's topic - for (PulsarService pulsar : pulsarServices) { - assertLookupHeartbeatOwner(pulsarService, - pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); - } - // Test lookup SLA namespace's topic - for (PulsarService pulsar : pulsarServices) { - assertLookupSLANamespaceOwner(pulsarService, - pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); + assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer().isActive()); + + // === Phase 2: add a 3rd broker using the OTHER table view impl === + // pulsar1/pulsar2 use serviceUnitStateTableViewClassName; pulsar3 deliberately + // uses the other one so the test exercises cross-impl lookups regardless of + // which parametrization we're running. + String otherClassName = + serviceUnitStateTableViewClassName.equals(ServiceUnitStateTableViewImpl.class.getName()) + ? ServiceUnitStateMetadataStoreTableViewImpl.class.getName() + : ServiceUnitStateTableViewImpl.class.getName(); + + ServiceConfiguration crossImplConf = getDefaultConf(); + crossImplConf.setAllowAutoTopicCreation(true); + crossImplConf.setForceDeleteNamespaceAllowed(true); + crossImplConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getCanonicalName()); + crossImplConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + crossImplConf.setLoadManagerServiceUnitStateTableViewClassName(otherClassName); + + try (var crossImplCtx = createAdditionalPulsarTestContext(crossImplConf)) { + var pulsar3 = crossImplCtx.getPulsarService(); + + // All three brokers (across both impls) must agree on topic ownership. + assertEquals(pulsar2.getAdminClient().lookups().lookupTopic(topic), ownershipBefore); + assertEquals(pulsar3.getAdminClient().lookups().lookupTopic(topic), ownershipBefore); + Optional webUrlPulsar3 = pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webUrlPulsar3.isPresent()); + assertEquals(webUrlPulsar3.get().toString(), webUrlBefore.get().toString()); + + // SLA monitor and heartbeat lookups must agree across impls in every direction. + List brokers = List.of(pulsar1, pulsar2, pulsar3); + for (PulsarService viewer : brokers) { + for (PulsarService owner : brokers) { + assertLookupHeartbeatOwner(viewer, owner.getBrokerId(), owner.getBrokerServiceUrl()); + assertLookupSLANamespaceOwner(viewer, owner.getBrokerId(), owner.getBrokerServiceUrl()); } } - // Start broker4 with ServiceUnitStateMetadataStoreTableViewImpl - ServiceConfiguration conf = getDefaultConf(); - conf.setAllowAutoTopicCreation(true); - conf.setForceDeleteNamespaceAllowed(true); - conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getCanonicalName()); - conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); - conf.setLoadManagerServiceUnitStateTableViewClassName( - ServiceUnitStateMetadataStoreTableViewImpl.class.getName()); - try (var additionPulsarTestContext = createAdditionalPulsarTestContext(conf)) { - @Cleanup - var pulsar4 = additionPulsarTestContext.getPulsarService(); - - Set availableCandidates = Sets.newHashSet( - pulsar1.getBrokerServiceUrl(), - pulsar2.getBrokerServiceUrl(), - pulsar3.getBrokerServiceUrl(), - pulsar4.getBrokerServiceUrl()); - String lookupResult4 = pulsar4.getAdminClient().lookups().lookupTopic(topic); - assertTrue(availableCandidates.contains(lookupResult4)); - - String lookupResult5 = pulsar1.getAdminClient().lookups().lookupTopic(topic); - String lookupResult6 = pulsar2.getAdminClient().lookups().lookupTopic(topic); - String lookupResult7 = pulsar3.getAdminClient().lookups().lookupTopic(topic); - assertEquals(lookupResult4, lookupResult5); - assertEquals(lookupResult4, lookupResult6); - assertEquals(lookupResult4, lookupResult7); - assertEquals(lookupResult4, lookupResultBefore1); - - - Pair topicAndBundle2 = - getBundleIsNotOwnByChangeEventTopic("testLoadBalancerServiceUnitTableViewSyncer2"); - String topic2 = topicAndBundle2.getLeft().toString(); - - String lookupResult8 = pulsar1.getAdminClient().lookups().lookupTopic(topic2); - String lookupResult9 = pulsar2.getAdminClient().lookups().lookupTopic(topic2); - String lookupResult10 = pulsar3.getAdminClient().lookups().lookupTopic(topic2); - String lookupResult11 = pulsar4.getAdminClient().lookups().lookupTopic(topic2); - assertEquals(lookupResult9, lookupResult8); - assertEquals(lookupResult10, lookupResult8); - assertEquals(lookupResult11, lookupResult8); - - Set availableWebUrlCandidates = Sets.newHashSet( - pulsar1.getWebServiceAddress(), - pulsar2.getWebServiceAddress(), - pulsar3.getWebServiceAddress(), - pulsar4.getWebServiceAddress()); - - webServiceUrl1 = - pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl1.isPresent()); - assertTrue(availableWebUrlCandidates.contains(webServiceUrl1.get().toString())); - - webServiceUrl2 = - pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl2.isPresent()); - assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString()); - - webServiceUrl3 = - pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl3.isPresent()); - assertTrue(availableWebUrlCandidates.contains(webServiceUrl3.get().toString())); - - var webServiceUrl4 = - pulsar4.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl4.isPresent()); - assertEquals(webServiceUrl4.get().toString(), webServiceUrl1.get().toString()); - assertEquals(webServiceUrl4.get().toString(), webServiceUrlBefore1.get().toString()); - - pulsarServices = List.of(pulsar1, pulsar2, pulsar3, pulsar4); - for (PulsarService pulsarService : pulsarServices) { - // Test lookup heartbeat namespace's topic - for (PulsarService pulsar : pulsarServices) { - assertLookupHeartbeatOwner(pulsarService, - pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); - } - // Test lookup SLA namespace's topic - for (PulsarService pulsar : pulsarServices) { - assertLookupSLANamespaceOwner(pulsarService, - pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); - } - } - // Simulate broker going offline: clean ownerships first (as disableBroker() does), - // then unregister from the broker registry. - var wrapper = (ExtensibleLoadManagerWrapper) pulsar4.getLoadManager().get(); - var loadManager4 = spy((ExtensibleLoadManagerImpl) - FieldUtils.readField(wrapper, "loadManager", true)); - ServiceUnitStateChannel channel4 = (ServiceUnitStateChannel) - FieldUtils.readField(loadManager4, "serviceUnitStateChannel", true); - channel4.cleanOwnerships(); - // Simulate broker going offline by removing it from the broker registry. - // Set state to Closed BEFORE deleting the ZK node to prevent the notification - // handler's session-expiry recovery from auto-re-registering broker4. - // In production, the PulsarService shuts down after unregister(), so the handler - // never fires. In tests, the service stays running, creating a race. - var registry4 = (BrokerRegistryImpl) loadManager4.getBrokerRegistry(); - registry4.state.set(BrokerRegistryImpl.State.Closed); - String brokerZkPath = "/loadbalance/brokers/" + pulsar4.getBrokerId(); - pulsar4.getLocalMetadataStore().delete(brokerZkPath, Optional.empty()).get(); - - NamespaceName slaMonitorNamespace = - getSLAMonitorNamespace(pulsar4.getBrokerId(), pulsar.getConfiguration()); - String slaMonitorTopic = slaMonitorNamespace.getPersistentTopicName("test"); - // Wait for ownership to be reassigned after broker unregistration - String pulsar4BrokerUrl = pulsar4.getBrokerServiceUrl(); - Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { - String lookupResult = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); - assertNotNull(lookupResult); - assertNotEquals(lookupResult, pulsar4BrokerUrl); - }); - String result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); - log.info("{} Namespace is re-owned by {}", slaMonitorTopic, result); - - Producer producer = pulsar.getClient().newProducer(Schema.STRING) - .topic(slaMonitorTopic).create(); - producer.send("t1"); - - // Test re-register broker and check the lookup result - // Reset state to Started to allow re-registration. - registry4.state.set(BrokerRegistryImpl.State.Started); - registry4.registerAsync().get(); - - // Wait for ownership to be reassigned back to pulsar4 after re-registration - Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { - String reRegResult = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); - assertNotNull(reRegResult); - assertEquals(reRegResult, pulsar4.getBrokerServiceUrl()); - }); - result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); - log.info("{} Namespace is re-owned by {}", slaMonitorTopic, result); + // === Phase 3: simulate the cross-impl broker going offline === + // Its SLA namespace must reassign to a remaining broker, and the ownership + // change must propagate through the syncer to brokers using the other impl. + var wrapper3 = (ExtensibleLoadManagerWrapper) pulsar3.getLoadManager().get(); + var loadManager3 = (ExtensibleLoadManagerImpl) + FieldUtils.readField(wrapper3, "loadManager", true); + ServiceUnitStateChannel channel3 = (ServiceUnitStateChannel) + FieldUtils.readField(loadManager3, "serviceUnitStateChannel", true); + channel3.cleanOwnerships(); + // Set state to Closed BEFORE deleting the ZK node to prevent the notification + // handler's session-expiry recovery from auto-re-registering broker3. In + // production the PulsarService shuts down after unregister(), so the handler + // never fires; in tests the service stays running and creates a race. + var registry3 = (BrokerRegistryImpl) loadManager3.getBrokerRegistry(); + registry3.state.set(BrokerRegistryImpl.State.Closed); + pulsar3.getLocalMetadataStore() + .delete("/loadbalance/brokers/" + pulsar3.getBrokerId(), Optional.empty()).get(); + + String slaMonitorTopic = getSLAMonitorNamespace(pulsar3.getBrokerId(), pulsar.getConfiguration()) + .getPersistentTopicName("test"); + String pulsar3BrokerUrl = pulsar3.getBrokerServiceUrl(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + String reassigned = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + assertNotNull(reassigned); + assertNotEquals(reassigned, pulsar3BrokerUrl); + }); - producer.send("t2"); - Producer producer1 = pulsar.getClient().newProducer(Schema.STRING) - .topic(slaMonitorTopic).create(); - producer1.send("t3"); + // Send a message while the topic is owned by the reassigned broker; this must + // remain durable when ownership migrates back below. + @Cleanup + Producer producer = pulsar.getClient().newProducer(Schema.STRING) + .topic(slaMonitorTopic).create(); + producer.send("offline"); + + // === Phase 4: re-register the broker and verify ownership returns === + registry3.state.set(BrokerRegistryImpl.State.Started); + registry3.registerAsync().get(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> + assertEquals(pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic), + pulsar3.getBrokerServiceUrl())); + + // Same producer reconnects to the new owner; a fresh producer also works. + producer.send("after-reconnect"); + @Cleanup + Producer producer2 = pulsar.getClient().newProducer(Schema.STRING) + .topic(slaMonitorTopic).create(); + producer2.send("from-new-producer"); - producer.close(); - producer1.close(); - @Cleanup - Consumer consumer = pulsar.getClient().newConsumer(Schema.STRING) - .topic(slaMonitorTopic) - .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) - .subscriptionName("test") - .subscribe(); - // receive message t1 t2 t3 - assertEquals(consumer.receive().getValue(), "t1"); - assertEquals(consumer.receive().getValue(), "t2"); - assertEquals(consumer.receive().getValue(), "t3"); - } + @Cleanup + Consumer consumer = pulsar.getClient().newConsumer(Schema.STRING) + .topic(slaMonitorTopic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("test") + .subscribe(); + assertEquals(consumer.receive().getValue(), "offline"); + assertEquals(consumer.receive().getValue(), "after-reconnect"); + assertEquals(consumer.receive().getValue(), "from-new-producer"); } + // === Phase 5: disable the syncer and verify it deactivates === pulsar.getAdminClient().brokers() .deleteDynamicConfiguration("loadBalancerServiceUnitTableViewSyncer"); - // Wait for config deletion to propagate before leader transition Awaitility.await().untilAsserted(() -> assertFalse(pulsar1.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled())); - Awaitility.await().untilAsserted(() -> - assertFalse(pulsar2.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled())); - makeSecondaryAsLeader(); - Awaitility.await().atMost(60, TimeUnit.SECONDS) + primaryLoadManager.monitor(); + Awaitility.await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> assertFalse(primaryLoadManager.getServiceUnitStateTableViewSyncer() .isActive())); - Awaitility.await().atMost(60, TimeUnit.SECONDS) - .untilAsserted(() -> assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer() - .isActive())); + assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer().isActive()); } private void assertLookupHeartbeatOwner(PulsarService pulsar, From 235275d1206ddc2bdc006b694023f896724243fd Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 1 May 2026 06:09:54 -0700 Subject: [PATCH 077/201] [fix][test] Fix flaky testGetExcludedBookiesWithIsolationGroups (#25640) (cherry picked from commit 6edd35defd2bc78c943b0542b210d394ec3b8690) --- .../IsolatedBookieEnsemblePlacementPolicyTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java index 936b04386ff7b..0dc996c7d7def 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java @@ -751,6 +751,13 @@ public void testGetExcludedBookiesWithIsolationGroups() throws Exception { NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies); + // Wait for the async cache load triggered by initialize() to complete; otherwise + // getExcludedBookiesWithIsolationGroups returns an empty set when cachedRackConfiguration + // is still null. Same pattern used in testBookieInfoChange (#25473). + Awaitility.await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> + assertNotNull(isolationPolicy.getBookieMappingCache() + .getIfCached(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH))); + /* Test common cases */ MutablePair, Set> groups = new MutablePair<>(); groups.setLeft(Sets.newHashSet(isolationGroup1)); From d0d5f52e87cbee71521b343f7d0aa6e8b0e89b7b Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 1 May 2026 06:11:51 -0700 Subject: [PATCH 078/201] [fix][test] Make NamespacesTest.cleanupAfterMethod tolerant of transient infra failures (#25641) (cherry picked from commit ee7dc689408386f59c311c1d3f16eb35c793afca) --- .../apache/pulsar/broker/admin/NamespacesTest.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java index 6ff2d8a8216b2..16d53b6f96a86 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java @@ -187,12 +187,18 @@ public void cleanup() throws Exception { } @AfterMethod(alwaysRun = true) - public void cleanupAfterMethod() throws Exception{ - // cleanup. + public void cleanupAfterMethod() { + // Best-effort cleanup. Transient infra failures (e.g. ZK session expiry from a + // long-running test or GC pause) can make the admin call fail here; log and + // continue so the test method's actual result is preserved. Set existsNsSetAferSetup = Stream.concat(testLocalNamespaces.stream(), testGlobalNamespaces.stream()) .map(Objects::toString).collect(Collectors.toSet()); - cleanupNamespaceByPredicate(this.testTenant, v -> !existsNsSetAferSetup.contains(v)); - cleanupNamespaceByPredicate(this.testOtherTenant, v -> !existsNsSetAferSetup.contains(v)); + try { + cleanupNamespaceByPredicate(this.testTenant, v -> !existsNsSetAferSetup.contains(v)); + cleanupNamespaceByPredicate(this.testOtherTenant, v -> !existsNsSetAferSetup.contains(v)); + } catch (Exception e) { + log.warn("Failed to clean up namespaces after test method", e); + } } protected void customizeNewPulsarClientBuilder(ClientBuilder clientBuilder) { From a598969d3a82e2ce9f9e5b126aa595075d1fbf05 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 6 May 2026 00:22:09 +0300 Subject: [PATCH 079/201] [improve][test] Set diskUsageThreshold to 0.999 for tests to effectively disable the check (#25677) (cherry picked from commit cc78878528a18a620b4118defc367ca2eac783e0) --- .../org/apache/pulsar/broker/service/SharedPulsarCluster.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedPulsarCluster.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedPulsarCluster.java index a561712b8cad8..3d2ab914ed739 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedPulsarCluster.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedPulsarCluster.java @@ -112,6 +112,8 @@ private void start() throws Exception { bkConf.setNumLongPollWorkerThreads(1); bkConf.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap); bkConf.setLedgerStorageClass("org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage"); + bkConf.setDiskUsageThreshold(0.999F); + bkConf.setDiskUsageWarnThreshold(0.99F); bkCluster = BKCluster.builder() .baseServerConfiguration(bkConf) From 4779f2618595a5929428a07eaad6f19ad1a9fe02 Mon Sep 17 00:00:00 2001 From: Praveen Kumar <122990319+Praveenkumar76@users.noreply.github.com> Date: Wed, 6 May 2026 17:04:05 +0530 Subject: [PATCH 080/201] [fix][test] Fix flaky OneWayReplicatorDeduplicationTest.testDeduplication (#25679) (cherry picked from commit 9e40b5534dd0166de05db4762265d0d7c05cd4a2) --- .../pulsar/broker/service/OneWayReplicatorDeduplicationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorDeduplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorDeduplicationTest.java index 353d2ab89e29e..344279d25373f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorDeduplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorDeduplicationTest.java @@ -114,6 +114,7 @@ protected void setConfigDefaults(ServiceConfiguration config, String clusterName LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { super.setConfigDefaults(config, clusterName, bookkeeperEnsemble, brokerConfigZk); // For check whether deduplication snapshot has done. + config.setBrokerDeduplicationSnapshotIntervalSeconds(1); config.setBrokerDeduplicationEntriesInterval(10); config.setReplicationStartAt("earliest"); // To cover more cases, write more than one ledger. From d88c6f403499c998c0ed36e0236f8a98d8640b3e Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 14 May 2026 15:21:16 +0800 Subject: [PATCH 081/201] [fix][proxy] Close channel on connection failure (#25770) (cherry picked from commit 35cf8f1fab754073105199269044e64de67cc732) --- .../org/apache/pulsar/proxy/server/DirectProxyHandler.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index f707abbc06af6..773cbc5fbcd96 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -194,6 +194,10 @@ protected void initChannel(SocketChannel ch) { // Close the connection if the connection attempt has failed. log.warn("[{}] Establishing connection to {} ({}) failed. Closing inbound channel.", inboundChannel, targetBrokerAddress, brokerHostAndPort, future.cause()); + Channel channel = f.channel(); + if (channel != null) { + channel.close(); + } inboundChannel.close(); } }); From 306fc4e585530fbff33da91ba88d3a6c06017124 Mon Sep 17 00:00:00 2001 From: zzb <48124861+zhaizhibo@users.noreply.github.com> Date: Fri, 15 May 2026 17:07:17 +0800 Subject: [PATCH 082/201] [fix][broker] Fix PulsarService.closeAsync where Condition.signalAll was called without holding a lock (#25777) Co-authored-by: zhaizhibo (cherry picked from commit a20008899b7a89171ae44d40893eb353bb78ab70) --- .../apache/pulsar/broker/PulsarService.java | 9 ++++-- .../pulsar/broker/PulsarServiceCloseTest.java | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 5670579372590..749dfc13b17df 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -756,8 +756,13 @@ public CompletableFuture closeAsync(boolean waitForWebServiceToStop) { } else { LOG.warn("Closed with errors", t); } - state = State.Closed; - isClosedCondition.signalAll(); + mutex.lock(); + try { + state = State.Closed; + isClosedCondition.signalAll(); + } finally { + mutex.unlock(); + } return null; }); return closeFuture; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceCloseTest.java index 683e15c2a02f6..8b3b8ad911063 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceCloseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceCloseTest.java @@ -18,9 +18,12 @@ */ package org.apache.pulsar.broker; +import static org.testng.Assert.fail; import static org.testng.AssertJUnit.assertFalse; import static org.testng.AssertJUnit.assertTrue; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; @@ -73,4 +76,29 @@ public void closeInTimeTest() throws Exception { } } + @Test(timeOut = 60_000) + public void testWaitUntilClosedConcurrentWithCloseAsync() throws Exception { + // Start closeAsync() - it initiates close and returns a future + CompletableFuture closeFuture = pulsar.closeAsync(); + + // Start waitUntilClosed() in a separate thread BEFORE close completes. + // This thread will enter mutex.lock() -> await() and block there, + // relying on signalAll() to be woken up when close finishes. + CompletableFuture waitFuture = CompletableFuture.runAsync(() -> { + try { + pulsar.waitUntilClosed(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + }); + + try { + closeFuture.get(30, TimeUnit.SECONDS); + waitFuture.get(30, TimeUnit.SECONDS); + } catch (Exception e) { + fail("Should not throw exception"); + } + log.info("waitUntilClosed() returned successfully while closeAsync() was in progress"); + } } From aadb3167aaadd25e834ba945d0ed3042b87d230d Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Fri, 15 May 2026 19:39:29 +0800 Subject: [PATCH 083/201] [improve][fn] make built-in connector reload incremental (#25773) Signed-off-by: Dream95 (cherry picked from commit 02cab7abea62740d9a415f16fc45bcb6af963175) --- .../apache/pulsar/common/nar/FileUtils.java | 31 ++++ .../apache/pulsar/common/nar/NarUnpacker.java | 33 +--- .../functions/worker/ConnectorsManager.java | 28 +++- ...ConnectorsManagerReloadConnectorsTest.java | 82 ++++++++++ .../pulsar/functions/utils/io/Connector.java | 26 ++++ .../functions/utils/io/ConnectorUtils.java | 78 +++++++++- .../utils/io/ReloadConnectorsResult.java | 29 ++++ .../utils/io/ConnectorUtilsReloadTest.java | 141 ++++++++++++++++++ 8 files changed, 407 insertions(+), 41 deletions(-) create mode 100644 pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/ConnectorsManagerReloadConnectorsTest.java create mode 100644 pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ReloadConnectorsResult.java create mode 100644 pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/io/ConnectorUtilsReloadTest.java diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java index 7f31367634567..55711850022ed 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java @@ -25,8 +25,11 @@ package org.apache.pulsar.common.nar; import java.io.File; +import java.io.FileInputStream; import java.io.FilenameFilter; import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.Collection; import java.util.zip.ZipEntry; @@ -44,6 +47,34 @@ public class FileUtils { public static final long MILLIS_BETWEEN_ATTEMPTS = 50L; + /** + * Calculates an md5 sum of the specified file. + * + * @param file + * to calculate the md5sum of + * @return the md5sum bytes + * @throws IOException + * if cannot read file + */ + public static byte[] calculateMd5sum(final File file) throws IOException { + try (final FileInputStream inputStream = new FileInputStream(file)) { + // codeql[java/weak-cryptographic-algorithm] - md5 is sufficient for this use case + final MessageDigest md5 = MessageDigest.getInstance("md5"); + + final byte[] buffer = new byte[1024]; + int read = inputStream.read(buffer); + + while (read > -1) { + md5.update(buffer, 0, read); + read = inputStream.read(buffer); + } + + return md5.digest(); + } catch (NoSuchAlgorithmException nsae) { + throw new IllegalArgumentException(nsae); + } + } + public static void ensureDirectoryExistAndCanReadAndWrite(final File dir) throws IOException { if (dir.exists() && !dir.isDirectory()) { throw new IOException(dir.getAbsolutePath() + " is not a directory"); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java index ef802674b421a..2ea35f6418034 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java @@ -25,7 +25,6 @@ import com.google.common.annotations.VisibleForTesting; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -35,8 +34,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.Base64; import java.util.Enumeration; import java.util.concurrent.ConcurrentHashMap; @@ -77,7 +74,7 @@ static File doUnpackNar(final File nar, final File baseWorkingDirectory, Runnabl throw new IOException("Cannot create " + parentDirectory); } } - String md5Sum = Base64.getUrlEncoder().withoutPadding().encodeToString(calculateMd5sum(nar)); + String md5Sum = Base64.getUrlEncoder().withoutPadding().encodeToString(FileUtils.calculateMd5sum(nar)); // ensure that one process can extract the files File lockFile = new File(parentDirectory, "." + md5Sum + ".lock"); // prevent OverlappingFileLockException by ensuring that one thread tries to create a lock in this JVM @@ -171,32 +168,4 @@ private static void makeFile(final InputStream inputStream, final File file) thr } } } - - /** - * Calculates an md5 sum of the specified file. - * - * @param file - * to calculate the md5sum of - * @return the md5sum bytes - * @throws IOException - * if cannot read file - */ - protected static byte[] calculateMd5sum(final File file) throws IOException { - try (final FileInputStream inputStream = new FileInputStream(file)) { - // codeql[java/weak-cryptographic-algorithm] - md5 is sufficient for this use case - final MessageDigest md5 = MessageDigest.getInstance("md5"); - - final byte[] buffer = new byte[1024]; - int read = inputStream.read(buffer); - - while (read > -1) { - md5.update(buffer, 0, read); - read = inputStream.read(buffer); - } - - return md5.digest(); - } catch (NoSuchAlgorithmException nsae) { - throw new IllegalArgumentException(nsae); - } - } } \ No newline at end of file diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java index 19d31d0f63b1d..a8ee6f3ce20c8 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.nio.file.Path; +import java.util.Collection; import java.util.List; import java.util.TreeMap; import java.util.stream.Collectors; @@ -31,6 +32,7 @@ import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.utils.io.ConnectorUtils; +import org.apache.pulsar.functions.utils.io.ReloadConnectorsResult; @Slf4j public class ConnectorsManager implements AutoCloseable { @@ -48,12 +50,16 @@ public ConnectorsManager(WorkerConfig workerConfig) throws IOException { } private static TreeMap createConnectors(WorkerConfig workerConfig) throws IOException { - boolean enableClassloading = workerConfig.getEnableClassloadingOfBuiltinFiles() - || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); + boolean enableClassloading = isEnableClassloading(workerConfig); return ConnectorUtils.searchForConnectors(workerConfig.getConnectorsDirectory(), workerConfig.getNarExtractionDirectory(), enableClassloading); } + private static boolean isEnableClassloading(WorkerConfig workerConfig) { + return workerConfig.getEnableClassloadingOfBuiltinFiles() + || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); + } + @VisibleForTesting public void addConnector(String connectorType, Connector connector) { connectors.put(connectorType, connector); @@ -89,9 +95,13 @@ public Path getSinkArchive(String sinkType) { } public void reloadConnectors(WorkerConfig workerConfig) throws IOException { - TreeMap oldConnectors = connectors; - this.connectors = createConnectors(workerConfig); - closeConnectors(oldConnectors); + ReloadConnectorsResult reload = ConnectorUtils.reloadConnectors( + this.connectors, + workerConfig.getConnectorsDirectory(), + workerConfig.getNarExtractionDirectory(), + isEnableClassloading(workerConfig)); + this.connectors = reload.connectors(); + closeConnectors(reload.connectorsToClose()); } @Override @@ -99,14 +109,18 @@ public void close() { closeConnectors(connectors); } - private void closeConnectors(TreeMap connectorMap) { - connectorMap.values().forEach(connector -> { + private void closeConnectors(Collection connectors) { + connectors.forEach(connector -> { try { connector.close(); } catch (Exception e) { log.warn("Failed to close connector", e); } }); + } + + private void closeConnectors(TreeMap connectorMap) { + closeConnectors(connectorMap.values()); connectorMap.clear(); } diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/ConnectorsManagerReloadConnectorsTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/ConnectorsManagerReloadConnectorsTest.java new file mode 100644 index 0000000000000..82c280f97ad65 --- /dev/null +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/ConnectorsManagerReloadConnectorsTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.worker; + +import static org.testng.Assert.assertSame; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.apache.pulsar.common.io.ConnectorDefinition; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.functions.utils.io.Connector; +import org.testng.annotations.Test; + +/** + * Tests {@link ConnectorsManager#reloadConnectors(WorkerConfig)} for incremental reload behavior, + * ensuring unchanged connectors are reused instead of being recreated. + */ +public class ConnectorsManagerReloadConnectorsTest { + + private static void writeMinimalNar(Path narPath, ConnectorDefinition def) throws IOException { + byte[] yaml = ObjectMapperFactory.getYamlMapper().getObjectMapper().writeValueAsBytes(def); + try (OutputStream os = Files.newOutputStream(narPath); + ZipOutputStream zos = new ZipOutputStream(os)) { + ZipEntry entry = new ZipEntry("META-INF/services/pulsar-io.yaml"); + zos.putNextEntry(entry); + zos.write(yaml); + zos.closeEntry(); + } + } + + private static ConnectorDefinition sampleDefinition(String name) { + ConnectorDefinition def = new ConnectorDefinition(); + def.setName(name); + def.setSinkClass("org.example.Sink"); + def.setSourceClass("org.example.Source"); + return def; + } + + @Test + public void reloadWhenNarUnchangedReusesSameConnectorInstance() throws Exception { + Path dir = Files.createTempDirectory("mgr-conn-reload-"); + Path nar = dir.resolve("c1.nar"); + writeMinimalNar(nar, sampleDefinition("c-one")); + + WorkerConfig workerConfig = new WorkerConfig(); + workerConfig.setConnectorsDirectory(dir.toString()); + workerConfig.setNarExtractionDirectory(NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR); + workerConfig.setEnableClassloadingOfBuiltinFiles(false); + + try (ConnectorsManager manager = new ConnectorsManager(workerConfig)) { + Connector before = manager.getConnector("c-one"); + before.getConnectorFunctionPackage(); + + manager.reloadConnectors(workerConfig); + + Connector after = manager.getConnector("c-one"); + assertSame(after, before); + before.getConnectorFunctionPackage(); + } + } + +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java index 5fcc22747c516..bff477b40cda3 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java @@ -28,6 +28,8 @@ public class Connector implements AutoCloseable { private final Path archivePath; + /** MD5 hex of archive file contents; empty when {@link #archivePath} is null (test doubles). */ + private final String archiveMd5Hex; private final String narExtractionDirectory; private final boolean enableClassloading; private ValidatableFunctionPackage connectorFunctionPackage; @@ -38,16 +40,40 @@ public class Connector implements AutoCloseable { public Connector(Path archivePath, ConnectorDefinition connectorDefinition, String narExtractionDirectory, boolean enableClassloading) { + this(archivePath, connectorDefinition, narExtractionDirectory, enableClassloading, null); + } + + /** + * @param precomputedArchiveMd5Hex MD5 hex of {@code archivePath} contents; if null and path is non-null, + * the hash is computed once at construction time. + */ + public Connector(Path archivePath, ConnectorDefinition connectorDefinition, String narExtractionDirectory, + boolean enableClassloading, String precomputedArchiveMd5Hex) { this.archivePath = archivePath; this.connectorDefinition = connectorDefinition; this.narExtractionDirectory = narExtractionDirectory; this.enableClassloading = enableClassloading; + if (archivePath != null) { + try { + this.archiveMd5Hex = precomputedArchiveMd5Hex != null + ? precomputedArchiveMd5Hex + : ConnectorUtils.computeArchiveMd5Hex(archivePath); + } catch (java.io.IOException e) { + throw new java.io.UncheckedIOException(e); + } + } else { + this.archiveMd5Hex = ""; + } } public Path getArchivePath() { return archivePath; } + public String getArchiveMd5Hex() { + return archiveMd5Hex; + } + public synchronized ValidatableFunctionPackage getConnectorFunctionPackage() { checkState(); if (connectorFunctionPackage == null) { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java index 71cab749ba06c..d08a494288458 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java @@ -24,7 +24,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HexFormat; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -39,6 +41,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.io.ConfigFieldDefinition; import org.apache.pulsar.common.io.ConnectorDefinition; +import org.apache.pulsar.common.nar.FileUtils; import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.functions.utils.Exceptions; import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; @@ -52,7 +55,16 @@ @Slf4j public class ConnectorUtils { - private static final String PULSAR_IO_SERVICE_NAME = "pulsar-io.yaml"; + /** + * Computes MD5 digest of a file as lower-case hex (for connector archive identity on reload). + */ + public static String computeArchiveMd5Hex(Path path) throws IOException { + return calculateMd5Hex(path.toAbsolutePath().normalize().toFile()); + } + + private static String calculateMd5Hex(File file) throws IOException { + return HexFormat.of().formatHex(FileUtils.calculateMd5sum(file)); + } /** * Extract the Pulsar IO Source class from a connector archive. @@ -171,7 +183,8 @@ public static TreeMap searchForConnectors(String connectorsDi try { ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(archive.toFile()); log.info("Found connector {} from {}", cntDef, archive); - Connector connector = new Connector(archive, cntDef, narExtractionDirectory, enableClassloading); + Connector connector = new Connector(archive, cntDef, narExtractionDirectory, + enableClassloading); connectors.put(cntDef.getName(), connector); } catch (Throwable t) { log.warn("Failed to load connector from {}", archive, t); @@ -180,4 +193,65 @@ public static TreeMap searchForConnectors(String connectorsDi } return connectors; } + + /** + * Reloads connectors from disk against {@code previous}, reusing {@link Connector} instances when path and + * archive MD5 are unchanged (keeps class loaders open). New or changed archives get new instances. + *

+ * {@link ReloadConnectorsResult#connectorsToClose()} lists connectors evicted from the active set (replaced or + * no longer present on disk); the caller must {@link Connector#close()} each (typically via + * {@code ConnectorsManager}). + * + * @param previous connectors from the previous scan (may be empty, never null) + * @param connectorsDirectory same semantics as {@link #searchForConnectors} + * @param narExtractionDirectory same semantics as {@link #searchForConnectors} + * @param enableClassloading same semantics as {@link #searchForConnectors} + * @return new map keyed by connector name (reused values are identical instances from {@code previous}) and + * connectors the caller should close + */ + public static ReloadConnectorsResult reloadConnectors( + TreeMap previous, + String connectorsDirectory, + String narExtractionDirectory, + boolean enableClassloading) throws IOException { + + TreeMap remaining = new TreeMap<>(previous); + TreeMap next = new TreeMap<>(); + List toClose = new ArrayList<>(); + + Path dir = Paths.get(connectorsDirectory).toAbsolutePath().normalize(); + if (!dir.toFile().exists()) { + toClose.addAll(remaining.values()); + return new ReloadConnectorsResult(next, toClose); + } + + try (DirectoryStream stream = Files.newDirectoryStream(dir, "*.nar")) { + for (Path archive : stream) { + try { + ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(archive.toFile()); + String name = cntDef.getName(); + String md5Hex = computeArchiveMd5Hex(archive); + Connector prev = remaining.remove(name); + if (prev != null + && prev.getArchivePath() != null + && archive.equals(prev.getArchivePath()) + && md5Hex.equals(prev.getArchiveMd5Hex())) { + next.put(name, prev); + } else { + if (prev != null) { + log.info("Reloading changed connector name={} archive={} previousArchive={}", name, archive, + prev.getArchivePath()); + toClose.add(prev); + } + next.put(name, new Connector(archive, cntDef, narExtractionDirectory, enableClassloading, + md5Hex)); + } + } catch (Throwable t) { + log.warn("Failed to load connector archive={}", archive, t); + } + } + } + toClose.addAll(remaining.values()); + return new ReloadConnectorsResult(next, toClose); + } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ReloadConnectorsResult.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ReloadConnectorsResult.java new file mode 100644 index 0000000000000..cdfab35692cc2 --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ReloadConnectorsResult.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils.io; + +import java.util.List; +import java.util.TreeMap; + +/** + * Result of {@link ConnectorUtils#reloadConnectors}: the new connector map and connectors evicted from the active set + * that the caller must close. + */ +public record ReloadConnectorsResult(TreeMap connectors, List connectorsToClose) { +} diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/io/ConnectorUtilsReloadTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/io/ConnectorUtilsReloadTest.java new file mode 100644 index 0000000000000..07b6459369a2c --- /dev/null +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/io/ConnectorUtilsReloadTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils.io; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotSame; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.TreeMap; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.apache.pulsar.common.io.ConnectorDefinition; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.testng.annotations.Test; + +@Test +public class ConnectorUtilsReloadTest { + + private static void closeEvicted(ReloadConnectorsResult reload) throws Exception { + for (Connector c : reload.connectorsToClose()) { + c.close(); + } + } + + private static void writeMinimalNar(Path narPath, ConnectorDefinition def) throws IOException { + byte[] yaml = ObjectMapperFactory.getYamlMapper().getObjectMapper().writeValueAsBytes(def); + try (OutputStream os = Files.newOutputStream(narPath); + ZipOutputStream zos = new ZipOutputStream(os)) { + ZipEntry entry = new ZipEntry("META-INF/services/pulsar-io.yaml"); + zos.putNextEntry(entry); + zos.write(yaml); + zos.closeEntry(); + } + } + + private static ConnectorDefinition sampleDefinition(String name) { + ConnectorDefinition def = new ConnectorDefinition(); + def.setName(name); + def.setSinkClass("org.example.Sink"); + def.setSourceClass("org.example.Source"); + return def; + } + + /** + * Historical {@code ConnectorsManager} reload replaced the whole map and closed every prior + * {@link Connector}, even when NAR files were unchanged. A caller keeping a reference to the + * pre-reload connector would then hit {@link IllegalStateException} on lazy use. + *

+ * Incremental reload must evict nothing, reuse the same instance, and leave that instance usable + * after the caller closes only {@link ReloadConnectorsResult#connectorsToClose()}. + */ + @Test + public void reloadUnchangedNarEvictsNothingAndKeepsSameConnectorUsable() throws Exception { + Path dir = Files.createTempDirectory("conn-reload-"); + Path nar = dir.resolve("c1.nar"); + writeMinimalNar(nar, sampleDefinition("c-one")); + + TreeMap first = + ConnectorUtils.searchForConnectors(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + Connector c1 = first.get("c-one"); + c1.getConnectorFunctionPackage(); + + ReloadConnectorsResult reload = ConnectorUtils.reloadConnectors( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + assertTrue(reload.connectorsToClose().isEmpty()); + closeEvicted(reload); + TreeMap second = reload.connectors(); + + assertSame(second.get("c-one"), c1); + c1.getConnectorFunctionPackage(); + } + + @Test + public void reloadReopensConnectorWhenNarContentChanges() throws Exception { + Path dir = Files.createTempDirectory("conn-reload-"); + Path nar = dir.resolve("c1.nar"); + writeMinimalNar(nar, sampleDefinition("c-one")); + + TreeMap first = + ConnectorUtils.searchForConnectors(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + Connector before = first.get("c-one"); + + ConnectorDefinition updated = sampleDefinition("c-one"); + updated.setDescription("changed"); + writeMinimalNar(nar, updated); + + ReloadConnectorsResult reload = ConnectorUtils.reloadConnectors( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + closeEvicted(reload); + TreeMap second = reload.connectors(); + + assertNotSame(second.get("c-one"), before); + assertThrows(IllegalStateException.class, before::getConnectorFunctionPackage); + } + + @Test + public void reloadClosesConnectorsRemovedFromDirectory() throws Exception { + Path dir = Files.createTempDirectory("conn-reload-"); + Path nar1 = dir.resolve("a.nar"); + Path nar2 = dir.resolve("b.nar"); + writeMinimalNar(nar1, sampleDefinition("conn-a")); + writeMinimalNar(nar2, sampleDefinition("conn-b")); + + TreeMap first = + ConnectorUtils.searchForConnectors(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + Connector removed = first.get("conn-b"); + Files.delete(nar2); + + ReloadConnectorsResult reload = ConnectorUtils.reloadConnectors( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + closeEvicted(reload); + TreeMap second = reload.connectors(); + + assertEquals(second.size(), 1); + assertSame(second.get("conn-a"), first.get("conn-a")); + assertThrows(IllegalStateException.class, removed::getConnectorFunctionPackage); + } + +} From eee5d9d3ec11b95a00596ec1a561e32cd5ce29da Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 15 May 2026 16:00:53 +0300 Subject: [PATCH 084/201] [improve][build] Upgrade org.apache.kerby:kerb-simplekdc from 1.1.1 to 2.1.1 (#25785) (cherry picked from commit 20736e70fed3b1427af43ab339aad63ee6c0294f) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ac3035737fe95..7a304e9625035 100644 --- a/pom.xml +++ b/pom.xml @@ -326,7 +326,7 @@ flexible messaging model and an intuitive client API. 3.7.0 2.2 5.4.0 - 1.1.1 + 2.1.1 7.7.1 5.19.0 3.25.0-GA From 18dd778f18b6da50bcb6427d3fbbdca263c5043d Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Sat, 16 May 2026 16:58:59 +0800 Subject: [PATCH 085/201] [refactor][fn] Use Map instead of TreeMap for connector/function API types (#25790) Signed-off-by: Dream95 (cherry picked from commit eba873d9c3926b806f5462af834867ccb692fdfb) --- .../org/apache/pulsar/functions/LocalRunner.java | 11 +++++------ .../pulsar/functions/worker/ConnectorsManager.java | 7 ++++--- .../pulsar/functions/worker/FunctionsManager.java | 9 +++++---- .../functions/utils/functions/FunctionUtils.java | 3 ++- .../pulsar/functions/utils/io/ConnectorUtils.java | 4 ++-- .../functions/utils/io/ReloadConnectorsResult.java | 4 ++-- .../utils/io/ConnectorUtilsReloadTest.java | 14 +++++++------- 7 files changed, 27 insertions(+), 25 deletions(-) diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java index 1e2d4e0b21b32..bfb3eab6ff370 100644 --- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java +++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java @@ -38,7 +38,6 @@ import java.util.Optional; import java.util.Timer; import java.util.TimerTask; -import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -724,7 +723,7 @@ private ClassLoader isBuiltIn(String component, ComponentType componentType) private ClassLoader isBuiltInFunction(String functionType) throws IOException { // Validate the connector type from the locally available connectors - TreeMap functions = getFunctions(); + Map functions = getFunctions(); String functionName = functionType.replaceFirst("^builtin://", ""); FunctionArchive function = functions.get(functionName); @@ -738,7 +737,7 @@ private ClassLoader isBuiltInFunction(String functionType) throws IOException { private ClassLoader isBuiltInSource(String sourceType) throws IOException { // Validate the connector type from the locally available connectors - TreeMap connectors = getConnectors(); + Map connectors = getConnectors(); String source = sourceType.replaceFirst("^builtin://", ""); Connector connector = connectors.get(source); @@ -752,7 +751,7 @@ private ClassLoader isBuiltInSource(String sourceType) throws IOException { private ClassLoader isBuiltInSink(String sinkType) throws IOException { // Validate the connector type from the locally available connectors - TreeMap connectors = getConnectors(); + Map connectors = getConnectors(); String sink = sinkType.replaceFirst("^builtin://", ""); Connector connector = connectors.get(sink); @@ -764,11 +763,11 @@ private ClassLoader isBuiltInSink(String sinkType) throws IOException { } } - private TreeMap getFunctions() throws IOException { + private Map getFunctions() throws IOException { return FunctionUtils.searchForFunctions(functionsDir, narExtractionDirectory, true); } - private TreeMap getConnectors() throws IOException { + private Map getConnectors() throws IOException { return ConnectorUtils.searchForConnectors(connectorsDir, narExtractionDirectory, true); } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java index a8ee6f3ce20c8..74435f99bcf3f 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java @@ -23,6 +23,7 @@ import java.nio.file.Path; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.TreeMap; import java.util.stream.Collectors; import lombok.Getter; @@ -38,7 +39,7 @@ public class ConnectorsManager implements AutoCloseable { @Getter - private volatile TreeMap connectors; + private volatile Map connectors; @VisibleForTesting public ConnectorsManager() { @@ -49,7 +50,7 @@ public ConnectorsManager(WorkerConfig workerConfig) throws IOException { this.connectors = createConnectors(workerConfig); } - private static TreeMap createConnectors(WorkerConfig workerConfig) throws IOException { + private static Map createConnectors(WorkerConfig workerConfig) throws IOException { boolean enableClassloading = isEnableClassloading(workerConfig); return ConnectorUtils.searchForConnectors(workerConfig.getConnectorsDirectory(), workerConfig.getNarExtractionDirectory(), enableClassloading); @@ -119,7 +120,7 @@ private void closeConnectors(Collection connectors) { }); } - private void closeConnectors(TreeMap connectorMap) { + private void closeConnectors(Map connectorMap) { closeConnectors(connectorMap.values()); connectorMap.clear(); } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java index 5ab7ff7221abb..cdd772495023e 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.List; +import java.util.Map; import java.util.TreeMap; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -32,7 +33,7 @@ @Slf4j public class FunctionsManager implements AutoCloseable { - private TreeMap functions; + private Map functions; @VisibleForTesting public FunctionsManager() { @@ -61,12 +62,12 @@ public List getFunctionDefinitions() { } public void reloadFunctions(WorkerConfig workerConfig) throws IOException { - TreeMap oldFunctions = functions; + Map oldFunctions = functions; this.functions = createFunctions(workerConfig); closeFunctions(oldFunctions); } - private static TreeMap createFunctions(WorkerConfig workerConfig) throws IOException { + private static Map createFunctions(WorkerConfig workerConfig) throws IOException { boolean enableClassloading = workerConfig.getEnableClassloadingOfBuiltinFiles() || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); return FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory(), @@ -79,7 +80,7 @@ public void close() { closeFunctions(functions); } - private void closeFunctions(TreeMap functionMap) { + private void closeFunctions(Map functionMap) { functionMap.values().forEach(functionArchive -> { try { functionArchive.close(); diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java index 24f801f3da26f..f4d45edf36301 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java @@ -25,6 +25,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Map; import java.util.TreeMap; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -75,7 +76,7 @@ public static T getPulsarIOServiceConfig(NarClassLoader narClassLoader, Clas .readValue(narClassLoader.getServiceDefinition(PULSAR_IO_SERVICE_NAME), valueType); } - public static TreeMap searchForFunctions(String functionsDirectory, + public static Map searchForFunctions(String functionsDirectory, String narExtractionDirectory, boolean enableClassloading) throws IOException { Path path = Paths.get(functionsDirectory).toAbsolutePath().normalize(); diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java index d08a494288458..0ff7e9427b3fd 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java @@ -165,7 +165,7 @@ private static List getAllFields(TypeDefinition type) { return fields; } - public static TreeMap searchForConnectors(String connectorsDirectory, + public static Map searchForConnectors(String connectorsDirectory, String narExtractionDirectory, boolean enableClassloading) throws IOException { Path path = Paths.get(connectorsDirectory).toAbsolutePath().normalize(); @@ -210,7 +210,7 @@ public static TreeMap searchForConnectors(String connectorsDi * connectors the caller should close */ public static ReloadConnectorsResult reloadConnectors( - TreeMap previous, + Map previous, String connectorsDirectory, String narExtractionDirectory, boolean enableClassloading) throws IOException { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ReloadConnectorsResult.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ReloadConnectorsResult.java index cdfab35692cc2..b46aa38feadbd 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ReloadConnectorsResult.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ReloadConnectorsResult.java @@ -19,11 +19,11 @@ package org.apache.pulsar.functions.utils.io; import java.util.List; -import java.util.TreeMap; +import java.util.Map; /** * Result of {@link ConnectorUtils#reloadConnectors}: the new connector map and connectors evicted from the active set * that the caller must close. */ -public record ReloadConnectorsResult(TreeMap connectors, List connectorsToClose) { +public record ReloadConnectorsResult(Map connectors, List connectorsToClose) { } diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/io/ConnectorUtilsReloadTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/io/ConnectorUtilsReloadTest.java index 07b6459369a2c..0e05912d8c960 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/io/ConnectorUtilsReloadTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/io/ConnectorUtilsReloadTest.java @@ -27,7 +27,7 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.TreeMap; +import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.apache.pulsar.common.io.ConnectorDefinition; @@ -77,7 +77,7 @@ public void reloadUnchangedNarEvictsNothingAndKeepsSameConnectorUsable() throws Path nar = dir.resolve("c1.nar"); writeMinimalNar(nar, sampleDefinition("c-one")); - TreeMap first = + Map first = ConnectorUtils.searchForConnectors(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); Connector c1 = first.get("c-one"); c1.getConnectorFunctionPackage(); @@ -86,7 +86,7 @@ public void reloadUnchangedNarEvictsNothingAndKeepsSameConnectorUsable() throws first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); assertTrue(reload.connectorsToClose().isEmpty()); closeEvicted(reload); - TreeMap second = reload.connectors(); + Map second = reload.connectors(); assertSame(second.get("c-one"), c1); c1.getConnectorFunctionPackage(); @@ -98,7 +98,7 @@ public void reloadReopensConnectorWhenNarContentChanges() throws Exception { Path nar = dir.resolve("c1.nar"); writeMinimalNar(nar, sampleDefinition("c-one")); - TreeMap first = + Map first = ConnectorUtils.searchForConnectors(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); Connector before = first.get("c-one"); @@ -109,7 +109,7 @@ public void reloadReopensConnectorWhenNarContentChanges() throws Exception { ReloadConnectorsResult reload = ConnectorUtils.reloadConnectors( first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); closeEvicted(reload); - TreeMap second = reload.connectors(); + Map second = reload.connectors(); assertNotSame(second.get("c-one"), before); assertThrows(IllegalStateException.class, before::getConnectorFunctionPackage); @@ -123,7 +123,7 @@ public void reloadClosesConnectorsRemovedFromDirectory() throws Exception { writeMinimalNar(nar1, sampleDefinition("conn-a")); writeMinimalNar(nar2, sampleDefinition("conn-b")); - TreeMap first = + Map first = ConnectorUtils.searchForConnectors(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); Connector removed = first.get("conn-b"); Files.delete(nar2); @@ -131,7 +131,7 @@ public void reloadClosesConnectorsRemovedFromDirectory() throws Exception { ReloadConnectorsResult reload = ConnectorUtils.reloadConnectors( first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); closeEvicted(reload); - TreeMap second = reload.connectors(); + Map second = reload.connectors(); assertEquals(second.size(), 1); assertSame(second.get("conn-a"), first.get("conn-a")); From 1042e4f818ff41d6e4f3d64b3811fa0027f06435 Mon Sep 17 00:00:00 2001 From: sinan liu Date: Mon, 18 May 2026 09:49:50 +0800 Subject: [PATCH 086/201] [improve][broker] Prevent stale replicator pending reads after termination (#25767) (cherry picked from commit 8f9f5b49d631e235e86d79e48a63722e74db4413) --- .../persistent/PersistentReplicator.java | 21 +++++-- .../PersistentReplicatorInflightTaskTest.java | 60 +++++++++++++++++++ 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 4202f812a4aac..b125bd0306bdd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -501,7 +501,7 @@ public CompletableFuture getFuture() { @Override public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - InFlightTask inFlightTask = (InFlightTask) ctx; + completeFailedReadTask(ctx); if (state != Started) { log.info("[{}] Replicator was disconnected while reading entries." + " Stop reading. Replicator state: {}", @@ -522,11 +522,9 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { terminate(); return; } else if (!(exception instanceof TooManyRequestsException)) { - inFlightTask.setEntries(Collections.emptyList()); log.error("[{}] Error reading entries at {}. Retrying to read in {}s. ({})", replicatorId, ctx, waitTimeMillis / 1000.0, exception.getMessage(), exception); } else { - inFlightTask.setEntries(Collections.emptyList()); log.debug("[{}] Throttled by bookies while reading at {}. Retrying to read in {}s. ({})", replicatorId, ctx, waitTimeMillis / 1000.0, exception.getMessage(), exception); @@ -535,6 +533,21 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { brokerService.executor().schedule(this::readMoreEntries, waitTimeMillis, TimeUnit.MILLISECONDS); } + private void completeFailedReadTask(Object ctx) { + if (!(ctx instanceof InFlightTask)) { + log.error("[{}] Unexpected read entries failed context {}", replicatorId, ctx); + return; + } + + InFlightTask inFlightTask = (InFlightTask) ctx; + if (inFlightTask.entries == null) { + inFlightTask.setEntries(Collections.emptyList()); + } else { + log.error("[{}] Unexpected completed in-flight task in read entries failed callback. inFlightTask={}", + replicatorId, inFlightTask); + } + } + public CompletableFuture clearBacklog() { CompletableFuture future = new CompletableFuture<>(); @@ -990,4 +1003,4 @@ protected boolean hasPendingRead() { String getReplicatorId() { return replicatorId; } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java index e8a7a11a2db13..c825d49caa62e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java @@ -28,17 +28,25 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerTest; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.service.BrokerServiceInternalMethodInvoker; import org.apache.pulsar.broker.service.OneWayReplicatorTestBase; import org.apache.pulsar.broker.service.persistent.PersistentReplicator.InFlightTask; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; +import org.awaitility.Awaitility; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.testng.Assert; @@ -107,6 +115,58 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { assertTrue(counter.get() <= 1); } + @Test + public void testReadEntriesFailedCompletesInFlightTaskAfterReplicatorTerminated() throws Exception { + String topicName = BrokerTestUtil.newUniqueName("persistent://" + nonReplicatedNamespace + "/tp_"); + CountDownLatch readStarted = new CountDownLatch(1); + CountDownLatch failRead = new CountDownLatch(1); + Producer producer = null; + try { + admin1.topics().createNonPartitionedTopic(topicName); + admin2.topics().createNonPartitionedTopic(topicName); + producer = client1.newProducer(Schema.STRING).topic(topicName).create(); + producer.send("msg"); + + PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false) + .join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger(); + ManagedLedgerTest.makeReadEntryProbFail(ml, () -> { + readStarted.countDown(); + try { + if (!failRead.await(30, TimeUnit.SECONDS)) { + return new ManagedLedgerException("Timed out waiting to fail read entries"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return new ManagedLedgerException(e); + } + return new ManagedLedgerException.TooManyRequestsException("mocked read failure"); + }); + + pulsar1.getConfig().setReplicationStartAt("earliest"); + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); + assertTrue(readStarted.await(30, TimeUnit.SECONDS)); + + PersistentReplicator replicator = (PersistentReplicator) topic.getReplicators().get(cluster2); + Assert.assertNotNull(replicator, "Replicator should not be null"); + Assert.assertTrue(replicator.hasPendingRead()); + + replicator.terminate(); + Assert.assertTrue(replicator.getState() != AbstractReplicator.State.Started); + failRead.countDown(); + + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> + Assert.assertFalse(replicator.hasPendingRead())); + } finally { + failRead.countDown(); + if (producer != null) { + producer.close(); + } + admin1.topics().delete(topicName, true); + admin2.topics().delete(topicName, true); + } + } + @Test public void testCreateOrRecycleInFlightTaskIntoQueue() throws Exception { log.info("Starting testCreateOrRecycleInFlightTaskIntoQueue"); From 814668f58325c91c1ea1de82144539e0306bce74 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 18 May 2026 16:37:37 +0800 Subject: [PATCH 087/201] [fix][broker] Use effective offload policies for extra configs (#25781) (cherry picked from commit d22970d58e793641ce55373d6eb6ca6225edd006) --- .../broker/admin/impl/NamespacesBase.java | 6 ++- .../broker/admin/AdminApiOffloadTest.java | 46 +++++++++++++++++++ .../policies/data/OffloadPoliciesImpl.java | 27 ++++++++++- .../policies/data/OffloadPoliciesTest.java | 22 ++++++++- 4 files changed, 98 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 1e99aa0716ba5..7b5a4fbb00554 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2929,8 +2929,12 @@ protected void internalScanOffloadedLedgers(OffloaderObjectsScannerUtils.Scanner validateNamespacePolicyOperation(namespaceName, PolicyName.OFFLOAD, PolicyOperation.READ); Policies policies = getNamespacePolicies(namespaceName); + OffloadPoliciesImpl nsLevelOffloadPolicies = (OffloadPoliciesImpl) policies.offload_policies; + OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration(null, + OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies), + pulsar().getConfig().getProperties()); LedgerOffloader managedLedgerOffloader = pulsar() - .getManagedLedgerOffloader(namespaceName, (OffloadPoliciesImpl) policies.offload_policies); + .getManagedLedgerOffloader(namespaceName, offloadPolicies); String localClusterName = pulsar().getConfiguration().getClusterName(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java index cadfd759c15e7..18c2861354bc3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java @@ -37,8 +37,12 @@ */ package org.apache.pulsar.broker.admin; +import static org.apache.pulsar.common.policies.data.OffloadPoliciesImpl.EXTRA_CONFIG_PREFIX; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -56,6 +60,9 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.core.Response; import org.apache.bookkeeper.mledger.LedgerOffloader; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; @@ -78,6 +85,7 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.awaitility.Awaitility; +import org.mockito.ArgumentCaptor; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -118,6 +126,44 @@ public void cleanup() throws Exception { super.internalCleanup(); } + @Test + public void testScanOffloadedLedgersUsesMergedNamespaceOffloadPolicies() throws Exception { + pulsar.getConfig().getProperties().setProperty("managedLedgerOffloadDriver", "aws-s3"); + pulsar.getConfig().getProperties().setProperty("managedLedgerOffloadBucket", "broker-bucket"); + pulsar.getConfig().getProperties().setProperty("managedLedgerOffloadRegion", "us-east-1"); + pulsar.getConfig().getProperties().setProperty( + EXTRA_CONFIG_PREFIX + "tieredStorageBucketPrefix", "broker-prefix"); + + admin.namespaces().setOffloadThreshold(myNamespace, 1024L); + + LedgerOffloader offloader = mock(LedgerOffloader.class); + doReturn(Map.of()).when(offloader).getOffloadDriverMetadata(); + doNothing().when(offloader).scanLedgers(any(), anyMap()); + ArgumentCaptor offloadPoliciesCaptor = ArgumentCaptor.forClass(OffloadPoliciesImpl.class); + doReturn(offloader).when(pulsar).getManagedLedgerOffloader(eq(NamespaceName.get(myNamespace)), + offloadPoliciesCaptor.capture()); + + Client client = ClientBuilder.newClient(); + try { + try (Response response = client.target(pulsar.getWebServiceAddress() + + "/admin/v2/namespaces/" + myNamespace + "/scanOffloadedLedgers").request().get()) { + assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); + response.readEntity(String.class); + } + } finally { + client.close(); + } + + OffloadPoliciesImpl offloadPolicies = offloadPoliciesCaptor.getValue(); + assertEquals(offloadPolicies.getManagedLedgerOffloadDriver(), "aws-s3"); + assertEquals(offloadPolicies.getManagedLedgerOffloadBucket(), "broker-bucket"); + assertEquals(offloadPolicies.getManagedLedgerOffloadRegion(), "us-east-1"); + assertEquals(offloadPolicies.getManagedLedgerOffloadThresholdInBytes(), Long.valueOf(1024L)); + assertEquals(offloadPolicies.getManagedLedgerExtraConfigurations(), + Map.of("tieredStorageBucketPrefix", "broker-prefix")); + verify(offloader).scanLedgers(any(), anyMap()); + } + private void testOffload(String topicName, String mlName) throws Exception { LedgerOffloader offloader = mock(LedgerOffloader.class); when(offloader.getOffloadDriverName()).thenReturn("mock"); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java index 93f090c9ddf83..c5579d1008c78 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java @@ -428,7 +428,10 @@ public static OffloadPoliciesImpl mergeConfiguration(OffloadPoliciesImpl topicLe OffloadPoliciesImpl offloadPolicies = new OffloadPoliciesImpl(); for (Field field : CONFIGURATION_FIELDS) { Object object; - if (topicLevelPolicies != null && field.get(topicLevelPolicies) != null) { + if (field.getName().equals("managedLedgerExtraConfigurations")) { + object = mergeManagedLedgerExtraConfigurations(topicLevelPolicies, nsLevelPolicies, + brokerProperties); + } else if (topicLevelPolicies != null && field.get(topicLevelPolicies) != null) { object = field.get(topicLevelPolicies); } else if (nsLevelPolicies != null && field.get(nsLevelPolicies) != null) { object = field.get(nsLevelPolicies); @@ -453,6 +456,28 @@ public static OffloadPoliciesImpl mergeConfiguration(OffloadPoliciesImpl topicLe } } + private static Map mergeManagedLedgerExtraConfigurations(OffloadPoliciesImpl topicLevelPolicies, + OffloadPoliciesImpl nsLevelPolicies, + Properties brokerProperties) { + Map mergedExtraConfigurations = new HashMap<>(); + putAllExtraConfigurations(mergedExtraConfigurations, getExtraConfigurations(brokerProperties)); + if (nsLevelPolicies != null) { + putAllExtraConfigurations(mergedExtraConfigurations, + nsLevelPolicies.getManagedLedgerExtraConfigurations()); + } + if (topicLevelPolicies != null) { + putAllExtraConfigurations(mergedExtraConfigurations, + topicLevelPolicies.getManagedLedgerExtraConfigurations()); + } + return mergedExtraConfigurations.isEmpty() ? null : mergedExtraConfigurations; + } + + private static void putAllExtraConfigurations(Map target, Map extraConfigurations) { + if (extraConfigurations != null && !extraConfigurations.isEmpty()) { + target.putAll(extraConfigurations); + } + } + /** * Make configurations of the OffloadPolicies compatible with the config file. * diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java index 0662be3f05435..b3b02e46a58bc 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java @@ -353,16 +353,36 @@ public void higherLevelExtraConfigOverridesBrokerExtraConfigMergeTest() { Properties brokerProperties = new Properties(); brokerProperties.setProperty("managedLedgerOffloadDriver", "aws-s3"); brokerProperties.setProperty(EXTRA_CONFIG_PREFIX + "tieredStorageBucketPrefix", "broker-prefix"); + brokerProperties.setProperty(EXTRA_CONFIG_PREFIX + "brokerOnly", "broker-value"); OffloadPoliciesImpl topicLevelPolicies = new OffloadPoliciesImpl(); topicLevelPolicies.getManagedLedgerExtraConfigurations().put("tieredStorageBucketPrefix", "topic-prefix"); + topicLevelPolicies.getManagedLedgerExtraConfigurations().put("topicOnly", "topic-value"); OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration(topicLevelPolicies, null, brokerProperties); Assert.assertNotNull(offloadPolicies); assertEquals(offloadPolicies.getManagedLedgerExtraConfigurations(), - Map.of("tieredStorageBucketPrefix", "topic-prefix")); + Map.of("tieredStorageBucketPrefix", "topic-prefix", + "brokerOnly", "broker-value", + "topicOnly", "topic-value")); + } + + @Test + public void emptyHigherLevelExtraConfigInheritsBrokerExtraConfigMergeTest() { + Properties brokerProperties = new Properties(); + brokerProperties.setProperty("managedLedgerOffloadDriver", "aws-s3"); + brokerProperties.setProperty(EXTRA_CONFIG_PREFIX + "tieredStorageBucketPrefix", "broker-prefix"); + + OffloadPoliciesImpl nsLevelPolicies = new OffloadPoliciesImpl(); + + OffloadPoliciesImpl offloadPolicies = + OffloadPoliciesImpl.mergeConfiguration(null, nsLevelPolicies, brokerProperties); + + Assert.assertNotNull(offloadPolicies); + assertEquals(offloadPolicies.getManagedLedgerExtraConfigurations(), + Map.of("tieredStorageBucketPrefix", "broker-prefix")); } @Test From 7b4ad2f6be2cf118007622b3ee5fcdf5142d3b9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 15:42:53 +0300 Subject: [PATCH 088/201] [fix][sec] Bump org.asynchttpclient:async-http-client from 2.14.5 to 2.15.0 (#25818) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Lari Hotari (cherry picked from commit d1a3081556ac7d74203cda2d72f3d15f8ffb830f) --- distribution/server/src/assemble/LICENSE.bin.txt | 4 ++-- distribution/shell/src/assemble/LICENSE.bin.txt | 4 ++-- pom.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 7fbc59162167a..aabc4e361600b 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -389,8 +389,8 @@ The Apache Software License, Version 2.0 * AirCompressor - io.airlift-aircompressor-2.0.3.jar * AsyncHttpClient - - org.asynchttpclient-async-http-client-2.14.5.jar - - org.asynchttpclient-async-http-client-netty-utils-2.14.5.jar + - org.asynchttpclient-async-http-client-2.15.0.jar + - org.asynchttpclient-async-http-client-netty-utils-2.15.0.jar * Jetty - org.eclipse.jetty-jetty-alpn-client-12.1.9.jar - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.9.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index bbd7e8892bda6..c7776f68f0034 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -398,8 +398,8 @@ The Apache Software License, Version 2.0 * AirCompressor - aircompressor-2.0.3.jar * AsyncHttpClient - - async-http-client-2.14.5.jar - - async-http-client-netty-utils-2.14.5.jar + - async-http-client-2.15.0.jar + - async-http-client-netty-utils-2.15.0.jar * Jetty - jetty-alpn-client-12.1.9.jar - jetty-client-12.1.9.jar diff --git a/pom.xml b/pom.xml index 7a304e9625035..543e364bbd752 100644 --- a/pom.xml +++ b/pom.xml @@ -264,7 +264,7 @@ flexible messaging model and an intuitive client API. 0.16.1 8.1.1 2.0.3 - 2.14.5 + 2.15.0 3.19.0 2.21.0 1.20.0 From f9b8ac461180f5412a152974671b43d5075e9eac Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Mon, 18 May 2026 17:14:10 +0800 Subject: [PATCH 089/201] [fix][broker] Fix ManagedLedgerImpl.advanceCursorsIfNecessary() method may lose non-durable cursor properties in race condition (#25796) --- .../mledger/impl/ManagedLedgerImpl.java | 2 +- .../mledger/impl/ManagedLedgerTest.java | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 4c2a344d85c44..54be0723d4742 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -3281,7 +3281,7 @@ void advanceCursorsIfNecessary(List ledgersToDelete) throws LedgerNo && highestPositionToDelete.compareTo(cursor.getManagedLedger() .getLastConfirmedEntry()) <= 0 && !(!cursor.isDurable() && cursor instanceof NonDurableCursorImpl && ((NonDurableCursorImpl) cursor).isReadCompacted())) { - cursor.asyncMarkDelete(highestPositionToDelete, cursor.getProperties(), new MarkDeleteCallback() { + cursor.asyncMarkDelete(highestPositionToDelete, null, new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index a03de59736e9d..693823f8b9439 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -26,6 +26,7 @@ import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; @@ -5358,4 +5359,65 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { // Verify properties are preserved after cursor reset assertEquals(cursor.getProperties(), expectedProperties); } + + @Test + @SuppressWarnings("unchecked") + public void testAdvanceCursorsIfNecessaryNeverLoseMarkDeleteProperties() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(1); + config.setRetentionTime(0, TimeUnit.SECONDS); + config.setRetentionSizeInMB(0); + + @Cleanup + ManagedLedgerImpl ledger = + (ManagedLedgerImpl) factory.open("testAdvanceCursorsIfNecessaryNeverLoseMarkDeleteProperties", config); + @Cleanup + ManagedCursorImpl durableCursor = (ManagedCursorImpl) ledger.openCursor("durableCursor1"); + @Cleanup + NonDurableCursorImpl realNonDurableCursor = + (NonDurableCursorImpl) ledger.newNonDurableCursor(PositionFactory.EARLIEST); + NonDurableCursorImpl nonDurableCursor = spy(realNonDurableCursor); + + ledger.getCursors().removeCursor(realNonDurableCursor.getName()); + ledger.getCursors().add(nonDurableCursor, null); + + CountDownLatch advanceCursorsMarkDeleteEnteredLatch = new CountDownLatch(1); + CountDownLatch nonDurableCursorsMarkDeleteCompletedLatch = new CountDownLatch(1); + CountDownLatch advanceCursorsMarkDeleteCompletedLatch = new CountDownLatch(1); + + doAnswer(invocation -> { + Map invocationProperties = invocation.getArgument(1); + // Pause the advanceCursorsIfNecessary mark-delete so the nonDurableCursor markDelete() can complete first. + if (invocationProperties == null || invocationProperties.isEmpty()) { + advanceCursorsMarkDeleteEnteredLatch.countDown(); + assertTrue(nonDurableCursorsMarkDeleteCompletedLatch.await(5, TimeUnit.SECONDS)); + try { + return invocation.callRealMethod(); + } finally { + advanceCursorsMarkDeleteCompletedLatch.countDown(); + } + } + + return invocation.callRealMethod(); + }).when(nonDurableCursor) + .internalAsyncMarkDelete(any(Position.class), nullable(Map.class), any(MarkDeleteCallback.class), + nullable(Object.class), nullable(Runnable.class)); + + ledger.addEntry("entry-1".getBytes(Encoding)); + Position pos2 = ledger.addEntry("entry-2".getBytes(Encoding)); + + // Mark-delete the durable cursor to trigger trimming, which advances non-durable cursors. + durableCursor.markDelete(pos2); + assertTrue(advanceCursorsMarkDeleteEnteredLatch.await(5, TimeUnit.SECONDS)); + + String propertyKey = "test-property"; + Map properties = new HashMap<>(); + properties.put(propertyKey, 1L); + nonDurableCursor.markDelete(pos2, properties); + nonDurableCursorsMarkDeleteCompletedLatch.countDown(); + + assertTrue(advanceCursorsMarkDeleteCompletedLatch.await(5, TimeUnit.SECONDS)); + assertEquals(nonDurableCursor.getMarkDeletedPosition(), pos2); + assertEquals(nonDurableCursor.getProperties(), properties); + } } From 9e350856b2cd8939709f80e87b302b729b5efe2a Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Tue, 19 May 2026 19:43:54 +0800 Subject: [PATCH 090/201] [improve][fn] Avoid gRPC timeout when getting status of a dead process runtime (#25819) Signed-off-by: Dream95 (cherry picked from commit 5fd7518bd5522a0084d876c745a0ec870f1f2316) --- .../pulsar/functions/runtime/process/ProcessRuntime.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java index e59fcadc729ed..24c41431c382c 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java @@ -242,6 +242,15 @@ public CompletableFuture getFunctionStatus(int instanceId) { retval.completeExceptionally(new RuntimeException("Not alive")); return retval; } + if (!isAlive()) { + FunctionStatus.Builder builder = FunctionStatus.newBuilder(); + builder.setRunning(false); + if (deathException != null && deathException.getMessage() != null) { + builder.setFailureException(deathException.getMessage()); + } + retval.complete(builder.build()); + return retval; + } ListenableFuture response = stub.withDeadlineAfter(GRPC_TIMEOUT_SECS, TimeUnit.SECONDS) .getFunctionStatus(Empty.newBuilder().build()); Futures.addCallback(response, new FutureCallback() { From e9ba10bd3849af30ab433a33f0a4835a04614cdb Mon Sep 17 00:00:00 2001 From: sbourkeostk <58816076+sbourkeostk@users.noreply.github.com> Date: Thu, 21 May 2026 10:25:36 +0100 Subject: [PATCH 091/201] [fix][fn] Fix functions update issue where artifact is provided as a http url (#25840) Co-authored-by: Stephen Bourke Co-authored-by: Lari Hotari Co-authored-by: Zixuan Liu (cherry picked from commit 235a7a8a4ca3cb6914f965ae2407030dbd1fa128) --- .../worker/rest/api/ComponentImpl.java | 4 +- .../v3/AbstractFunctionApiResourceTest.java | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index 4058ebb667c06..f3bebd78de697 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -1776,7 +1776,7 @@ protected File getPackageFile(FunctionDetails.ComponentType componentType, Strin if (isNotBlank(functionPkgUrl)) { componentPackageFile = getPackageFile(componentType, functionPkgUrl); } else if (existingPackagePath.startsWith(Utils.FILE) || existingPackagePath.startsWith(Utils.HTTP)) { - if (!worker().getPackageUrlValidator().isValidPackageUrl(componentType, functionPkgUrl)) { + if (!worker().getPackageUrlValidator().isValidPackageUrl(componentType, existingPackagePath)) { throw new IllegalArgumentException("Function Package url is not valid." + "supported url (http/https/file)"); } @@ -1785,7 +1785,7 @@ protected File getPackageFile(FunctionDetails.ComponentType componentType, Strin } catch (Exception e) { throw new IllegalArgumentException(String.format("Encountered error \"%s\" " + "when getting %s package from %s", e.getMessage(), - ComponentTypeUtils.toString(componentType), functionPkgUrl)); + ComponentTypeUtils.toString(componentType), existingPackagePath)); } } else if (Utils.hasPackageTypePrefix(existingPackagePath)) { componentPackageFile = getPackageFile(componentType, existingPackagePath); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java index d7c5b877f636d..16cf98b98472b 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java @@ -1028,6 +1028,45 @@ public void testUpdateFunctionWithUrl() throws IOException { } + @Test + public void testUpdateFunctionWithExistingFileUrl() throws IOException { + + String fileLocation = FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + String filePackageUrl = "file://" + fileLocation; + + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setOutput(OUTPUT_TOPIC); + functionConfig.setOutputSerdeClassName(OUTPUT_SERDE_CLASS_NAME); + functionConfig.setTenant(TENANT); + functionConfig.setNamespace(NAMESPACE); + functionConfig.setName(FUNCTION); + functionConfig.setClassName(CLASS_NAME); + // increment parallelism to avoid 'Update contains no change' exception + functionConfig.setParallelism(PARALLELISM + 1); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + functionConfig.setCustomSerdeInputs(TOPICS_TO_SER_DE_CLASS_NAME); + + FunctionMetaData existingMetaData = FunctionMetaData.newBuilder() + .setFunctionDetails(createDefaultFunctionDetails()) + .setPackageLocation(org.apache.pulsar.functions.proto.Function.PackageLocationMetaData.newBuilder() + .setPackagePath(filePackageUrl) + .build()) + .build(); + + when(mockedManager.containsFunction(eq(TENANT), eq(NAMESPACE), eq(FUNCTION))).thenReturn(true); + when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(existingMetaData); + + updateFunction( + TENANT, + NAMESPACE, + FUNCTION, + null, + null, + null, + functionConfig, + null, null); + } + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") public void testUpdateFunctionFailure() throws Exception { try { From 981bef6bdde7db688ef955fb734f60e0d4d5c45c Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 20 May 2026 00:59:49 -0700 Subject: [PATCH 092/201] [fix][test] Add timeout to initial receives in ResendRequestTest.testSharedSingleAckedPartitionedTopic (#25828) (cherry picked from commit 55e6022f2a12cd775f90e09a8756feb02dd2b1b0) --- .../apache/pulsar/broker/service/ResendRequestTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java index 5bc3e7e94800f..7ffec5933d7ca 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java @@ -514,8 +514,11 @@ public void testSharedSingleAckedPartitionedTopic() throws Exception { } // 4. Receive messages - Message message1 = consumer1.receive(); - Message message2 = consumer2.receive(); + // Use timeouts on the initial receives — with a Shared subscription the broker may + // dispatch all messages to a single consumer (receiverQueueSize is larger than the + // number of messages per partition), and a blocking receive() would hang. + Message message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS); + Message message2 = consumer2.receive(5000, TimeUnit.MILLISECONDS); int messageCount1 = 0; int messageCount2 = 0; int ackCount1 = 0; From 4a5bf97af4066c99050e1fd4c988ea3854afd6c8 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Fri, 22 May 2026 22:10:26 +0800 Subject: [PATCH 093/201] [fix][test] Fix flaky ResendRequestTest.testSharedSingleAckedPartitionedTopic() test (#25852) (cherry picked from commit 11f7bcdb4ff7fd10d1b278626609c7bd07cc9c2f) --- .../broker/service/ResendRequestTest.java | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java index 7ffec5933d7ca..9761042019043 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java @@ -38,6 +38,7 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.ConsumerBase; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; @@ -493,37 +494,41 @@ public void testSharedSingleAckedPartitionedTopic() throws Exception { Random rn = new Random(); // 1. producer connect - Producer producer = pulsarClient.newProducer().topic(topicName) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create(); - - // 2. Create consumer - Consumer consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) - .receiverQueueSize(7).subscriptionType(SubscriptionType.Shared).subscribe(); - @Cleanup - PulsarClient newPulsarClient = newPulsarClient(); - Consumer consumer2 = newPulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) - .receiverQueueSize(7).subscriptionType(SubscriptionType.Shared).subscribe(); + Producer producer = pulsarClient.newProducer().topic(topicName) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create(); - // 3. producer publish messages + // 2. producer publish messages for (int i = 0; i < totalMessages; i++) { String message = messagePredicate + i; log.info("Message produced: " + message); producer.send(message.getBytes()); } + // 3. Create consumer + @Cleanup + Consumer consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) + .receiverQueueSize(2).subscriptionType(SubscriptionType.Shared).subscriptionInitialPosition( + SubscriptionInitialPosition.Earliest).subscribe(); + + @Cleanup + PulsarClient newPulsarClient = newPulsarClient(); + Consumer consumer2 = newPulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) + .receiverQueueSize(2).subscriptionType(SubscriptionType.Shared).subscriptionInitialPosition( + SubscriptionInitialPosition.Earliest).subscribe(); + // 4. Receive messages - // Use timeouts on the initial receives — with a Shared subscription the broker may - // dispatch all messages to a single consumer (receiverQueueSize is larger than the - // number of messages per partition), and a blocking receive() would hang. - Message message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS); - Message message2 = consumer2.receive(5000, TimeUnit.MILLISECONDS); int messageCount1 = 0; int messageCount2 = 0; int ackCount1 = 0; int ackCount2 = 0; - do { + while (true) { + Message message1 = consumer1.receive(500, TimeUnit.MILLISECONDS); + Message message2 = consumer2.receive(500, TimeUnit.MILLISECONDS); + if (message1 == null && message2 == null) { + break; + } if (message1 != null) { log.info("Consumer1 received " + new String(message1.getData())); messageCount1 += 1; @@ -542,9 +547,7 @@ public void testSharedSingleAckedPartitionedTopic() throws Exception { ackCount2 += 1; } } - message1 = consumer1.receive(500, TimeUnit.MILLISECONDS); - message2 = consumer2.receive(500, TimeUnit.MILLISECONDS); - } while (message1 != null || message2 != null); + } log.info("messageCount1 = " + messageCount1); log.info("messageCount2 = " + messageCount2); log.info("ackCount1 = " + ackCount1); @@ -559,10 +562,13 @@ public void testSharedSingleAckedPartitionedTopic() throws Exception { } // 6. Check if Messages redelivered again - message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS); - message2 = consumer2.receive(5000, TimeUnit.MILLISECONDS); messageCount1 = 0; - do { + while (true) { + Message message1 = consumer1.receive(500, TimeUnit.MILLISECONDS); + Message message2 = consumer2.receive(500, TimeUnit.MILLISECONDS); + if (message1 == null && message2 == null) { + break; + } if (message1 != null) { log.info("Consumer1 received " + new String(message1.getData())); messageCount1 += 1; @@ -571,10 +577,7 @@ public void testSharedSingleAckedPartitionedTopic() throws Exception { log.info("Consumer2 received " + new String(message2.getData())); messageCount2 += 1; } - message1 = consumer1.receive(1000, TimeUnit.MILLISECONDS); - message2 = consumer2.receive(1000, TimeUnit.MILLISECONDS); - } while (message1 != null || message2 != null); - + } log.info("messageCount1 = " + messageCount1); log.info("messageCount2 = " + messageCount2); log.info("ackCount1 = " + ackCount1); From 4dcd9a9638d8eba76eb75eaf437353952ba94752 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 22 May 2026 17:08:33 +0300 Subject: [PATCH 094/201] [improve][build] Remove kotlin-stdlib override; upgrade okhttp3 5.3.2 and okio 3.17.0 (#25855) (cherry picked from commit 842785e0d3b629126a84b27c699fa541ae805c85) --- .../server/src/assemble/LICENSE.bin.txt | 13 +++++------ pom.xml | 23 ++----------------- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index aabc4e361600b..87b86715ee65f 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -434,16 +434,15 @@ The Apache Software License, Version 2.0 * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.45.0.jar * Apache Thrift - org.apache.thrift-libthrift-0.23.0.jar * OkHttp3 - - com.squareup.okhttp3-logging-interceptor-5.3.1.jar - - com.squareup.okhttp3-okhttp-5.3.1.jar - - com.squareup.okhttp3-okhttp-jvm-5.3.1.jar + - com.squareup.okhttp3-logging-interceptor-5.3.2.jar + - com.squareup.okhttp3-okhttp-5.3.2.jar + - com.squareup.okhttp3-okhttp-jvm-5.3.2.jar * Okio - - com.squareup.okio-okio-3.16.3.jar - - com.squareup.okio-okio-jvm-3.16.3.jar + - com.squareup.okio-okio-3.17.0.jar + - com.squareup.okio-okio-jvm-3.17.0.jar * Javassist -- org.javassist-javassist-3.25.0-GA.jar * Kotlin Standard Lib - - org.jetbrains.kotlin-kotlin-stdlib-1.8.20.jar - - org.jetbrains.kotlin-kotlin-stdlib-common-1.8.20.jar + - org.jetbrains.kotlin-kotlin-stdlib-2.2.21.jar - org.jetbrains-annotations-13.0.jar * gRPC - io.grpc-grpc-all-1.75.0.jar diff --git a/pom.xml b/pom.xml index 543e364bbd752..2e16c75157e76 100644 --- a/pom.xml +++ b/pom.xml @@ -288,11 +288,9 @@ flexible messaging model and an intuitive client API. 5.18.1 23.0.0 0.9.6 - 5.3.1 + 5.3.2 - 3.16.3 - - 1.8.20 + 3.17.0 1.0 9.1.6 6.2.12 @@ -1650,23 +1648,6 @@ flexible messaging model and an intuitive client API. import - - org.jetbrains.kotlin - kotlin-stdlib - ${kotlin-stdlib.version} - - - org.jetbrains.kotlin - kotlin-stdlib-common - ${kotlin-stdlib.version} - - - - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - ${kotlin-stdlib.version} - - com.github.luben zstd-jni From bc56bfe97f8df3b083701ba2689f69276592590c Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 24 May 2026 20:24:03 +0800 Subject: [PATCH 095/201] [improve][client] In cases where there is a risk of message loss, adjust the log level to error (#25854) (cherry picked from commit 09035ffddada146f3ef014777ee5e5766f01f3f2) --- .../pulsar/client/impl/ConsumerBase.java | 21 ++++++++++++------- .../pulsar/client/impl/ConsumerImpl.java | 12 +++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index aed525c9eeeb3..8137f2386a7d6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -336,10 +336,14 @@ protected CompletableFuture> nextPendingReceive() { protected void completePendingReceive(CompletableFuture> receivedFuture, Message message) { getInternalExecutor(message).execute(() -> { - if (!receivedFuture.complete(message)) { - log.warn("Race condition detected. receive future was already completed (cancelled={}) and message was " - + "dropped. message={}", - receivedFuture.isCancelled(), message); + if (!receivedFuture.complete(message) && getState() != State.Closing && getState() != State.Closed) { + log.error("Race condition detected, receive future was already completed and message was dropped." + + " In other words, the message was dropped internally, the client-side will encounter a" + + " crucial issue: this message will never be consumed until the consumer is restarted or" + + " the topic is unloaded. Under normal circumstances, this won't happen. It only occurs when" + + " user itself has completed the completable future object returned by" + + " \"consumer.receiveAsync()\". message={}, cancelled={}", message, + receivedFuture.isCancelled()); } }); } @@ -1103,9 +1107,12 @@ protected final void notifyPendingBatchReceivedCallBack(CompletableFuture> future, Messages messages) { if (!future.complete(messages)) { - log.warn("Race condition detected. batch receive future was already completed (cancelled={}) and messages" - + " were dropped. messages={}", - future.isCancelled(), messages); + log.warn("Race condition detected, receive future was already completed and message was dropped." + + " In other words, the message was dropped internally, the client-side will encounter a" + + " crucial issue: these message will never be consumed until the consumer is restarted or" + + " the topic is unloaded. Under normal circumstances, this won't happen. It only occurs when" + + " user itself has completed the completable future object returned by" + + " \"consumer.batchReceiveAsync()\". messages={}, cancelled={}", messages, future.isCancelled()); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 1b844d120b39f..5c1d94992262e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1722,12 +1722,24 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m */ void notifyPendingReceivedCallback(final Message message, Exception exception) { if (pendingReceives.isEmpty()) { + if (getState() != State.Closing && getState() != State.Closed) { + log.error("If you received this log, it means that you encountered a bug: a message was" + + " dropped internally, the client-side will encounter a crucial issue: this message will" + + " never be consumed until the consumer is restarted or the topic is unloaded. message={}," + + " pendingReceives-size={}", message, pendingReceives.size()); + } return; } // fetch receivedCallback from queue final CompletableFuture> receivedFuture = nextPendingReceive(); if (receivedFuture == null) { + if (getState() != State.Closing && getState() != State.Closed) { + log.error("The pendingReceives pulled out a null conpletableFuture object. If you received this log," + + " it means that you encountered a bug: a message was" + + " dropped internally, the client-side will encounter a crucial issue: this message will never" + + " be consumed until the consumer is restarted or the topic is unloaded. message={}", message); + } return; } From 3f59a3d496add4aedf68aecc0f889a27fba82507 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 25 May 2026 20:30:23 +0800 Subject: [PATCH 096/201] [fix][client] Fix failed to close consumer because of the error: param memorySize is a negative value (#25805) (cherry picked from commit 59d1495fef46075986a92065bd64229fe0f354ea) --- .../apache/pulsar/client/impl/ConsumerBase.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 8137f2386a7d6..95a21b6c92a16 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -976,13 +976,17 @@ protected boolean enqueueMessageAndCheckBatchReceive(Message message) { // synchronize redeliverUnacknowledgedMessages(). incomingQueueLock.lock(); try { - if (canEnqueueMessage(message) && incomingMessages.offer(message)) { - // After we have enqueued the messages on `incomingMessages` queue, we cannot touch the message - // instance anymore, since for pooled messages, this instance was possibly already been released - // and recycled. + if (canEnqueueMessage(message)) { INCOMING_MESSAGES_SIZE_UPDATER.addAndGet(this, messageSize); - getMemoryLimitController().ifPresent(limiter -> limiter.forceReserveMemory(messageSize)); - updateAutoScaleReceiverQueueHint(); + if (incomingMessages.offer(message)) { + // After we have enqueued the messages on `incomingMessages` queue, we cannot touch the message + // instance anymore, since for pooled messages, this instance was possibly already been released + // and recycled. + getMemoryLimitController().ifPresent(limiter -> limiter.forceReserveMemory(messageSize)); + updateAutoScaleReceiverQueueHint(); + } else { + INCOMING_MESSAGES_SIZE_UPDATER.addAndGet(this, -messageSize); + } } } finally { incomingQueueLock.unlock(); From 758d6e927eb4e969b24b9741cb9b3c75c86ee283 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 21 May 2026 09:44:56 +0300 Subject: [PATCH 097/201] [fix][sec] Upgrade commons-configuration2 to 2.15.0 to address CVE-2026-45205 (#25844) (cherry picked from commit 7220158b928b2b9dc8151400a1a596d93ecd5637) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 87b86715ee65f..f5ccf40a1d206 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -289,7 +289,7 @@ The Apache Software License, Version 2.0 - commons-logging-commons-logging-1.3.5.jar - org.apache.commons-commons-collections4-4.5.0.jar - org.apache.commons-commons-compress-1.28.0.jar - - org.apache.commons-commons-configuration2-2.12.0.jar + - org.apache.commons-commons-configuration2-2.15.0.jar - org.apache.commons-commons-lang3-3.19.0.jar - org.apache.commons-commons-text-1.14.0.jar * Netty diff --git a/pom.xml b/pom.xml index 2e16c75157e76..ac3689f913ec2 100644 --- a/pom.xml +++ b/pom.xml @@ -372,7 +372,7 @@ flexible messaging model and an intuitive client API. 3.33.0 9.37.4 1.11.0 - 2.12.0 + 2.15.0 2.1.10 1.10.3 3.0.2 From 6334d191ed1d7d1f700802fc79232c2db0d1e2c4 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 26 May 2026 11:48:35 +0300 Subject: [PATCH 098/201] [improve][misc] Upgrade Netty to 4.1.134 (#25870) (cherry picked from commit a24a3b50b46abbacafcdef23d23ee8a03c3e25ac) --- .../server/src/assemble/LICENSE.bin.txt | 40 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 38 +++++++++--------- pom.xml | 2 +- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index f5ccf40a1d206..bddcd8f54588a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -293,26 +293,26 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-lang3-3.19.0.jar - org.apache.commons-commons-text-1.14.0.jar * Netty - - io.netty-netty-buffer-4.1.133.Final.jar - - io.netty-netty-codec-4.1.133.Final.jar - - io.netty-netty-codec-dns-4.1.133.Final.jar - - io.netty-netty-codec-http-4.1.133.Final.jar - - io.netty-netty-codec-http2-4.1.133.Final.jar - - io.netty-netty-codec-socks-4.1.133.Final.jar - - io.netty-netty-codec-haproxy-4.1.133.Final.jar - - io.netty-netty-common-4.1.133.Final.jar - - io.netty-netty-handler-4.1.133.Final.jar - - io.netty-netty-handler-proxy-4.1.133.Final.jar - - io.netty-netty-resolver-4.1.133.Final.jar - - io.netty-netty-resolver-dns-4.1.133.Final.jar - - io.netty-netty-resolver-dns-classes-macos-4.1.133.Final.jar - - io.netty-netty-resolver-dns-native-macos-4.1.133.Final-osx-aarch_64.jar - - io.netty-netty-resolver-dns-native-macos-4.1.133.Final-osx-x86_64.jar - - io.netty-netty-transport-4.1.133.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.133.Final.jar - - io.netty-netty-transport-native-epoll-4.1.133.Final-linux-aarch_64.jar - - io.netty-netty-transport-native-epoll-4.1.133.Final-linux-x86_64.jar - - io.netty-netty-transport-native-unix-common-4.1.133.Final.jar + - io.netty-netty-buffer-4.1.134.Final.jar + - io.netty-netty-codec-4.1.134.Final.jar + - io.netty-netty-codec-dns-4.1.134.Final.jar + - io.netty-netty-codec-http-4.1.134.Final.jar + - io.netty-netty-codec-http2-4.1.134.Final.jar + - io.netty-netty-codec-socks-4.1.134.Final.jar + - io.netty-netty-codec-haproxy-4.1.134.Final.jar + - io.netty-netty-common-4.1.134.Final.jar + - io.netty-netty-handler-4.1.134.Final.jar + - io.netty-netty-handler-proxy-4.1.134.Final.jar + - io.netty-netty-resolver-4.1.134.Final.jar + - io.netty-netty-resolver-dns-4.1.134.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.1.134.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.1.134.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.1.134.Final-osx-x86_64.jar + - io.netty-netty-transport-4.1.134.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.134.Final.jar + - io.netty-netty-transport-native-epoll-4.1.134.Final-linux-aarch_64.jar + - io.netty-netty-transport-native-epoll-4.1.134.Final-linux-x86_64.jar + - io.netty-netty-transport-native-unix-common-4.1.134.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-linux-aarch_64.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-linux-x86_64.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index c7776f68f0034..f8076d313ba94 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -345,22 +345,22 @@ The Apache Software License, Version 2.0 - commons-text-1.14.0.jar - commons-compress-1.28.0.jar * Netty - - netty-buffer-4.1.133.Final.jar - - netty-codec-4.1.133.Final.jar - - netty-codec-dns-4.1.133.Final.jar - - netty-codec-http-4.1.133.Final.jar - - netty-codec-socks-4.1.133.Final.jar - - netty-codec-haproxy-4.1.133.Final.jar - - netty-common-4.1.133.Final.jar - - netty-handler-4.1.133.Final.jar - - netty-handler-proxy-4.1.133.Final.jar - - netty-resolver-4.1.133.Final.jar - - netty-resolver-dns-4.1.133.Final.jar - - netty-transport-4.1.133.Final.jar - - netty-transport-classes-epoll-4.1.133.Final.jar - - netty-transport-native-epoll-4.1.133.Final-linux-aarch_64.jar - - netty-transport-native-epoll-4.1.133.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.133.Final.jar + - netty-buffer-4.1.134.Final.jar + - netty-codec-4.1.134.Final.jar + - netty-codec-dns-4.1.134.Final.jar + - netty-codec-http-4.1.134.Final.jar + - netty-codec-socks-4.1.134.Final.jar + - netty-codec-haproxy-4.1.134.Final.jar + - netty-common-4.1.134.Final.jar + - netty-handler-4.1.134.Final.jar + - netty-handler-proxy-4.1.134.Final.jar + - netty-resolver-4.1.134.Final.jar + - netty-resolver-dns-4.1.134.Final.jar + - netty-transport-4.1.134.Final.jar + - netty-transport-classes-epoll-4.1.134.Final.jar + - netty-transport-native-epoll-4.1.134.Final-linux-aarch_64.jar + - netty-transport-native-epoll-4.1.134.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.134.Final.jar - netty-tcnative-boringssl-static-2.0.77.Final.jar - netty-tcnative-boringssl-static-2.0.77.Final-linux-aarch_64.jar - netty-tcnative-boringssl-static-2.0.77.Final-linux-x86_64.jar @@ -371,9 +371,9 @@ The Apache Software License, Version 2.0 - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - - netty-resolver-dns-classes-macos-4.1.133.Final.jar - - netty-resolver-dns-native-macos-4.1.133.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.133.Final-osx-x86_64.jar + - netty-resolver-dns-classes-macos-4.1.134.Final.jar + - netty-resolver-dns-native-macos-4.1.134.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.134.Final-osx-x86_64.jar * Prometheus client - simpleclient-0.16.0.jar - simpleclient_log4j2-0.16.0.jar diff --git a/pom.xml b/pom.xml index ac3689f913ec2..6e9a41961ec3f 100644 --- a/pom.xml +++ b/pom.xml @@ -189,7 +189,7 @@ flexible messaging model and an intuitive client API. 1.1.10.8 4.1.12.1 5.7.1 - 4.1.133.Final + 4.1.134.Final 0.0.26.Final 12.1.9 From d93426e0a45091e77d3bb52d4a8ac1f1fc175352 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Mon, 25 May 2026 15:06:40 +0800 Subject: [PATCH 099/201] [fix][test] Stabilize WebService rate limiting test (#25866) (cherry picked from commit 3b6a76096740870e87c93919b21ddbd1ee2cdeed) --- .../pulsar/broker/web/WebServiceTest.java | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java index 0f9ccdf4af714..c307eb9b1aea6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java @@ -269,7 +269,8 @@ public void testTlsAuthDisallowInsecure() throws Exception { @Test public void testRateLimiting() throws Exception { - setupEnv(false, false, false, false, 10.0, false); + double rateLimit = 10.0; + setupEnv(false, false, false, false, rateLimit, false); // setupEnv makes HTTP calls to create the cluster, tenant, and namespace. var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); @@ -283,7 +284,7 @@ public void testRateLimiting() throws Exception { // Make requests without exceeding the max rate for (int i = 0; i < 5; i++) { makeHttpRequest(false, false); - Thread.sleep(200); + Thread.sleep(rateLimitPauseMillis(rateLimit)); } metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); @@ -573,13 +574,32 @@ private void setupEnv(boolean enableFilter, boolean enableTls, boolean enableAut } catch (ConflictException ce) { // This is OK. } + sleepForRateLimiter(rateLimit); + try { pulsarAdmin.tenants().createTenant("my-property", TenantInfo.builder().allowedClusters(Sets.newHashSet(config.getClusterName())).build()); + } catch (Exception e) { + // This is OK. + } + sleepForRateLimiter(rateLimit); + + try { pulsarAdmin.namespaces().createNamespace("my-property/my-namespace"); } catch (Exception e) { // This is OK. } + sleepForRateLimiter(rateLimit); + } + + private static void sleepForRateLimiter(double rateLimit) throws InterruptedException { + if (rateLimit > 0) { + Thread.sleep(rateLimitPauseMillis(rateLimit)); + } + } + + private static long rateLimitPauseMillis(double rateLimit) { + return (long) Math.ceil((1000.0 / rateLimit) * 2); } @AfterMethod(alwaysRun = true) From 5991a1d685e4057b9c41c3973afbb7c568f188eb Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Mon, 25 May 2026 20:28:29 +0800 Subject: [PATCH 100/201] [fix][fn] Fix Go function runtime to continue after user exceptions and add neg-ack tests (#25867) Signed-off-by: Dream95 (cherry picked from commit a5c1029668e182b05579d223683d5e6ea82f84fe) --- pulsar-function-go/pf/instance.go | 41 +++++++++++++------ pulsar-function-go/pf/instance_test.go | 32 +++++++++++++++ .../latest-version-image/Dockerfile | 1 + .../exceptionFunc/exceptionFunc.go | 41 +++++++++++++++++++ .../functions/PulsarFunctionsTest.java | 4 ++ .../functions/PulsarFunctionsTestBase.java | 1 + .../functions/go/PulsarFunctionsGoTest.java | 5 +++ 7 files changed, 112 insertions(+), 13 deletions(-) create mode 100644 tests/docker-images/latest-version-image/go-examples/exceptionFunc/exceptionFunc.go diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index af8a4e0157b76..2cdfc8a6e9497 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -164,7 +164,6 @@ CLOSE: case cm := <-channel: msgInput := cm.Message atMostOnce := gi.context.instanceConf.funcDetails.ProcessingGuarantees == pb.ProcessingGuarantees_ATMOST_ONCE - atLeastOnce := gi.context.instanceConf.funcDetails.ProcessingGuarantees == pb.ProcessingGuarantees_ATLEAST_ONCE autoAck := gi.context.instanceConf.funcDetails.AutoAck //nolint:staticcheck if autoAck && atMostOnce { gi.ackInputMessage(msgInput) @@ -177,12 +176,8 @@ CLOSE: output, err := gi.handlerMsg(msgInput) if err != nil { - log.Errorf("handler message error:%v", err) - if autoAck && atLeastOnce { - gi.nackInputMessage(msgInput) - } - gi.stats.incrTotalUserExceptions(err) - return err + gi.handleUserError(msgInput, err) + continue } gi.stats.processTimeEnd() @@ -391,6 +386,29 @@ func (gi *goInstance) setupConsumer() (chan pulsar.ConsumerMessage, error) { return channel, nil } +func (gi *goInstance) shouldNackInputOnFailure() bool { + guarantee := gi.context.instanceConf.funcDetails.ProcessingGuarantees + return guarantee == pb.ProcessingGuarantees_ATLEAST_ONCE || + guarantee == pb.ProcessingGuarantees_MANUAL +} + +func (gi *goInstance) handleUserError(msgInput pulsar.Message, err error) { + log.Errorf("handler message error:%v", err) + if gi.shouldNackInputOnFailure() { + gi.nackInputMessage(msgInput) + } + gi.stats.incrTotalUserExceptions(err) + gi.stats.processTimeEnd() +} + +func (gi *goInstance) handlePublishError(msgInput pulsar.Message, err error) { + if gi.context.instanceConf.funcDetails.ProcessingGuarantees == pb.ProcessingGuarantees_ATLEAST_ONCE { + gi.nackInputMessage(msgInput) + } + gi.stats.incrTotalSysExceptions(err) + log.Errorf("failed to publish output message: %v", err) +} + func (gi *goInstance) handlerMsg(input pulsar.Message) (output []byte, err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -420,11 +438,8 @@ func (gi *goInstance) processResult(msgInput pulsar.Message, output []byte) { // semantics, ensure we nack so someone else can get it, in case we are the only handler. Then mark // exception and fail out. if err != nil { - if autoAck && atLeastOnce { - gi.nackInputMessage(msgInput) - } - gi.stats.incrTotalSysExceptions(err) - log.Fatal(err) + gi.handlePublishError(msgInput, err) + return } // Otherwise the message succeeded. If the SDK is entrusted with responding and we are using // atLeastOnce delivery semantics, ack the message. @@ -437,7 +452,7 @@ func (gi *goInstance) processResult(msgInput pulsar.Message, output []byte) { return } - // No output from the function or no output topic. Ack if we need to and mark the success before rturning. + // No output from the function or no output topic. Ack if we need to and mark the success before returning. if autoAck && atLeastOnce { gi.ackInputMessage(msgInput) } diff --git a/pulsar-function-go/pf/instance_test.go b/pulsar-function-go/pf/instance_test.go index bf45ae3a8917e..447d3a22f8d88 100644 --- a/pulsar-function-go/pf/instance_test.go +++ b/pulsar-function-go/pf/instance_test.go @@ -27,6 +27,8 @@ import ( "time" "github.com/stretchr/testify/assert" + + pb "github.com/apache/pulsar/pulsar-function-go/pb" ) func testProcessSpawnerHealthCheckTimer( @@ -115,3 +117,33 @@ func Test_goInstance_handlerMsg(t *testing.T) { assert.Equal(t, "output", string(output)) assert.Equal(t, message, fc.record) } + +func newTestGoInstance(guarantee pb.ProcessingGuarantees) *goInstance { + return &goInstance{ + context: &FunctionContext{ + instanceConf: &instanceConf{ + funcDetails: pb.FunctionDetails{ + ProcessingGuarantees: guarantee, + }, + }, + }, + } +} + +func TestShouldNackInputOnFailure(t *testing.T) { + tests := []struct { + name string + guarantee pb.ProcessingGuarantees + want bool + }{ + {"atLeastOnce", pb.ProcessingGuarantees_ATLEAST_ONCE, true}, + {"manual", pb.ProcessingGuarantees_MANUAL, true}, + {"atMostOnce", pb.ProcessingGuarantees_ATMOST_ONCE, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + instance := newTestGoInstance(tt.guarantee) + assert.Equal(t, tt.want, instance.shouldNackInputOnFailure()) + }) + } +} diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index 6fe2ef656a4a9..08587af16c76e 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -24,6 +24,7 @@ ARG GOLANG_IMAGE FROM $GOLANG_IMAGE as pulsar-function-go COPY target/pulsar-function-go/ /go/src/github.com/apache/pulsar/pulsar-function-go +COPY go-examples/exceptionFunc/ /go/src/github.com/apache/pulsar/pulsar-function-go/examples/exceptionFunc/ RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go && go install ./... RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/pf && go install RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/examples && go install ./... diff --git a/tests/docker-images/latest-version-image/go-examples/exceptionFunc/exceptionFunc.go b/tests/docker-images/latest-version-image/go-examples/exceptionFunc/exceptionFunc.go new file mode 100644 index 0000000000000..80ace6e21b9b4 --- /dev/null +++ b/tests/docker-images/latest-version-image/go-examples/exceptionFunc/exceptionFunc.go @@ -0,0 +1,41 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package main + +import ( + "context" + "errors" + + "github.com/apache/pulsar/pulsar-function-go/pf" +) + +var i int + +func HandleException(ctx context.Context, in []byte) ([]byte, error) { + i++ + if i%10 == 0 { + return nil, errors.New("test") + } + return []byte(string(in) + "!"), nil +} + +func main() { + pf.Start(HandleException) +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index fdda3144e8bd4..82418d9d9dc7f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -404,6 +404,10 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { submitFunction( runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE, EXCEPTION_PYTHON_CLASS, schema, null); + } else if (runtime == Runtime.GO) { + submitFunction( + runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_GO_FILE, + null, schema, null); } else { submitFunction( runtime, inputTopicName, outputTopicName, functionName, null, EXCEPTION_JAVA_CLASS, schema, null); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java index 288ced63ae5eb..84b31dca9a021 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java @@ -78,6 +78,7 @@ public abstract class PulsarFunctionsTestBase extends PulsarTestSuite { public static final String EXCLAMATION_GO_FILE = "exclamationFunc"; public static final String PUBLISH_FUNCTION_GO_FILE = "exclamationFunc"; + public static final String EXCEPTION_GO_FILE = "exceptionFunc"; public static final String LOGGING_JAVA_CLASS = "org.apache.pulsar.functions.api.examples.LoggingFunction"; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java index 0550fd94ebee2..6e631adafbf7f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java @@ -39,4 +39,9 @@ public void testGoExclamationMultiInputsFunction() throws Exception { testExclamationFunction(Runtime.GO, false, false, true, false); } + @Test(groups = {"go_function", "function"}) + public void testGoFunctionNegAck() throws Exception { + testFunctionNegAck(Runtime.GO); + } + } From 5f10bf0c96fa98e619af599ae208bc5694f1c789 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Mon, 25 May 2026 20:28:47 +0800 Subject: [PATCH 101/201] [fix][test] Fix flaky ProducerCleanupTest timer cleanup (#25864) (cherry picked from commit 2e02b7830ee14e8d3476dba03f7fa6fd0da175b8) --- .../pulsar/client/api/ProducerCleanupTest.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCleanupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCleanupTest.java index 2a7fcb3eb17af..09c0113c0a163 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCleanupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCleanupTest.java @@ -20,8 +20,10 @@ import io.netty.util.HashedWheelTimer; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import org.apache.pulsar.broker.service.SharedPulsarBaseTest; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.Test; @@ -29,14 +31,17 @@ public class ProducerCleanupTest extends SharedPulsarBaseTest { @Test - public void testAllTimerTaskShouldCanceledAfterProducerClosed() throws PulsarClientException, InterruptedException { - Producer producer = pulsarClient.newProducer() + public void testAllTimerTaskShouldCanceledAfterProducerClosed() throws PulsarClientException { + @Cleanup + PulsarClient client = newPulsarClient(); + Producer producer = client.newProducer() .topic(newTopicName()) - .sendTimeout(1, TimeUnit.SECONDS) + .sendTimeout(15, TimeUnit.SECONDS) .create(); producer.close(); - Thread.sleep(2000); - HashedWheelTimer timer = (HashedWheelTimer) ((PulsarClientImpl) pulsarClient).timer(); - Assert.assertEquals(timer.pendingTimeouts(), 0); + HashedWheelTimer timer = (HashedWheelTimer) ((PulsarClientImpl) client).timer(); + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> Assert.assertEquals(timer.pendingTimeouts(), 0)); } } From b3c7f394c86b43bcd86fc2172a58b90e622b394a Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Sat, 30 May 2026 16:16:16 +0800 Subject: [PATCH 102/201] [fix][test] Fix flaky PulsarFunctionTlsTest.testFunctionsCreation() test (#25889) (cherry picked from commit a6af80198f26d31e991e880db3f9b7d601a579c2) --- .../worker/PulsarFunctionTlsTest.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java index e23dd51dc4a5b..7872b79566715 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java @@ -20,8 +20,6 @@ import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; @@ -42,10 +40,10 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.functions.FunctionConfig; -import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.ObjectMapperFactory; @@ -260,23 +258,16 @@ public void testFunctionsCreation() throws Exception { log.info(" -------- Start test function : {}", functionName); - int finalI = i; - // Wait for a leader to be ready and create the function. - // The createFunctionWithUrl call is included in the retry loop because a leadership - // transition can happen between the leader check and the actual API call, causing - // a 503 "Leader not yet ready" error. final PulsarAdmin createAdmin = pulsarAdmins[i]; - Awaitility.await().atMost(1, TimeUnit.MINUTES).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { - final PulsarWorkerService workerService = ((PulsarWorkerService) fnWorkerServices[finalI]); - final LeaderService leaderService = workerService.getLeaderService(); - assertNotNull(leaderService); - if (!leaderService.isLeader()) { - final WorkerInfo workerInfo = workerService.getMembershipManager().getLeader(); - assertTrue(workerInfo != null - && !workerInfo.getWorkerId().equals(workerService.getWorkerConfig().getWorkerId())); - } - createAdmin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl); - }); + // During function-worker leadership election/switchover, the coordination topic can already point to + // the new leader while that worker is still finishing its leader initialization. In that short window + // the internal /functions/leader request returns a transient 503 "Leader not yet ready", so retry only + // that condition and let all other failures surface immediately. + Awaitility.await().atMost(1, TimeUnit.MINUTES) + .pollInterval(1, TimeUnit.SECONDS) + .ignoreExceptionsMatching(PulsarFunctionTlsTest::isLeaderNotReady) + .untilAsserted(() -> createAdmin.functions() + .createFunctionWithUrl(functionConfig, jarFilePathUrl)); // Function creation is not strongly consistent, so this test can fail with a get that is too eager and // does not have retries. @@ -293,6 +284,14 @@ public void testFunctionsCreation() throws Exception { } } + private static boolean isLeaderNotReady(Throwable e) { + return e instanceof PulsarAdminException + && ((PulsarAdminException) e).getStatusCode() == 503 + && String.valueOf(((PulsarAdminException) e).getHttpError()) + .contains("Leader not yet ready"); + } + + protected static FunctionConfig createFunctionConfig( String jarFile, String tenant, From 579d807bee42d84ecdacf4b446b0cdcbffcf42b9 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Mon, 25 May 2026 20:29:10 +0800 Subject: [PATCH 103/201] [fix][broker] Fix compaction cursor reset may lose mark-delete properties (#25862) (cherry picked from commit 9b15504a696e3ea21426b9e7b92a1dd613120da5) --- .../mledger/impl/ManagedCursorImpl.java | 2 +- .../mledger/impl/ManagedCursorTest.java | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 4267016ac8383..43a786ef4cf17 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1679,7 +1679,7 @@ public void operationFailed(ManagedLedgerException exception) { persistentMarkDeletePosition = null; inProgressMarkDeletePersistPosition = null; - internalAsyncMarkDelete(newMarkDeletePosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(), + internalAsyncMarkDelete(newMarkDeletePosition, isCompactionCursor() ? null : Collections.emptyMap(), new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index a3449298fd89c..351c806f53263 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -23,6 +23,7 @@ import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.eq; @@ -6026,6 +6027,89 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { assertEquals(properties.get(propertyKey), lastIndex - 1); } + @Test + @SuppressWarnings("unchecked") + public void testCompactionCursorResetNeverLoseMarkDeleteProperties() throws Exception { + @Cleanup + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open( + "testCompactionCursorResetNeverLoseMarkDeleteProperties", + new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("__compaction"); + ManagedCursorImpl spyCursor = spy(cursor); + ledger.getCursors().removeCursor(cursor.getName()); + ledger.getCursors().add(spyCursor, null); + + ledger.addEntry("entry-1".getBytes(Encoding)); + Position markDeletePosition = ledger.addEntry("entry-2".getBytes(Encoding)); + + String compactedLedgerProperty = "CompactedTopicLedger"; + Map properties = Map.of(compactedLedgerProperty, 123456L); + + CountDownLatch markDeleteEntered = new CountDownLatch(1); + CountDownLatch resetEntered = new CountDownLatch(1); + CountDownLatch markDeleteReturned = new CountDownLatch(1); + CountDownLatch markDeleteCompleted = new CountDownLatch(1); + CountDownLatch resetCompleted = new CountDownLatch(1); + + doAnswer(invocation -> { + Map invocationProperties = invocation.getArgument(1); + if (invocationProperties != null && invocationProperties.containsKey(compactedLedgerProperty)) { + // Hold the compaction mark-delete after it enters internalAsyncMarkDelete, but before its + // properties can update lastMarkDeleteEntry. + markDeleteEntered.countDown(); + assertTrue(resetEntered.await(5, TimeUnit.SECONDS)); + try { + return invocation.callRealMethod(); + } finally { + markDeleteReturned.countDown(); + } + } + + if (invocationProperties == null || invocationProperties.isEmpty()) { + // Let reset capture its properties argument first, then persist it only after the compaction + // mark-delete has completed the real internalAsyncMarkDelete call. + resetEntered.countDown(); + assertTrue(markDeleteReturned.await(5, TimeUnit.SECONDS)); + return invocation.callRealMethod(); + } + + return invocation.callRealMethod(); + }).when(spyCursor).internalAsyncMarkDelete(any(Position.class), nullable(Map.class), + any(MarkDeleteCallback.class), nullable(Object.class), nullable(Runnable.class)); + + // Start compaction mark-delete from another thread because the spy intentionally blocks it. + CompletableFuture.runAsync(() -> spyCursor.asyncMarkDelete( + markDeletePosition, properties, new MarkDeleteCallback() { + @Override + public void markDeleteComplete(Object ctx) { + markDeleteCompleted.countDown(); + } + + @Override + public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { + } + }, null)); + + assertTrue(markDeleteEntered.await(5, TimeUnit.SECONDS)); + // Reset the compaction cursor while the previous mark-delete with properties is still in progress. + spyCursor.asyncResetCursor(markDeletePosition, false, new AsyncCallbacks.ResetCursorCallback() { + @Override + public void resetComplete(Object ctx) { + resetCompleted.countDown(); + } + + @Override + public void resetFailed(ManagedLedgerException exception, Object ctx) { + } + }); + + assertTrue(markDeleteCompleted.await(5, TimeUnit.SECONDS)); + assertTrue(resetCompleted.await(5, TimeUnit.SECONDS)); + + assertEquals(spyCursor.getMarkDeletedPosition(), markDeletePosition); + assertEquals(spyCursor.getProperties(), properties); + } + class TestPulsarMockBookKeeper extends PulsarMockBookKeeper { Map ledgerErrors = new HashMap<>(); From 25af7cd00f9eaafcea217f037c98538a88f20a5b Mon Sep 17 00:00:00 2001 From: Pratik Katti <90851204+pratt4@users.noreply.github.com> Date: Mon, 25 May 2026 20:58:25 +0530 Subject: [PATCH 104/201] Return 400 for invalid reader messageId query parameter (#25865) (cherry picked from commit b406518fd2633c842d63d31ea16fe94e015ccd88) --- .../pulsar/websocket/ReaderHandler.java | 36 +++++-- .../pulsar/websocket/ReaderHandlerTest.java | 97 ++++++++++++++++++- 2 files changed, 121 insertions(+), 12 deletions(-) diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java index 7cee6005f05d6..c4bb952c62886 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java @@ -121,11 +121,17 @@ public ReaderHandler(WebSocketService service, HttpServletRequest request, Jetty } allowConnect = true; } catch (Exception e) { - log.warn("[{}:{}] Failed in creating reader {} on topic {}", request.getRemoteAddr(), - request.getRemotePort(), subscription, topic, e); + int errorCode = getErrorCode(e); + boolean isKnownError = errorCode != HttpServletResponse.SC_INTERNAL_SERVER_ERROR; + if (isKnownError) { + log.warn("[{}:{}] Failed in creating reader {} on topic {}: {}", request.getRemoteAddr(), + request.getRemotePort(), subscription, topic, e.getMessage()); + } else { + log.error("[{}:{}] Failed in creating reader {} on topic {}", request.getRemoteAddr(), + request.getRemotePort(), subscription, topic, e); + } try { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, - "Failed to create reader: " + e.getMessage()); + response.sendError(errorCode, getErrorMessage(e)); } catch (IOException e1) { log.warn("[{}:{}] Failed to send error: {}", request.getRemoteAddr(), request.getRemotePort(), e1.getMessage(), e1); @@ -340,13 +346,25 @@ private int getReceiverQueueSize() { return size; } - private MessageId getMessageId() throws IOException { + private MessageId getMessageId() { MessageId messageId = MessageId.latest; - if (isNotBlank(queryParams.get("messageId"))) { - if (queryParams.get("messageId").equals("earliest")) { + String messageIdParam = queryParams.get("messageId"); + if (isNotBlank(messageIdParam)) { + if (messageIdParam.equals("earliest")) { messageId = MessageId.earliest; - } else if (!queryParams.get("messageId").equals("latest")) { - messageId = MessageIdImpl.fromByteArray(Base64.getDecoder().decode(queryParams.get("messageId"))); + } else if (!messageIdParam.equals("latest")) { + final byte[] decoded; + try { + decoded = Base64.getDecoder().decode(messageIdParam); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid messageId base64 value", e); + } + + try { + messageId = MessageIdImpl.fromByteArray(decoded); + } catch (IOException | RuntimeException e) { + throw new IllegalArgumentException("Invalid messageId value", e); + } } } return messageId; diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java index a79899ab6fac7..9db1561ec5a9a 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java @@ -21,17 +21,22 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.Base64; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Function; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClient; @@ -43,12 +48,98 @@ import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.client.impl.MultiTopicsReaderImpl; import org.apache.pulsar.client.impl.ReaderImpl; +import org.apache.pulsar.common.api.proto.MessageIdData; import org.eclipse.jetty.ee8.websocket.server.JettyServerUpgradeResponse; import org.testng.Assert; import org.testng.annotations.Test; public class ReaderHandlerTest { + @Test + @SuppressWarnings("unchecked") + public void testInvalidMessageIdBase64ReturnsBadRequest() throws IOException { + WebSocketService wss = mock(WebSocketService.class); + PulsarClient mockedClient = mock(PulsarClient.class); + when(wss.getPulsarClient()).thenReturn(mockedClient); + ReaderBuilder mockedReaderBuilder = mock(ReaderBuilder.class); + when(mockedClient.newReader()).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.topic(any())).thenReturn(mockedReaderBuilder); + // Ensure the chain doesn't NPE after startMessageId() if parsing unexpectedly succeeds. + when(mockedReaderBuilder.startMessageId(any())).thenReturn(mockedReaderBuilder); + + Map params = new HashMap<>(); + params.put("messageId", new String[] { "invalidMessageId" }); + + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); + when(request.getParameterMap()).thenReturn(params); + + JettyServerUpgradeResponse servletUpgradeResponse = mock(JettyServerUpgradeResponse.class); + new ReaderHandler(wss, request, servletUpgradeResponse); + + verify(servletUpgradeResponse, times(1)) + .sendError(eq(HttpServletResponse.SC_BAD_REQUEST), anyString()); + } + + @Test + @SuppressWarnings("unchecked") + public void testInvalidMessageIdBytesReturnsBadRequest() throws IOException { + WebSocketService wss = mock(WebSocketService.class); + PulsarClient mockedClient = mock(PulsarClient.class); + when(wss.getPulsarClient()).thenReturn(mockedClient); + ReaderBuilder mockedReaderBuilder = mock(ReaderBuilder.class); + when(mockedClient.newReader()).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.topic(any())).thenReturn(mockedReaderBuilder); + // Ensure the chain doesn't NPE after startMessageId() if parsing unexpectedly succeeds. + when(mockedReaderBuilder.startMessageId(any())).thenReturn(mockedReaderBuilder); + + // "AQID" is valid Base64, but it doesn't decode into a valid Pulsar MessageId structure. + Map params = new HashMap<>(); + params.put("messageId", new String[] { "AQID" }); + + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); + when(request.getParameterMap()).thenReturn(params); + + JettyServerUpgradeResponse servletUpgradeResponse = mock(JettyServerUpgradeResponse.class); + new ReaderHandler(wss, request, servletUpgradeResponse); + + verify(servletUpgradeResponse, times(1)) + .sendError(eq(HttpServletResponse.SC_BAD_REQUEST), anyString()); + } + + @Test + @SuppressWarnings("unchecked") + public void testInvalidMessageIdRuntimeParseFailureReturnsBadRequest() throws IOException { + WebSocketService wss = mock(WebSocketService.class); + PulsarClient mockedClient = mock(PulsarClient.class); + when(wss.getPulsarClient()).thenReturn(mockedClient); + ReaderBuilder mockedReaderBuilder = mock(ReaderBuilder.class); + when(mockedClient.newReader()).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.topic(any())).thenReturn(mockedReaderBuilder); + // Ensure the chain doesn't NPE after startMessageId() if parsing unexpectedly succeeds. + when(mockedReaderBuilder.startMessageId(any())).thenReturn(mockedReaderBuilder); + + MessageIdData invalidBatchMessageId = new MessageIdData() + .setLedgerId(1) + .setEntryId(2) + .setBatchIndex(0) + .setBatchSize(-1); + Map params = new HashMap<>(); + params.put("messageId", new String[] { + Base64.getEncoder().encodeToString(invalidBatchMessageId.toByteArray()) }); + + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); + when(request.getParameterMap()).thenReturn(params); + + JettyServerUpgradeResponse servletUpgradeResponse = mock(JettyServerUpgradeResponse.class); + new ReaderHandler(wss, request, servletUpgradeResponse); + + verify(servletUpgradeResponse, times(1)) + .sendError(eq(HttpServletResponse.SC_BAD_REQUEST), anyString()); + } + @Test @SuppressWarnings("unchecked") public void testCreateReaderImp() throws IOException { @@ -68,7 +159,7 @@ public void testCreateReaderImp() throws IOException { when(consumerImp.getSubscription()).thenReturn(subName); when(mockedReader.getConsumer()).thenReturn(consumerImp); HttpServletRequest request = mock(HttpServletRequest.class); - when(request.getRequestURI()).thenReturn("/ws/v2/producer/persistent/my-property/my-ns/my-topic"); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); // create reader handler JettyServerUpgradeResponse servletUpgradeResponse = mock(JettyServerUpgradeResponse.class); ReaderHandler readerHandler = new ReaderHandler(wss, request, servletUpgradeResponse); @@ -97,7 +188,7 @@ public void testCreateMultipleTopicReaderImp() throws IOException { when(consumerImp.getSubscription()).thenReturn(subName); when(mockedReader.getMultiTopicsConsumer()).thenReturn(consumerImp); HttpServletRequest request = mock(HttpServletRequest.class); - when(request.getRequestURI()).thenReturn("/ws/v2/producer/persistent/my-property/my-ns/my-topic"); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); // create reader handler JettyServerUpgradeResponse servletUpgradeResponse = mock(JettyServerUpgradeResponse.class); ReaderHandler readerHandler = new ReaderHandler(wss, request, servletUpgradeResponse); @@ -122,7 +213,7 @@ public void testCreateIllegalReaderImp() throws IOException { IllegalReader illegalReader = new IllegalReader(); when(mockedReaderBuilder.create()).thenReturn(illegalReader); HttpServletRequest request = mock(HttpServletRequest.class); - when(request.getRequestURI()).thenReturn("/ws/v2/producer/persistent/my-property/my-ns/my-topic"); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); // create reader handler JettyServerUpgradeResponse servletUpgradeResponse = spy(JettyServerUpgradeResponse.class); new ReaderHandler(wss, request, servletUpgradeResponse); From c13b393a109726d0439e9e8f933b27d40e7d85a2 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Mon, 25 May 2026 23:30:50 +0800 Subject: [PATCH 105/201] [fix][broker] Fix PersistentMessageExpiryMonitor findEntryComplete() method may lose mark-delete properties in race condition (#25803) (cherry picked from commit 47eec875a81d11a030a3781cbb123e273ced5e88) --- .../PersistentMessageExpiryMonitor.java | 3 +- .../service/PersistentMessageFinderTest.java | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index 0ad8e8cfe7867..05638478c4f00 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -240,8 +240,7 @@ public void findEntryComplete(Position position, Object ctx) { } log.info("[{}][{}] Expiring all messages until position {}", topicName, subName, position); Position prevMarkDeletePos = cursor.getMarkDeletedPosition(); - cursor.asyncMarkDelete(position, cursor.getProperties(), markDeleteCallback, - cursor.getNumberOfEntriesInBacklog(false)); + cursor.asyncMarkDelete(position, null, markDeleteCallback, cursor.getNumberOfEntriesInBacklog(false)); if (!Objects.equals(cursor.getMarkDeletedPosition(), prevMarkDeletePos) && subscription != null) { subscription.updateLastMarkDeleteAdvancedTimestamp(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java index d8dc64d049023..d6591d2e0fc57 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -34,11 +35,14 @@ import io.netty.buffer.UnpooledByteBufAllocator; import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -1099,4 +1103,58 @@ public void testGetFindPositionRange_SingleClosedLedger() { assertNull(range.getRight()); assertEquals(range.getLeft(), PositionFactory.create(1, 9)); } + + @Test + @SuppressWarnings("unchecked") + void testExpireMessagesNeverLoseMarkDeleteProperties() throws Exception { + final String ledgerAndCursorName = "testExpireMessagesNeverLoseMarkDeleteProperties"; + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setRetentionSizeInMB(10); + config.setRetentionTime(1, TimeUnit.HOURS); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerAndCursorName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); + ManagedCursorImpl spyCursor = spy(cursor); + + Position pos1 = ledger.addEntry(createMessageWrittenToLedger("msg-1")); + Position pos2 = ledger.addEntry(createMessageWrittenToLedger("msg-2")); + + CountDownLatch expiryMarkDeleteEnteredLatch = new CountDownLatch(1); + CountDownLatch cursorMarkDeleteCompletedLatch = new CountDownLatch(1); + CountDownLatch expiryMarkDeleteCompletedLatch = new CountDownLatch(1); + + doAnswer(invocation -> { + Map invocationProperties = invocation.getArgument(1); + // Pause the expiry-triggered mark-delete so the user markDelete() can complete first. + if (invocationProperties == null || invocationProperties.isEmpty()) { + expiryMarkDeleteEnteredLatch.countDown(); + assertTrue(cursorMarkDeleteCompletedLatch.await(5, TimeUnit.SECONDS)); + try { + return invocation.callRealMethod(); + } finally { + expiryMarkDeleteCompletedLatch.countDown(); + } + } + + return invocation.callRealMethod(); + }).when(spyCursor) + .asyncMarkDelete(any(Position.class), nullable(Map.class), any(AsyncCallbacks.MarkDeleteCallback.class), + nullable(Object.class)); + + PersistentTopic topic = mockPersistentTopic("topicname"); + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(topic, + spyCursor.getName(), spyCursor, null); + + CompletableFuture.runAsync(() -> monitor.findEntryComplete(pos2, null)); + assertTrue(expiryMarkDeleteEnteredLatch.await(5, TimeUnit.SECONDS)); + + Map properties = new HashMap<>(); + properties.put("test-property", 1L); + spyCursor.markDelete(pos1, properties); + cursorMarkDeleteCompletedLatch.countDown(); + + assertTrue(expiryMarkDeleteCompletedLatch.await(5, TimeUnit.SECONDS)); + assertEquals(spyCursor.getMarkDeletedPosition(), pos2); + assertEquals(spyCursor.getProperties(), properties); + } } From 879cc01697e80cab65feeaccef7189e48c45751f Mon Sep 17 00:00:00 2001 From: Yike Xiao Date: Wed, 27 May 2026 23:18:58 +0800 Subject: [PATCH 106/201] [fix][bk] Fix NPE in IsolatedBookieEnsemblePlacementPolicy when policy class does not match (#25825) Co-authored-by: Claude Sonnet 4.6 (cherry picked from commit b93fe9e78f3929b52c33475aa2962a0450a7e2de) --- ...IsolatedBookieEnsemblePlacementPolicy.java | 28 ++-- ...atedBookieEnsemblePlacementPolicyTest.java | 136 ++++++++++++++++++ 2 files changed, 152 insertions(+), 12 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java index 4ef1c594be444..f2c1b5e736191 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.bookie.rackawareness; +import static java.util.Collections.emptySet; import static org.apache.pulsar.bookie.rackawareness.BookieRackAffinityMapping.METADATA_STORE_INSTANCE; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; @@ -164,11 +165,13 @@ private static Optional getEnsemblePlacementPolic return Optional.empty(); } - private static Pair, Set> getIsolationGroup( + @VisibleForTesting + Pair, Set> getIsolationGroup( EnsemblePlacementPolicyConfig ensemblePlacementPolicyConfig) { - MutablePair, Set> pair = new MutablePair<>(); - String className = IsolatedBookieEnsemblePlacementPolicy.class.getName(); - if (ensemblePlacementPolicyConfig.getPolicyClass().getName().equals(className)) { + // Retain compatibility with ZkIsolatedBookieEnsemblePlacementPolicy + Class policyClass = ensemblePlacementPolicyConfig.getPolicyClass(); + if (IsolatedBookieEnsemblePlacementPolicy.class.isAssignableFrom(policyClass)) { + MutablePair, Set> pair = new MutablePair<>(emptySet(), emptySet()); Map properties = ensemblePlacementPolicyConfig.getProperties(); String primaryIsolationGroupString = ConfigurationStringUtil .castToString(properties.getOrDefault(ISOLATION_BOOKIE_GROUPS, "")); @@ -176,21 +179,22 @@ private static Pair, Set> getIsolationGroup( .castToString(properties.getOrDefault(SECONDARY_ISOLATION_BOOKIE_GROUPS, "")); if (!primaryIsolationGroupString.isEmpty()) { pair.setLeft(Sets.newHashSet(primaryIsolationGroupString.split(","))); - } else { - pair.setLeft(Collections.emptySet()); } if (!secondaryIsolationGroupString.isEmpty()) { pair.setRight(Sets.newHashSet(secondaryIsolationGroupString.split(","))); - } else { - pair.setRight(Collections.emptySet()); } + return pair; + } else { + log.info("The ensemble placement policy class [{}] is not compatible with " + + "IsolatedBookieEnsemblePlacementPolicy, fallback to use defaultIsolationGroups", + ensemblePlacementPolicyConfig.getPolicyClass().getName()); + return defaultIsolationGroups; } - return pair; } @VisibleForTesting Set getExcludedBookiesWithIsolationGroups(int ensembleSize, - Pair, Set> isolationGroups) { + Pair, Set> isolationGroups) { Set excludedBookies = new HashSet<>(); if (isolationGroups != null && isolationGroups.getLeft().contains(PULSAR_SYSTEM_TOPIC_ISOLATION_GROUP)) { return excludedBookies; @@ -213,8 +217,8 @@ Set getExcludedBookiesWithIsolationGroups(int ensembleSize, return excludedBookies; } int totalAvailableBookiesInPrimaryGroup = 0; - Set primaryIsolationGroup = Collections.emptySet(); - Set secondaryIsolationGroup = Collections.emptySet(); + Set primaryIsolationGroup = emptySet(); + Set secondaryIsolationGroup = emptySet(); Set primaryGroupBookies = new HashSet<>(); if (isolationGroups != null) { primaryIsolationGroup = isolationGroups.getLeft(); diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java index 0dc996c7d7def..fc67395c813c4 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java @@ -42,12 +42,14 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; +import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.feature.SettableFeatureProvider; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.policies.data.BookieInfo; import org.apache.pulsar.common.policies.data.BookiesRackConfiguration; import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig; @@ -57,6 +59,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreFactory; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl; +import org.apache.pulsar.zookeeper.ZkIsolatedBookieEnsemblePlacementPolicy; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -844,6 +847,139 @@ public void testGetExcludedBookiesWithIsolationGroups() throws Exception { assertTrue(blacklist.isEmpty()); } + /** + * Regression test for the NPE reported in the stack trace below. When custom metadata carries an + * {@link EnsemblePlacementPolicyConfig} whose policy class does NOT match + * {@link IsolatedBookieEnsemblePlacementPolicy}, the old {@code getIsolationGroup()} returned a + * {@code MutablePair} with {@code null} left/right, which caused a {@link NullPointerException} in + * {@code getExcludedBookiesWithIsolationGroups} when {@code getLeft().contains(...)} was called. + * + * 

+     * java.lang.NullPointerException: Cannot invoke "java.util.Set.contains(Object)"
+     *     because the return value of "org.apache.commons.lang3.tuple.Pair.getLeft()" is null
+     *     at IsolatedBookieEnsemblePlacementPolicy.getExcludedBookiesWithIsolationGroups(...)
+     *     at IsolatedBookieEnsemblePlacementPolicy.getExcludedBookies(...)
+     *     at IsolatedBookieEnsemblePlacementPolicy.replaceBookie(...)
+     *
+ */ + @Test + public void testReplaceBookieWithNonMatchingPolicyClassShouldNotThrowNPE() throws Exception { + Map> bookieMapping = new HashMap<>(); + Map group1 = new HashMap<>(); + group1.put(BOOKIE1, BookieInfo.builder().rack("rack0").build()); + group1.put(BOOKIE2, BookieInfo.builder().rack("rack1").build()); + group1.put(BOOKIE3, BookieInfo.builder().rack("rack0").build()); + group1.put(BOOKIE4, BookieInfo.builder().rack("rack1").build()); + bookieMapping.put("group1", group1); + + store.put(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, jsonMapper.writeValueAsBytes(bookieMapping), + Optional.empty()).join(); + + IsolatedBookieEnsemblePlacementPolicy isolationPolicy = new IsolatedBookieEnsemblePlacementPolicy(); + ClientConfiguration bkClientConf = new ClientConfiguration(); + bkClientConf.setProperty(BookieRackAffinityMapping.METADATA_STORE_INSTANCE, store); + bkClientConf.setProperty(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "group1"); + isolationPolicy.initialize(bkClientConf, Optional.empty(), timer, SettableFeatureProvider.DISABLE_ALL, + NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies); + + // Use a policy class that does NOT match IsolatedBookieEnsemblePlacementPolicy. + // In the old code this caused getIsolationGroup() to return a MutablePair with null left/right, + // triggering NPE at the getLeft().contains() call in getExcludedBookiesWithIsolationGroups. + EnsemblePlacementPolicyConfig policyConfig = new EnsemblePlacementPolicyConfig( + RackawareEnsemblePlacementPolicy.class, Collections.emptyMap()); + Map customMetadata = new HashMap<>(); + customMetadata.put(EnsemblePlacementPolicyConfig.ENSEMBLE_PLACEMENT_POLICY_CONFIG, policyConfig.encode()); + + BookieId bookie1Id = new BookieSocketAddress(BOOKIE1).toBookieId(); + BookieId bookie2Id = new BookieSocketAddress(BOOKIE2).toBookieId(); + + // Must not throw NullPointerException; BKNotEnoughBookiesException is acceptable. + isolationPolicy.replaceBookie(2, 2, 2, customMetadata, + Arrays.asList(bookie1Id, bookie2Id), bookie2Id, null); + } + + /** + * Verifies that {@link IsolatedBookieEnsemblePlacementPolicy#getIsolationGroup} treats + * {@link ZkIsolatedBookieEnsemblePlacementPolicy} (a subclass) exactly like + * {@link IsolatedBookieEnsemblePlacementPolicy} itself when reading isolation groups from + * {@link EnsemblePlacementPolicyConfig} properties. + * + *

Legacy Pulsar clusters may have persisted {@code EnsemblePlacementPolicyConfig} entries whose + * {@code policyClass} field is set to {@code ZkIsolatedBookieEnsemblePlacementPolicy}. The + * {@code isAssignableFrom} check in {@code getIsolationGroup} must recognise this subclass so that + * the isolation groups are read from the stored properties rather than falling back to the + * policy-level defaults. + */ + @Test + public void testGetIsolationGroupWithZkCompatiblePolicyClass() throws Exception { + // Group1 → default isolation group configured on the policy. + // Group2 → isolation group carried inside the custom metadata (ZkIsolated class). + final String defaultGroup = "Group1"; + final String customGroup = "Group2"; + + Map> bookieMapping = new HashMap<>(); + Map group1 = new HashMap<>(); + group1.put(BOOKIE1, BookieInfo.builder().rack("rack0").build()); + group1.put(BOOKIE2, BookieInfo.builder().rack("rack0").build()); + Map group2 = new HashMap<>(); + group2.put(BOOKIE3, BookieInfo.builder().rack("rack1").build()); + group2.put(BOOKIE4, BookieInfo.builder().rack("rack1").build()); + bookieMapping.put(defaultGroup, group1); + bookieMapping.put(customGroup, group2); + + store.put(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, jsonMapper.writeValueAsBytes(bookieMapping), + Optional.empty()).join(); + + IsolatedBookieEnsemblePlacementPolicy isolationPolicy = new IsolatedBookieEnsemblePlacementPolicy(); + ClientConfiguration bkClientConf = new ClientConfiguration(); + bkClientConf.setProperty(BookieRackAffinityMapping.METADATA_STORE_INSTANCE, store); + bkClientConf.setProperty(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, defaultGroup); + isolationPolicy.initialize(bkClientConf, Optional.empty(), timer, SettableFeatureProvider.DISABLE_ALL, + NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies); + + // --- unit-level: getIsolationGroup should parse properties, not fall back to defaults --- + Map props = new HashMap<>(); + props.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, customGroup); + props.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, "secondaryGroup"); + EnsemblePlacementPolicyConfig zkConfig = new EnsemblePlacementPolicyConfig( + ZkIsolatedBookieEnsemblePlacementPolicy.class, props); + + Pair, Set> groups = isolationPolicy.getIsolationGroup(zkConfig); + assertEquals(groups.getLeft(), Sets.newHashSet(customGroup), + "primary group must be read from ZkIsolated config properties"); + assertEquals(groups.getRight(), Sets.newHashSet("secondaryGroup"), + "secondary group must be read from ZkIsolated config properties"); + + // --- integration-level: newEnsemble must select bookies from the ZkIsolated config group --- + Map placementPolicyProperties = new HashMap<>(); + placementPolicyProperties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, customGroup); + placementPolicyProperties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); + EnsemblePlacementPolicyConfig policyConfig = new EnsemblePlacementPolicyConfig( + ZkIsolatedBookieEnsemblePlacementPolicy.class, placementPolicyProperties); + Map customMetadata = new HashMap<>(); + customMetadata.put(EnsemblePlacementPolicyConfig.ENSEMBLE_PLACEMENT_POLICY_CONFIG, policyConfig.encode()); + + Set bookieIdGroup2 = new HashSet<>(); + bookieIdGroup2.add(new BookieSocketAddress(BOOKIE3).toBookieId()); + bookieIdGroup2.add(new BookieSocketAddress(BOOKIE4).toBookieId()); + + List ensemble = isolationPolicy + .newEnsemble(2, 2, 2, customMetadata, new HashSet<>()).getResult(); + assertTrue(bookieIdGroup2.containsAll(ensemble), + "ensemble should come from " + customGroup + " (ZkIsolated config), got " + ensemble); + + // Sanity-check: without custom metadata the default group1 bookies are chosen. + Set bookieIdGroup1 = new HashSet<>(); + bookieIdGroup1.add(new BookieSocketAddress(BOOKIE1).toBookieId()); + bookieIdGroup1.add(new BookieSocketAddress(BOOKIE2).toBookieId()); + List defaultEnsemble = isolationPolicy + .newEnsemble(2, 2, 2, Collections.emptyMap(), new HashSet<>()).getResult(); + assertTrue(bookieIdGroup1.containsAll(defaultEnsemble), + "default ensemble should come from " + defaultGroup + ", got " + defaultEnsemble); + } + // The policy gets the bookie info asynchronously before each query or update, when putting the bookie info into // the metadata store, the cache needs some time to receive the notification and update accordingly. private void updateBookieInfo(IsolatedBookieEnsemblePlacementPolicy isolationPolicy, byte[] bookieInfo) { From 3629ba3ca2dab191cfd4fbbd2bbc8fe6e13496f0 Mon Sep 17 00:00:00 2001 From: grishaf Date: Fri, 29 May 2026 10:13:15 +0300 Subject: [PATCH 107/201] [fix][broker] Fix non-batched null-value messages not removed during topic compaction (#25817) (cherry picked from commit 1fa9e3532b4c8978b25f16b43855948b54e95d17) --- .../compaction/AbstractTwoPhaseCompactor.java | 20 +++-- .../compaction/EventTimeOrderCompactor.java | 14 +--- .../pulsar/compaction/CompactionTest.java | 74 +++++++++++++++++++ 3 files changed, 93 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java index bdcdb50eb5f24..095c4861ffe80 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java @@ -449,14 +449,24 @@ private CompletableFuture addToCompactedLedger(LedgerHandle lh, RawMessage return bkf; } + /** + * Extract the partition key and the payload size for a non-batch message. + * + * @return a pair of (partitionKey, payloadSize), or null if the message has no partition key. + */ protected Pair extractKeyAndSize(RawMessage m, MessageMetadata msgMetadata) { - ByteBuf headersAndPayload = m.getHeadersAndPayload(); if (msgMetadata.hasPartitionKey()) { - int size = headersAndPayload.readableBytes(); - if (msgMetadata.hasUncompressedSize()) { - size = msgMetadata.getUncompressedSize(); + int payloadSize; + if (msgMetadata.hasNullValue() && msgMetadata.isNullValue()) { + payloadSize = 0; + } else if (msgMetadata.hasUncompressedSize()) { + payloadSize = msgMetadata.getUncompressedSize(); + } else { + ByteBuf headersAndPayload = m.getHeadersAndPayload().duplicate(); + Commands.skipMessageMetadata(headersAndPayload); + payloadSize = headersAndPayload.readableBytes(); } - return Pair.of(msgMetadata.getPartitionKey(), size); + return Pair.of(msgMetadata.getPartitionKey(), payloadSize); } else { return null; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java index db129b54533a8..ad6b5f28d306f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.compaction; -import io.netty.buffer.ByteBuf; import java.io.IOException; import java.util.List; import java.util.Map; @@ -139,17 +138,12 @@ protected boolean compactBatchMessage(String topic, Map keyAndSize = extractKeyAndSize(m, metadata); + if (keyAndSize == null) { return null; } + return new MessageCompactionData(m.getMessageId(), keyAndSize.getLeft(), + keyAndSize.getRight(), metadata.getEventTime()); } private List extractMessageCompactionDataFromBatch(RawMessage msg, MessageMetadata metadata) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 5fc6416ee9837..e5ccec35e9ffa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -640,6 +640,80 @@ public void testBatchMessageWithNullValue() throws Exception { assertEquals(messages.get(2).getKey(), "key5"); } + /** + * Write raw non-batch entries directly to the managed ledger without + * uncompressedSize, as seen with some non-Java clients. Verifies that + * null-value tombstones remove keys during compaction. + */ + @Test + public void testNonBatchedMessageWithNullValue() throws Exception { + String topic = "persistent://my-tenant/my-ns/non-batched-message-with-null-value"; + + admin.topics().createNonPartitionedTopic(topic); + pulsarClient.newConsumer().topic(topic).subscriptionName("sub1") + .receiverQueueSize(1).readCompacted(true).subscribe().close(); + + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + + long seqId = 0; + + // key1: value then null-value tombstone + ml.addEntry(buildNonBatchEntry("key1", "my-message-1".getBytes(), seqId++)); + ml.addEntry(buildNonBatchEntry("key1", null, seqId++)); + + // key2: value only (should survive) + ml.addEntry(buildNonBatchEntry("key2", "my-message-3".getBytes(), seqId++)); + + // key3: value then null-value tombstone + ml.addEntry(buildNonBatchEntry("key3", "my-message-4".getBytes(), seqId++)); + ml.addEntry(buildNonBatchEntry("key3", null, seqId++)); + + // key4: value only (should survive) + ml.addEntry(buildNonBatchEntry("key4", "my-message-6".getBytes(), seqId++)); + + compact(topic); + + List> messages = new ArrayList<>(); + try (Consumer consumer = pulsarClient.newConsumer().topic(topic) + .subscriptionName("sub1").receiverQueueSize(1).readCompacted(true).subscribe()) { + while (true) { + Message message = consumer.receive(5, TimeUnit.SECONDS); + if (message == null) { + break; + } + messages.add(message); + } + } + + assertEquals(messages.size(), 2); + assertEquals(messages.get(0).getKey(), "key2"); + assertEquals(messages.get(1).getKey(), "key4"); + } + + private byte[] buildNonBatchEntry(String key, byte[] payload, long sequenceId) { + org.apache.pulsar.common.api.proto.MessageMetadata metadata = + new org.apache.pulsar.common.api.proto.MessageMetadata(); + metadata.setPartitionKey(key); + metadata.setPublishTime(System.currentTimeMillis()); + metadata.setProducerName("test-non-batch"); + metadata.setSequenceId(sequenceId); + if (payload == null) { + metadata.setNullValue(true); + } + ByteBuf payloadBuf = io.netty.buffer.Unpooled.wrappedBuffer( + payload != null ? payload : new byte[0]); + ByteBuf entry = org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload( + org.apache.pulsar.common.protocol.Commands.ChecksumType.Crc32c, + metadata, payloadBuf); + byte[] bytes = new byte[entry.readableBytes()]; + entry.readBytes(bytes); + entry.release(); + payloadBuf.release(); + return bytes; + } + @Test public void testWholeBatchCompactedOut() throws Exception { String topic = "persistent://my-tenant/my-ns/whole-batch-compacted-out"; From 4f8ffc8291d7f8bc40e5250e27c7926339445dd1 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 24 Apr 2026 10:53:23 +0800 Subject: [PATCH 108/201] [improve][client] Best-effort retry for individual/batch-index acks on send failure when ackReceiptEnabled=false (#25525) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> (cherry picked from commit fae3df958f74f2afb0ba398b69d94fd3e6358c4f) --- .../apache/pulsar/client/impl/ClientCnx.java | 26 +++--- ...sistentAcknowledgmentsGroupingTracker.java | 75 +++++++++++++-- .../AcknowledgementsGroupingTrackerTest.java | 93 ++++++++++++++++++- .../ClientCnxRequestTimeoutQueueTest.java | 1 + .../client/impl/ClientTestFixtures.java | 5 + 5 files changed, 173 insertions(+), 27 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index eb3162e8ffa80..a51e3ac8796ee 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -1054,27 +1054,23 @@ CompletableFuture sendRequestWithId(ByteBuf cmd, long requestI } private void sendRequestAndHandleTimeout(ByteBuf requestMessage, long requestId, - RequestType requestType, boolean flush, - TimedCompletableFuture future) { + RequestType requestType, boolean flush, + TimedCompletableFuture future) { pendingRequests.put(requestId, future); - if (flush) { - ctx.writeAndFlush(requestMessage).addListener(writeFuture -> { - if (!writeFuture.isSuccess()) { - if (pendingRequests.remove(requestId, future) && !future.isDone()) { - log.warn("{} Failed to send {} to broker: {}", ctx.channel(), - requestType.getDescription(), writeFuture.cause().getMessage()); - future.completeExceptionally(writeFuture.cause()); - } + (flush ? ctx.writeAndFlush(requestMessage) : ctx.write(requestMessage)).addListener(writeFuture -> { + if (!writeFuture.isSuccess()) { + if (pendingRequests.remove(requestId, future) && !future.isDone()) { + log.warn("{} Failed to send {} to broker: {}", ctx.channel(), + requestType.getDescription(), writeFuture.cause().getMessage()); + future.completeExceptionally(writeFuture.cause()); } - }); - } else { - ctx.write(requestMessage, ctx().voidPromise()); - } + } + }); requestTimeoutQueue.add(new RequestTime(requestId, requestType)); } private CompletableFuture sendRequestAndHandleTimeout(ByteBuf requestMessage, long requestId, - RequestType requestType, boolean flush) { + RequestType requestType, boolean flush) { TimedCompletableFuture future = new TimedCompletableFuture<>(); sendRequestAndHandleTimeout(requestMessage, requestId, requestType, flush, future); return future; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java index 0598dc4fb3626..0a366a759d3f4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -279,6 +280,11 @@ private CompletableFuture doIndividualAckAsync(MessageIdAdv messageId) { return CompletableFuture.completedFuture(null); } + @VisibleForTesting + int getPendingIndividualAcksSize() { + return pendingIndividualAcks.size(); + } + private CompletableFuture doIndividualBatchAck(MessageIdAdv batchMessageId, Map properties) { if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) { @@ -405,7 +411,7 @@ private CompletableFuture doImmediateBatchIndexAck(MessageIdAdv msgId, int } CompletableFuture completableFuture = newMessageAckCommandAndWrite(cnx, consumer.consumerId, - msgId.getLedgerId(), msgId.getEntryId(), bitSet, ackType, properties, true, null, null); + msgId.getLedgerId(), msgId.getEntryId(), bitSet, ackType, properties, true, null, null, null); bitSet.recycle(); return completableFuture; } @@ -441,13 +447,15 @@ private void flushAsync(ClientCnx cnx) { newMessageAckCommandAndWrite(cnx, consumer.consumerId, messageId.getLedgerId(), messageId.getEntryId(), lastCumulativeAckToFlush.getBitSetRecyclable(), AckType.Cumulative, Collections.emptyMap(), false, - (TimedCompletableFuture) this.currentCumulativeAckFuture, null); + (TimedCompletableFuture) this.currentCumulativeAckFuture, null, null); this.consumer.unAckedChunkedMessageIdSequenceMap.remove(messageId); } // Flush all individual acks List> entriesToAck = new ArrayList<>(pendingIndividualAcks.size() + pendingIndividualBatchIndexAcks.size()); + List individualAcksToFlush = new ArrayList<>(pendingIndividualAcks.size()); + Map chunkedMessageIdsToRestore = new HashMap<>(); if (!pendingIndividualAcks.isEmpty()) { if (Commands.peerSupportsMultiMessageAcknowledgment(cnx.getRemoteEndpointProtocolVersion())) { // We can send 1 single protobuf command with all individual acks @@ -456,11 +464,13 @@ private void flushAsync(ClientCnx cnx) { if (msgId == null) { break; } + individualAcksToFlush.add(msgId); // if messageId is checked then all the chunked related to that msg also processed so, ack all of // them MessageIdImpl[] chunkMsgIds = this.consumer.unAckedChunkedMessageIdSequenceMap.get(msgId); if (chunkMsgIds != null && chunkMsgIds.length > 1) { + chunkedMessageIdsToRestore.put(msgId, chunkMsgIds); for (MessageIdImpl cMsgId : chunkMsgIds) { if (cMsgId != null) { entriesToAck.add(Triple.of(cMsgId.getLedgerId(), cMsgId.getEntryId(), null)); @@ -479,14 +489,16 @@ private void flushAsync(ClientCnx cnx) { if (msgId == null) { break; } + individualAcksToFlush.add(msgId); newMessageAckCommandAndWrite(cnx, consumer.consumerId, msgId.getLedgerId(), msgId.getEntryId(), null, AckType.Individual, Collections.emptyMap(), false, - null, null); + null, null, () -> restoreIndividualAck(msgId, null)); shouldFlush = true; } } } + List> batchIndexAcksToFlush = new ArrayList<>(); while (true) { Map.Entry entry = pendingIndividualBatchIndexAcks.pollFirstEntry(); @@ -494,6 +506,7 @@ private void flushAsync(ClientCnx cnx) { // The entry has been removed in a different thread break; } + batchIndexAcksToFlush.add(entry); entriesToAck.add(Triple.of( entry.getKey().getLedgerId(), entry.getKey().getEntryId(), entry.getValue())); } @@ -502,7 +515,9 @@ private void flushAsync(ClientCnx cnx) { newMessageAckCommandAndWrite(cnx, consumer.consumerId, 0L, 0L, null, AckType.Individual, null, true, - (TimedCompletableFuture) currentIndividualAckFuture, entriesToAck); + (TimedCompletableFuture) currentIndividualAckFuture, entriesToAck, + () -> restoreIndividualAndBatchIndexAcks(individualAcksToFlush, chunkedMessageIdsToRestore, + batchIndexAcksToFlush)); shouldFlush = true; } @@ -547,19 +562,19 @@ private CompletableFuture newImmediateAckAndFlush(long consumerId, Message } } completableFuture = newMessageAckCommandAndWrite(cnx, consumer.consumerId, 0L, 0L, - null, ackType, null, true, null, entriesToAck); + null, ackType, null, true, null, entriesToAck, null); } else { // if don't support multi message ack, it also support ack receipt, so we should not think about the // ack receipt in this logic for (MessageIdImpl cMsgId : chunkMsgIds) { newMessageAckCommandAndWrite(cnx, consumerId, cMsgId.getLedgerId(), cMsgId.getEntryId(), - bitSet, ackType, map, true, null, null); + bitSet, ackType, map, true, null, null, null); } completableFuture = CompletableFuture.completedFuture(null); } } else { completableFuture = newMessageAckCommandAndWrite(cnx, consumerId, msgId.getLedgerId(), msgId.getEntryId(), - bitSet, ackType, map, true, null, null); + bitSet, ackType, map, true, null, null, null); } return completableFuture; } @@ -569,7 +584,8 @@ private CompletableFuture newMessageAckCommandAndWrite( long entryId, BitSetRecyclable ackSet, AckType ackType, Map properties, boolean flush, TimedCompletableFuture timedCompletableFuture, - List> entriesToAck) { + List> entriesToAck, + Runnable writeFailureCallback) { if (consumer.isAckReceiptEnabled()) { final long requestId = consumer.getClient().newRequestId(); final ByteBuf cmd; @@ -611,14 +627,53 @@ private CompletableFuture newMessageAckCommandAndWrite( cmd = Commands.newMultiMessageAck(consumerId, entriesToAck, -1); } if (flush) { - cnx.ctx().writeAndFlush(cmd, cnx.ctx().voidPromise()); + if (writeFailureCallback == null) { + cnx.ctx().writeAndFlush(cmd, cnx.ctx().voidPromise()); + } else { + cnx.ctx().writeAndFlush(cmd).addListener(writeFuture -> { + if (!writeFuture.isSuccess()) { + writeFailureCallback.run(); + } + }); + } } else { - cnx.ctx().write(cmd, cnx.ctx().voidPromise()); + if (writeFailureCallback == null) { + cnx.ctx().write(cmd, cnx.ctx().voidPromise()); + } else { + cnx.ctx().write(cmd).addListener(writeFuture -> { + if (!writeFuture.isSuccess()) { + writeFailureCallback.run(); + } + }); + } } return CompletableFuture.completedFuture(null); } } + private void restoreIndividualAck(MessageIdAdv messageId, @Nullable MessageIdImpl[] chunkMsgIds) { + pendingIndividualAcks.add(messageId); + restoreChunkedMessageIds(messageId, chunkMsgIds); + } + + private void restoreIndividualAndBatchIndexAcks(List messageIds, + Map chunkMsgIds, + List> batchIndexAcks) { + pendingIndividualAcks.addAll(messageIds); + chunkMsgIds.forEach(this::restoreChunkedMessageIds); + batchIndexAcks.forEach(entry -> pendingIndividualBatchIndexAcks.merge(entry.getKey(), entry.getValue(), + (currentValue, valueToRestore) -> { + currentValue.and(valueToRestore); + return currentValue; + })); + } + + private void restoreChunkedMessageIds(MessageIdAdv messageId, @Nullable MessageIdImpl[] chunkMsgIds) { + if (chunkMsgIds != null) { + consumer.unAckedChunkedMessageIdSequenceMap.putIfAbsent(messageId, chunkMsgIds); + } + } + public Optional acquireReadLock() { Optional optionalLock = Optional.ofNullable(consumer.isAckReceiptEnabled() ? lock.readLock() : null); optionalLock.ifPresent(Lock::lock); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java index 0b91d86064cde..8bb85b054a3e3 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -28,9 +29,13 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.util.ReferenceCountUtil; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.GenericFutureListener; import java.util.ArrayList; import java.util.BitSet; import java.util.Collections; @@ -56,6 +61,8 @@ public class AcknowledgementsGroupingTrackerTest { private ConsumerImpl consumer; private EventLoopGroup eventLoopGroup; private AtomicBoolean returnCnx = new AtomicBoolean(true); + private ChannelHandlerContext successCtx; + private AtomicBoolean failAckCommandSend = new AtomicBoolean(false); @BeforeClass public void setup() throws NoSuchFieldException, IllegalAccessException { @@ -70,9 +77,9 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { doReturn(new ConsumerStatsRecorderImpl()).when(consumer).getStats(); doReturn(UnAckedMessageTracker.UNACKED_MESSAGE_TRACKER_DISABLED) .when(consumer).getUnAckedMessageTracker(); - ChannelHandlerContext ctx = ClientTestFixtures.mockChannelHandlerContext(); + successCtx = ClientTestFixtures.mockChannelHandlerContext(); doAnswer(invocation -> returnCnx.get() ? cnx : null).when(consumer).getClientCnx(); - doReturn(ctx).when(cnx).ctx(); + doReturn(successCtx).when(cnx).ctx(); } @DataProvider(name = "isNeedReceipt") @@ -323,6 +330,60 @@ public void testAckTrackerMultiAck(boolean isNeedReceipt) { tracker.close(); } + @Test + public void testFlushRetainsPendingIndividualAckOnSendFailureWithoutAckReceipt() throws Exception { + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setAcknowledgementsGroupTimeMicros(TimeUnit.SECONDS.toMicros(10)); + conf.setAckReceiptEnabled(false); + doReturn(false).when(consumer).isAckReceiptEnabled(); + PersistentAcknowledgmentsGroupingTracker tracker = + new PersistentAcknowledgmentsGroupingTracker(consumer, conf, eventLoopGroup); + + MessageIdImpl msg1 = new MessageIdImpl(5, 1, 0); + tracker.addAcknowledgment(msg1, AckType.Individual, Collections.emptyMap()); + assertEquals(tracker.getPendingIndividualAcksSize(), 1); + + doReturn(createFailedChannelHandlerContext()).when(cnx).ctx(); + + tracker.flush(); + + assertTrue(tracker.isDuplicate(msg1)); + assertEquals(tracker.getPendingIndividualAcksSize(), 1); + + doReturn(successCtx).when(cnx).ctx(); + + tracker.flush(); + + assertFalse(tracker.isDuplicate(msg1)); + assertEquals(tracker.getPendingIndividualAcksSize(), 0); + tracker.close(); + } + + @Test + public void testFlushFailsAckFutureOnSendFailureWithAckReceipt() throws Exception { + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setAcknowledgementsGroupTimeMicros(TimeUnit.SECONDS.toMicros(10)); + conf.setAckReceiptEnabled(true); + doReturn(true).when(consumer).isAckReceiptEnabled(); + PersistentAcknowledgmentsGroupingTracker tracker = + new PersistentAcknowledgmentsGroupingTracker(consumer, conf, eventLoopGroup); + + MessageIdImpl msg1 = new MessageIdImpl(5, 1, 0); + CompletableFuture ackFuture = + tracker.addAcknowledgment(msg1, AckType.Individual, Collections.emptyMap()); + assertEquals(tracker.getPendingIndividualAcksSize(), 1); + + failAckCommandSend.set(true); + tracker.flush(); + + assertTrue(ackFuture.isCompletedExceptionally()); + assertFalse(tracker.isDuplicate(msg1)); + assertEquals(tracker.getPendingIndividualAcksSize(), 0); + + failAckCommandSend.set(false); + tracker.close(); + } + @Test(dataProvider = "isNeedReceipt") public void testBatchAckTrackerMultiAck(boolean isNeedReceipt) throws Exception { ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); @@ -463,12 +524,40 @@ public ClientCnxTest(ClientConfigurationData conf, EventLoopGroup eventLoopGroup @Override public CompletableFuture newAckForReceipt(ByteBuf request, long requestId) { + if (failAckCommandSend.get()) { + return CompletableFuture.failedFuture(new RuntimeException("ack send failed")); + } return CompletableFuture.completedFuture(null); } @Override public void newAckForReceiptWithFuture(ByteBuf request, long requestId, TimedCompletableFuture future) { + if (failAckCommandSend.get()) { + future.completeExceptionally(new RuntimeException("ack send failed")); + } } } + + private ChannelHandlerContext createFailedChannelHandlerContext() { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + ChannelFuture listenerFuture = mock(ChannelFuture.class); + ChannelFuture failedFuture = mock(ChannelFuture.class); + when(failedFuture.isSuccess()).thenReturn(false); + when(failedFuture.cause()).thenReturn(new RuntimeException("ack send failed")); + doAnswer(invocation -> { + GenericFutureListener> listener = invocation.getArgument(0); + listener.operationComplete(failedFuture); + return listenerFuture; + }).when(listenerFuture).addListener(any()); + doAnswer(invocation -> { + ReferenceCountUtil.release(invocation.getArgument(0)); + return listenerFuture; + }).when(ctx).write(any()); + doAnswer(invocation -> { + ReferenceCountUtil.release(invocation.getArgument(0)); + return listenerFuture; + }).when(ctx).writeAndFlush(any()); + return ctx; + } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java index c4dab1ad351ab..7e5d98b136a40 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java @@ -64,6 +64,7 @@ void setupClientCnx() throws Exception { ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); when(ctx.writeAndFlush(any())).thenAnswer(args -> mock(ChannelFuture.class)); + when(ctx.write(any())).thenAnswer(args -> mock(ChannelFuture.class)); when(ctx.channel()).thenReturn(channel); when(channel.remoteAddress()).thenReturn(new InetSocketAddress(1234)); cnx.channelActive(ctx); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java index e4ab5da96f649..c8bcfc65a955c 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java @@ -151,6 +151,11 @@ public static ChannelHandlerContext mockChannelHandlerContext() { }).when(listenerFuture).addListener(any()); // handle write and writeAndFlush methods so that the input message is released + doAnswer(invocation -> { + Object msg = invocation.getArgument(0); + ReferenceCountUtil.release(msg); + return listenerFuture; + }).when(ctx).write(any()); doAnswer(invocation -> { Object msg = invocation.getArgument(0); ReferenceCountUtil.release(msg); From 8c38d72b2aa2e158f31345bcfadc21e6ae7e8e5b Mon Sep 17 00:00:00 2001 From: zhenJiangWang Date: Wed, 29 Apr 2026 10:47:18 +0800 Subject: [PATCH 109/201] [fix][broker] Fix precision loss in DataSketchesSummaryLogger by replacing LongAdder with DoubleAdder for sum accumulation (#25594) (cherry picked from commit 00577a5438bbb3d959e5ca2ea037e760219b95d8) --- .../prometheus/metrics/DataSketchesSummaryLogger.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java index 42c189d4bf3a7..7495f057aa007 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java @@ -22,6 +22,7 @@ import com.yahoo.sketches.quantiles.DoublesUnion; import com.yahoo.sketches.quantiles.DoublesUnionBuilder; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.DoubleAdder; import java.util.concurrent.atomic.LongAdder; public class DataSketchesSummaryLogger { @@ -37,7 +38,7 @@ public class DataSketchesSummaryLogger { */ private volatile DoublesSketch values; private final LongAdder countAdder = new LongAdder(); - private final LongAdder sumAdder = new LongAdder(); + private final DoubleAdder sumAdder = new DoubleAdder(); public DataSketchesSummaryLogger() { this.current = new ThreadLocalAccessor(); @@ -48,7 +49,7 @@ public void registerEvent(long eventLatency, TimeUnit unit) { double valueMillis = unit.toMicros(eventLatency) / 1000.0; countAdder.increment(); - sumAdder.add((long) valueMillis); + sumAdder.add(valueMillis); current.getLocalData().updateSuccess(valueMillis); } @@ -69,7 +70,7 @@ public long getCount() { return countAdder.sum(); } - public long getSum() { + public double getSum() { return sumAdder.sum(); } From 5c749e2051e2e3bd8848183ecba9ef26c191a65a Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 30 Apr 2026 06:18:54 -0700 Subject: [PATCH 110/201] [fix][broker] Fix stuck chunks in SharedConsumerAssignor permit tracking (#25620) (cherry picked from commit 759a5f520f81ba45caef9bd94a79997855a695fd) --- .../pulsar/broker/service/SharedConsumerAssignor.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java index bbf8dfd2b10fc..a317ad7560b23 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java @@ -89,7 +89,7 @@ public Map> assign(final List if (metadata == null || !metadata.hasUuid() || !metadata.hasChunkId() || !metadata.hasNumChunksFromMsg()) { consumerToEntries.computeIfAbsent(consumer, __ -> new ArrayList<>()).add(entryAndMetadata); } else { - final Consumer consumerForUuid = getConsumerForUuid(metadata, consumer, availablePermits); + final Consumer consumerForUuid = getConsumerForUuid(metadata, consumer); if (consumerForUuid == null) { unassignedMessageProcessor.accept(entryAndMetadata); continue; @@ -120,9 +120,7 @@ private Consumer getConsumer(final int numConsumers) { return null; } - private Consumer getConsumerForUuid(final MessageMetadata metadata, - final Consumer defaultConsumer, - final int currentAvailablePermits) { + private Consumer getConsumerForUuid(final MessageMetadata metadata, final Consumer defaultConsumer) { final String uuid = metadata.getUuid(); Consumer consumer = uuidToConsumer.get(uuid); if (consumer == null) { @@ -141,7 +139,9 @@ private Consumer getConsumerForUuid(final MessageMetadata metadata, // The last chunk is received, we should remove the cache uuidToConsumer.remove(uuid); } - consumerToPermits.put(consumer, currentAvailablePermits - 1); + // Decrement target consumer's permits, not the loop's local availablePermits — on a cache + // redirect those track different consumers. + consumerToPermits.put(consumer, permits - 1); return consumer; } } From bb54a9ba7a6c96a39b1b7ad2cc44aad7da47dcf7 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 20 May 2026 06:48:58 -0700 Subject: [PATCH 111/201] [fix][client] Reset higher-index states on recovery in SameAuthParamsLookupAutoClusterFailover (#25826) (cherry picked from commit 4229e197479e8ea10164c30abf95c41ceff2b344) --- ...meAuthParamsLookupAutoClusterFailover.java | 11 ++++ ...thParamsLookupAutoClusterFailoverTest.java | 63 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java index 5ae7b96906e8c..448dd95c4fdd9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java @@ -261,6 +261,17 @@ private void updateServiceUrl(int targetIndex) { try { pulsarClient.updateServiceUrl(targetUrl); pulsarClient.reloadLookUp(); + // When recovering to a higher-priority service, the check loop will only probe + // indices 0..targetIndex going forward. Any transient state (e.g., PreFail from + // a single timed-out probe) at higher indices would become stuck because those + // indices are no longer probed. Reset them so they start fresh if a future + // failover needs to consider them again. + if (targetIndex < currentPulsarServiceIndex) { + for (int i = targetIndex + 1; i < pulsarServiceStateArray.length; i++) { + pulsarServiceStateArray[i] = PulsarServiceState.Healthy; + checkCounterArray[i].setValue(0); + } + } currentPulsarServiceIndex = targetIndex; } catch (Exception e) { log.error("Failed to {}", logMsg, e); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java index a2470752fd931..d872dfc5672ec 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -167,6 +167,69 @@ public void testNoSpuriousRecoveryBounceAfterFailover() throws Exception { "Should stay at index 2, not bounce to index 1")); } + /** + * Reproduces the bug fixed by resetting state of higher-priority-than-target indices on + * recovery. The check loop only probes indices 0..currentPulsarServiceIndex, so if a + * higher index was left in a transient state (e.g., PreFail from a single timed-out + * probe) at the moment of recovery, it would stay there forever because no future probe + * ever visits it. + * + *

Scenario: failover 0 -> 2 (state[2]=Healthy). url1 recovers and is about to trigger + * recovery 2 -> 1. On the same check cycle that promotes state[1] to Healthy, url2 sees + * one transient probe failure that flips state[2] from Healthy to PreFail. The recovery + * fires (current 2 -> 1) and from that point on the loop only probes indices 0 and 1. + * + *

Without the fix, state[2] stays at PreFail forever. With the fix, state[2] is + * reset to Healthy when the recovery transition runs. + */ + @Test(timeOut = 30000) + public void testRecoveryResetsHigherIndexStaleState() throws Exception { + // url0 down, url1 down, url2 up. + setLookupResult(URL0, false); + setLookupResult(URL1, false); + setLookupResult(URL2, true); + + // Pre-set state[0]=Failed and trigger failover 0 -> 2. + runOnExecutor(() -> { + stateArray[0] = PulsarServiceState.Failed; + counterArray[0].setValue(0); + }); + runCheckCycle(); + runOnExecutor(() -> { + assertEquals(failover.getCurrentPulsarServiceIndex(), 2); + assertEquals(stateArray[1], PulsarServiceState.Failed); + assertEquals(stateArray[2], PulsarServiceState.Healthy); + }); + + // url1 becomes healthy; first check cycle moves state[1] Failed -> PreRecover. + setLookupResult(URL1, true); + runCheckCycle(); + runOnExecutor(() -> { + assertEquals(failover.getCurrentPulsarServiceIndex(), 2); + assertEquals(stateArray[1], PulsarServiceState.PreRecover); + assertEquals(stateArray[2], PulsarServiceState.Healthy); + }); + + // On the cycle that completes recovery (state[1] PreRecover -> Healthy and triggers + // updateServiceUrl(1)), inject a single failed probe at url2 so state[2] flips + // Healthy -> PreFail just before the index transition. + setLookupResult(URL2, false); + runCheckCycle(); + + // After recovery to index 1: + // - With the fix: state[2] is reset to Healthy on the transition. + // - Without the fix: state[2] is stuck at PreFail forever — the check loop now + // only iterates 0..1 and never visits index 2 again. + runOnExecutor(() -> { + assertEquals(failover.getCurrentPulsarServiceIndex(), 1); + assertEquals(stateArray[1], PulsarServiceState.Healthy); + assertEquals(stateArray[2], PulsarServiceState.Healthy, + "state[2] should be reset to Healthy on recovery, not stuck at PreFail"); + assertEquals(counterArray[2].intValue(), 0, + "counter[2] should be reset to 0 on recovery"); + }); + } + /** * Verifies that recovery still works correctly for a service that was marked Failed * by findFailoverTo, once that service becomes available again. From e6faf75cf76f35bce0dc48839c9c51c5e96d7a5d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 1 Jun 2026 23:59:29 +0300 Subject: [PATCH 112/201] [fix][test][branch-4.2] Fix PersistentMessageExpiryMonitorTest - master branch contains this change - the second parameter will also match a null value --- .../mledger/impl/PersistentMessageExpiryMonitorTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/PersistentMessageExpiryMonitorTest.java b/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/PersistentMessageExpiryMonitorTest.java index de7d87e4293c1..c2163a1c1309e 100644 --- a/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/PersistentMessageExpiryMonitorTest.java +++ b/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/PersistentMessageExpiryMonitorTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; import static org.testng.AssertJUnit.assertEquals; -import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -100,7 +99,7 @@ void testConcurrentlyExpireMessages() throws Exception { } }); return true; - }).when(spyCursor).asyncMarkDelete(any(Position.class), any(Map.class), + }).when(spyCursor).asyncMarkDelete(any(Position.class), any(), any(AsyncCallbacks.MarkDeleteCallback.class), any()); doAnswer(invocationOnMock -> { calledFindPositionCount.incrementAndGet(); From 35b4581fb0a735eeb64b652c381742470251c90d Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 30 Mar 2026 15:53:41 -0700 Subject: [PATCH 113/201] [fix][test] Fix flaky testMsgDropStat in NonPersistentTopicTest (#25426) (cherry picked from commit e9630c60e8bff865e1bfeb2ed26a6c1d65f040b4) --- .../client/api/NonPersistentTopicTest.java | 54 +++++++++---------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java index 284e6a6892856..10919bd37c694 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java @@ -59,7 +59,6 @@ import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.impl.ConsumerImpl; -import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.client.impl.PartitionedProducerImpl; import org.apache.pulsar.client.impl.ProducerImpl; @@ -886,30 +885,31 @@ public void testMsgDropStat() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(threads); byte[] msgData = "testData".getBytes(); + NonPersistentTopic topic = + (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); + /* - * Trigger at least one publisher drop through concurrent send() calls. + * Send concurrent bursts until publisher AND subscription drop rates are all > 0. + * + * Each burst uses a CyclicBarrier so all threads send simultaneously. With + * maxConcurrentNonPersistentMessagePerConnection = 0, ServerCnx drops overlapping + * sends (publisher drops). Once subscriber queues (size 1) are full, the dispatcher + * also drops delivered messages (subscription drops). * - * Uses CyclicBarrier to ensure all threads send simultaneously, creating overlap. - * With maxConcurrentNonPersistentMessagePerConnection = 0, ServerCnx#handleSend - * drops any send while another is in-flight, returning MessageId with entryId = -1. - * Awaitility repeats whole bursts (bounded to 20s) until a drop is observed. + * IMPORTANT: updateRates() calls Rate.calculateRate() which resets counters via + * sumThenReset(). We must keep sending fresh bursts so each updateRates() call + * sees new drops, rather than retrying with stale (reset) counters. */ - AtomicBoolean publisherDropSeen = new AtomicBoolean(false); - Awaitility.await().atMost(Duration.ofSeconds(20)).until(() -> { + Awaitility.await().atMost(Duration.ofSeconds(20)).pollInterval(Duration.ofMillis(100)).until(() -> { CyclicBarrier barrier = new CyclicBarrier(threads); CountDownLatch completionLatch = new CountDownLatch(threads); AtomicReference error = new AtomicReference<>(); - publisherDropSeen.set(false); for (int i = 0; i < threads; i++) { executor.submit(() -> { try { barrier.await(); - MessageId msgId = producer.send(msgData); - // Publisher drop is signaled by MessageIdImpl.entryId == -1 - if (msgId instanceof MessageIdImpl && ((MessageIdImpl) msgId).getEntryId() == -1) { - publisherDropSeen.set(true); - } + producer.send(msgData); } catch (Throwable t) { if (t instanceof InterruptedException) { Thread.currentThread().interrupt(); @@ -921,27 +921,23 @@ public void testMsgDropStat() throws Exception { }); } - // Wait for all sends to complete. - assertTrue(completionLatch.await(20, TimeUnit.SECONDS)); - - assertNull(error.get(), "Concurrent send encountered an exception"); - return publisherDropSeen.get(); - }); - - assertTrue(publisherDropSeen.get(), "Expected at least one publisher drop (entryId == -1)"); - - NonPersistentTopic topic = - (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); + completionLatch.await(20, TimeUnit.SECONDS); + if (error.get() != null) { + return false; + } - Awaitility.await().ignoreExceptions().untilAsserted(() -> { pulsar.getBrokerService().updateRates(); NonPersistentTopicStats stats = topic.getStats(false, false, false); + if (stats.getPublishers().isEmpty()) { + return false; + } NonPersistentPublisherStats npStats = stats.getPublishers().get(0); NonPersistentSubscriptionStats sub1Stats = stats.getSubscriptions().get("subscriber-1"); NonPersistentSubscriptionStats sub2Stats = stats.getSubscriptions().get("subscriber-2"); - assertTrue(npStats.getMsgDropRate() > 0); - assertTrue(sub1Stats.getMsgDropRate() > 0); - assertTrue(sub2Stats.getMsgDropRate() > 0); + return sub1Stats != null && sub2Stats != null + && npStats.getMsgDropRate() > 0 + && sub1Stats.getMsgDropRate() > 0 + && sub2Stats.getMsgDropRate() > 0; }); } finally { From cfdb73f9a2296ce5976b16082d299007cbf09329 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Wed, 3 Jun 2026 02:47:24 +0800 Subject: [PATCH 114/201] [fix][meta] Fix PulsarZooKeeperClient async addWatch callback retry behavior (#25913) (cherry picked from commit be9f97ac0f833f2dc74dc0f4538e647f7376461f) --- .../metadata/impl/PulsarZooKeeperClient.java | 10 ++-- .../pulsar/metadata/MetadataStoreTest.java | 58 +++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java index 6a995f20e745a..462df69b2ea2a 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java @@ -1163,7 +1163,7 @@ public String toString() { } @Override - public void addWatch(String basePath, Watcher watcher, AddWatchMode mode, VoidCallback cb, Object ctx) { + public void addWatch(String basePath, Watcher watcher, AddWatchMode mode, VoidCallback cb, Object context) { final Runnable proc = new ZkRetryRunnable(operationRetryPolicy, rateLimiter, setStats) { final VoidCallback vCb = new VoidCallback() { @@ -1174,7 +1174,7 @@ public void processResult(int rc, String path, Object ctx) { if (allowRetry(worker, rc)) { backOffAndRetry(that, worker.nextRetryWaitTime()); } else { - vCb.processResult(rc, basePath, ctx); + cb.processResult(rc, path, context); } } @@ -1184,15 +1184,15 @@ public void processResult(int rc, String path, Object ctx) { void zkRun() { ZooKeeper zkHandle = zk.get(); if (null == zkHandle) { - PulsarZooKeeperClient.super.addWatch(basePath, watcher, mode, cb, ctx); + PulsarZooKeeperClient.super.addWatch(basePath, watcher, mode, vCb, worker); } else { - zkHandle.addWatch(basePath, watcher, mode, cb, ctx); + zkHandle.addWatch(basePath, watcher, mode, vCb, worker); } } @Override public String toString() { - return String.format("setData (%s, mode = %s)", basePath, mode.name()); + return String.format("addWatch (%s, mode = %s)", basePath, mode.name()); } }; // execute it immediately diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java index 259261b18c6aa..bae1eae71efcf 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java @@ -19,6 +19,12 @@ package org.apache.pulsar.metadata; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -40,6 +46,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; @@ -49,6 +56,7 @@ import lombok.Cleanup; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.zookeeper.BoundExponentialBackoffRetryPolicy; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataStore; @@ -65,6 +73,9 @@ import org.apache.pulsar.metadata.impl.PulsarZooKeeperClient; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.apache.pulsar.metadata.impl.oxia.OxiaMetadataStore; +import org.apache.zookeeper.AddWatchMode; +import org.apache.zookeeper.AsyncCallback.VoidCallback; +import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; @@ -547,6 +558,53 @@ public void testZkLoadConfigFromFile() throws Exception { assertFalse(zooKeeper.getClientConfig().isSaslClientEnabled()); } + @Test + @SuppressWarnings("unchecked") + public void testAsyncAddWatchRetriesWithWrapperCallback() throws Exception { + String path = newKey(); + @Cleanup + PulsarZooKeeperClient zkClient = PulsarZooKeeperClient.newBuilder() + .connectString(zks.getConnectionString()) + .sessionTimeoutMs(3000) + .operationRetryPolicy(new BoundExponentialBackoffRetryPolicy(0, 0, 3)) + .build(); + + ZooKeeper mockZk = mock(ZooKeeper.class); + AtomicInteger attempts = new AtomicInteger(); + doAnswer(invocation -> { + // The wrapper callback should consume this recoverable failure and retry the addWatch operation. + int rc = attempts.incrementAndGet() == 1 + ? KeeperException.Code.CONNECTIONLOSS.intValue() + : KeeperException.Code.OK.intValue(); + String callbackPath = invocation.getArgument(0); + VoidCallback callback = invocation.getArgument(3); + Object callbackContext = invocation.getArgument(4); + callback.processResult(rc, callbackPath, callbackContext); + return null; + }).when(mockZk).addWatch(eq(path), any(Watcher.class), eq(AddWatchMode.PERSISTENT_RECURSIVE), + any(VoidCallback.class), any()); + + // Force the Pulsar wrapper to delegate the async addWatch call to our controlled ZooKeeper instance. + var zooKeeperRef = (AtomicReference) WhiteboxImpl.getInternalState(zkClient, "zk"); + zooKeeperRef.set(mockZk); + + CountDownLatch callbackCalled = new CountDownLatch(1); + AtomicInteger callbackRc = new AtomicInteger(Integer.MIN_VALUE); + zkClient.addWatch(path, event -> { + }, AddWatchMode.PERSISTENT_RECURSIVE, (rc, callbackPath, ctx) -> { + callbackRc.set(rc); + callbackCalled.countDown(); + }, null); + + assertTrue(callbackCalled.await(5, TimeUnit.SECONDS)); + + // The caller should only see the final successful result after the retry, not the first CONNECTIONLOSS. + assertEquals(callbackRc.get(), KeeperException.Code.OK.intValue()); + assertEquals(attempts.get(), 2); + verify(mockZk, times(2)).addWatch(eq(path), any(Watcher.class), eq(AddWatchMode.PERSISTENT_RECURSIVE), + any(VoidCallback.class), any()); + } + @Test public void testOxiaLoadConfigFromFile() throws Exception { final String metadataStoreName = UUID.randomUUID().toString().replaceAll("-", ""); From 155cb0202e98e6e8fd920dc0f1e26a2ef98995c6 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Wed, 3 Jun 2026 05:14:48 +0800 Subject: [PATCH 115/201] [fix][meta] Fix ZooKeeper session reconnect race condition in PulsarZooKeeperClient.clientCreator (#25910) (cherry picked from commit 5627c01b1ef04b8424b781eabc4dea6963faf847) --- .../metadata/impl/PulsarZooKeeperClient.java | 37 ++++++++++++++++--- .../metadata/impl/ZKSessionWatcher.java | 19 +++++++++- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java index 462df69b2ea2a..39921b0f89f19 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; @@ -122,16 +123,42 @@ public ZooKeeper call() throws KeeperException, InterruptedException { log.info("Reconnecting zookeeper {}.", connectString); // close the previous one closeZkHandle(); + + // ZooKeeper can deliver SyncConnected after createZooKeeper() returns but before zk.set(newZk) + // publishes the new instance. Hold these events until the new instance is published, so child + // watchers never observe a new-session event while PulsarZooKeeperClient still points at the + // old handle. + CountDownLatch newZkSetLatch = new CountDownLatch(1); + Watcher forwardEventsWatcher = event -> { + try { + boolean awaited = newZkSetLatch.await(sessionTimeoutMs, TimeUnit.MILLISECONDS); + if (!awaited) { + log.warn("Timed out waiting for ZooKeeper instance to be published before " + + "forwarding event {}", event); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.warn("Interrupted while waiting for ZooKeeper instance to be published, event {}", + event, e); + return; + } + watcherManager.process(event); + }; + ZooKeeper newZk; try { - newZk = createZooKeeper(); + newZk = createZooKeeper(forwardEventsWatcher); } catch (IOException | QuorumPeerConfig.ConfigException e) { log.error("Failed to create zookeeper instance to {} with config path {}", connectString, configPath, e); throw KeeperException.create(KeeperException.Code.CONNECTIONLOSS); } - waitForConnection(); + + // Publish the new instance before releasing the forwarding watcher. waitForConnection() must + // happen after countDown(), since it depends on the forwarded SyncConnected event. zk.set(newZk); + newZkSetLatch.countDown(); + waitForConnection(); log.info("ZooKeeper session {} is created to {}.", Long.toHexString(newZk.getSessionId()), connectString); return newZk; @@ -354,12 +381,12 @@ public void waitForConnection() throws KeeperException, InterruptedException { watcherManager.waitForConnection(); } - protected ZooKeeper createZooKeeper() throws IOException, QuorumPeerConfig.ConfigException { + protected ZooKeeper createZooKeeper(Watcher watcher) throws IOException, QuorumPeerConfig.ConfigException { if (null != configPath) { - return new ZooKeeper(connectString, sessionTimeoutMs, watcherManager, allowReadOnlyMode, + return new ZooKeeper(connectString, sessionTimeoutMs, watcher, allowReadOnlyMode, new ZKClientConfig(configPath)); } - return new ZooKeeper(connectString, sessionTimeoutMs, watcherManager, allowReadOnlyMode); + return new ZooKeeper(connectString, sessionTimeoutMs, watcher, allowReadOnlyMode); } @Override diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java index a840721023080..cc26231a74525 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java @@ -87,6 +87,7 @@ public void close() throws Exception { // in the future. private void checkConnectionStatus() { try { + long checkedSessionId = zk.getSessionId(); CompletableFuture future = new CompletableFuture<>(); zk.exists("/", false, (StatCallback) (rc, path, ctx, stat) -> { switch (KeeperException.Code.get(rc)) { @@ -112,7 +113,7 @@ private void checkConnectionStatus() { zkClientState = Watcher.Event.KeeperState.Disconnected; } - checkState(zkClientState); + checkStateIfSameSession(checkedSessionId, zkClientState); } catch (RejectedExecutionException | InterruptedException e) { task.cancel(true); } catch (Throwable t) { @@ -130,6 +131,22 @@ synchronized void setSessionInvalid() { currentStatus = SessionEvent.SessionLost; } + // PulsarZooKeeperClient publishes the new ZooKeeper instance before forwarding the corresponding session event to + // watcherManager, so zk.set(newZk) happens-before this watcher observes the new-session event. Keep the session-id + // check and state transition in the same synchronized section to prevent stale async probes from racing with that + // event and overwriting the state of the newly established session. + private synchronized void checkStateIfSameSession(long checkedSessionId, + Watcher.Event.KeeperState zkClientState) { + long currentSessionId = zk.getSessionId(); + if (checkedSessionId != currentSessionId) { + log.warn("Ignoring ZooKeeper session state from a stale session. checkedSessionId: {}," + + " currentSessionId: {}, zkClientState: {}", + checkedSessionId, currentSessionId, zkClientState); + return; + } + checkState(zkClientState); + } + private synchronized void checkState(Watcher.Event.KeeperState zkClientState) { switch (zkClientState) { case Expired: From 808c99160440cb10a555d2390a062e1a0bfdc633 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Wed, 3 Jun 2026 03:14:35 +0800 Subject: [PATCH 116/201] [fix][test] Fix flaky SameAuthParamsLookupAutoClusterFailoverTest.testAutoClusterFailover() test (#25892) (cherry picked from commit ed099501840fed113c630f92c5850faff8e118cc) --- .../broker/SameAuthParamsLookupAutoClusterFailoverTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java index febd0c4aad0fa..1314be12c9d0e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -98,7 +98,6 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { .tlsTrustCertsFilePath(CA_CERT_FILE_PATH); } final PulsarClient client = clientBuilder.build(); - failover.initialize(client); final EventLoopGroup executor = WhiteboxImpl.getInternalState(failover, "executor"); final PulsarServiceState[] stateArray = WhiteboxImpl.getInternalState(failover, "pulsarServiceStateArray"); From 88672993450939c756a99b2f2c2c05f02865f7a9 Mon Sep 17 00:00:00 2001 From: Pratik Katti <90851204+pratt4@users.noreply.github.com> Date: Wed, 3 Jun 2026 02:45:34 +0530 Subject: [PATCH 117/201] [improve][fn] make built-in functions reload incremental (#25868) (cherry picked from commit d57af8f37725ed16547139978f3a4d83022066c5) --- .../functions/worker/FunctionsManager.java | 30 +++- .../FunctionsManagerReloadFunctionsTest.java | 80 +++++++++ .../utils/functions/FunctionArchive.java | 28 +++ .../utils/functions/FunctionUtils.java | 77 +++++++++ .../functions/ReloadFunctionsResult.java | 29 ++++ .../functions/FunctionUtilsReloadTest.java | 159 ++++++++++++++++++ 6 files changed, 395 insertions(+), 8 deletions(-) create mode 100644 pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/FunctionsManagerReloadFunctionsTest.java create mode 100644 pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/ReloadFunctionsResult.java create mode 100644 pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/functions/FunctionUtilsReloadTest.java diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java index cdd772495023e..42841d1d695f1 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.nio.file.Path; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -30,10 +31,11 @@ import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.utils.functions.FunctionUtils; +import org.apache.pulsar.functions.utils.functions.ReloadFunctionsResult; @Slf4j public class FunctionsManager implements AutoCloseable { - private Map functions; + private volatile Map functions; @VisibleForTesting public FunctionsManager() { @@ -62,32 +64,44 @@ public List getFunctionDefinitions() { } public void reloadFunctions(WorkerConfig workerConfig) throws IOException { - Map oldFunctions = functions; - this.functions = createFunctions(workerConfig); - closeFunctions(oldFunctions); + ReloadFunctionsResult reload = FunctionUtils.reloadFunctions( + this.functions, + workerConfig.getFunctionsDirectory(), + workerConfig.getNarExtractionDirectory(), + isEnableClassloading(workerConfig)); + this.functions = reload.functions(); + closeFunctions(reload.functionsToClose()); } private static Map createFunctions(WorkerConfig workerConfig) throws IOException { - boolean enableClassloading = workerConfig.getEnableClassloadingOfBuiltinFiles() - || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); + boolean enableClassloading = isEnableClassloading(workerConfig); return FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory(), workerConfig.getNarExtractionDirectory(), enableClassloading); } + private static boolean isEnableClassloading(WorkerConfig workerConfig) { + return workerConfig.getEnableClassloadingOfBuiltinFiles() + || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); + } + @Override public void close() { closeFunctions(functions); } - private void closeFunctions(Map functionMap) { - functionMap.values().forEach(functionArchive -> { + private void closeFunctions(Collection functions) { + functions.forEach(functionArchive -> { try { functionArchive.close(); } catch (Exception e) { log.warn("Failed to close function archive", e); } }); + } + + private void closeFunctions(Map functionMap) { + closeFunctions(functionMap.values()); functionMap.clear(); } } diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/FunctionsManagerReloadFunctionsTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/FunctionsManagerReloadFunctionsTest.java new file mode 100644 index 0000000000000..0ea2ae2036ad4 --- /dev/null +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/FunctionsManagerReloadFunctionsTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.worker; + +import static org.testng.Assert.assertSame; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.functions.utils.functions.FunctionArchive; +import org.testng.annotations.Test; + +/** + * Tests {@link FunctionsManager#reloadFunctions(WorkerConfig)} for incremental reload behavior, + * ensuring unchanged functions are reused instead of being recreated. + */ +public class FunctionsManagerReloadFunctionsTest { + + private static void writeMinimalNar(Path narPath, FunctionDefinition def) throws IOException { + byte[] yaml = ObjectMapperFactory.getYamlMapper().getObjectMapper().writeValueAsBytes(def); + try (OutputStream os = Files.newOutputStream(narPath); + ZipOutputStream zos = new ZipOutputStream(os)) { + ZipEntry entry = new ZipEntry("META-INF/services/pulsar-io.yaml"); + zos.putNextEntry(entry); + zos.write(yaml); + zos.closeEntry(); + } + } + + private static FunctionDefinition sampleDefinition(String name) { + FunctionDefinition def = new FunctionDefinition(); + def.setName(name); + def.setFunctionClass("org.example.Function"); + return def; + } + + @Test + public void reloadWhenNarUnchangedReusesSameFunctionArchiveInstance() throws Exception { + Path dir = Files.createTempDirectory("mgr-fn-reload-"); + Path nar = dir.resolve("f1.nar"); + writeMinimalNar(nar, sampleDefinition("f-one")); + + WorkerConfig workerConfig = new WorkerConfig(); + workerConfig.setFunctionsDirectory(dir.toString()); + workerConfig.setNarExtractionDirectory(NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR); + workerConfig.setEnableClassloadingOfBuiltinFiles(false); + + try (FunctionsManager manager = new FunctionsManager(workerConfig)) { + FunctionArchive before = manager.getFunction("f-one"); + before.getFunctionPackage(); + + manager.reloadFunctions(workerConfig); + + FunctionArchive after = manager.getFunction("f-one"); + assertSame(after, before); + before.getFunctionPackage(); + } + } +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java index cfb213f34ed72..70b452eba39cf 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.functions.utils.functions; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Path; import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.functions.utils.FunctionFilePackage; @@ -25,6 +27,8 @@ public class FunctionArchive implements AutoCloseable { private final Path archivePath; + /** MD5 hex of archive file contents; empty when {@link #archivePath} is null (test doubles). */ + private final String archiveMd5Hex; private final FunctionDefinition functionDefinition; private final String narExtractionDirectory; private final boolean enableClassloading; @@ -33,16 +37,40 @@ public class FunctionArchive implements AutoCloseable { public FunctionArchive(Path archivePath, FunctionDefinition functionDefinition, String narExtractionDirectory, boolean enableClassloading) { + this(archivePath, functionDefinition, narExtractionDirectory, enableClassloading, null); + } + + /** + * @param precomputedArchiveMd5Hex MD5 hex of {@code archivePath} contents; if null and path is non-null, + * the hash is computed once at construction time. + */ + public FunctionArchive(Path archivePath, FunctionDefinition functionDefinition, String narExtractionDirectory, + boolean enableClassloading, String precomputedArchiveMd5Hex) { this.archivePath = archivePath; this.functionDefinition = functionDefinition; this.narExtractionDirectory = narExtractionDirectory; this.enableClassloading = enableClassloading; + if (archivePath != null) { + try { + this.archiveMd5Hex = precomputedArchiveMd5Hex != null + ? precomputedArchiveMd5Hex + : FunctionUtils.computeArchiveMd5Hex(archivePath); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + this.archiveMd5Hex = ""; + } } public Path getArchivePath() { return archivePath; } + public String getArchiveMd5Hex() { + return archiveMd5Hex; + } + public synchronized ValidatableFunctionPackage getFunctionPackage() { if (closed) { throw new IllegalStateException("FunctionArchive is already closed"); diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java index f4d45edf36301..1425b2985da92 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java @@ -25,12 +25,16 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; import java.util.Map; import java.util.TreeMap; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.common.nar.FileUtils; import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.zeroturnaround.zip.ZipUtil; @@ -42,6 +46,17 @@ public class FunctionUtils { private static final String PULSAR_IO_SERVICE_NAME = "pulsar-io.yaml"; + /** + * Computes MD5 digest of a file as lower-case hex (for function archive identity on reload). + */ + public static String computeArchiveMd5Hex(Path path) throws IOException { + return calculateMd5Hex(path.toAbsolutePath().normalize().toFile()); + } + + private static String calculateMd5Hex(File file) throws IOException { + return HexFormat.of().formatHex(FileUtils.calculateMd5sum(file)); + } + /** * Extract the Pulsar Function class from a function or archive. */ @@ -107,4 +122,66 @@ public static Map searchForFunctions(String functionsDi return functions; } + + /** + * Reloads functions from disk against {@code previous}, reusing {@link FunctionArchive} instances when path and + * archive MD5 are unchanged (keeps class loaders open). New or changed archives get new instances. + *

+ * {@link ReloadFunctionsResult#functionsToClose()} lists function archives evicted from the active set (replaced + * or no longer present on disk); the caller must {@link FunctionArchive#close()} each. + * + * @param previous functions from the previous scan (may be empty, never null) + * @param functionsDirectory same semantics as {@link #searchForFunctions} + * @param narExtractionDirectory same semantics as {@link #searchForFunctions} + * @param enableClassloading same semantics as {@link #searchForFunctions} + * @return new map keyed by function name (reused values are identical instances from {@code previous}) and + * functions the caller should close + */ + public static ReloadFunctionsResult reloadFunctions( + Map previous, + String functionsDirectory, + String narExtractionDirectory, + boolean enableClassloading) throws IOException { + + TreeMap remaining = new TreeMap<>(previous); + TreeMap next = new TreeMap<>(); + List toClose = new ArrayList<>(); + + Path dir = Paths.get(functionsDirectory).toAbsolutePath().normalize(); + if (!dir.toFile().exists()) { + toClose.addAll(remaining.values()); + return new ReloadFunctionsResult(next, toClose); + } + + try (DirectoryStream stream = Files.newDirectoryStream(dir, "*.nar")) { + for (Path archive : stream) { + try { + FunctionDefinition funcDef = FunctionUtils.getFunctionDefinition(archive.toFile()); + if (!StringUtils.isEmpty(funcDef.getFunctionClass())) { + String name = funcDef.getName(); + String md5Hex = computeArchiveMd5Hex(archive); + FunctionArchive prev = remaining.remove(name); + if (prev != null + && prev.getArchivePath() != null + && archive.equals(prev.getArchivePath()) + && md5Hex.equals(prev.getArchiveMd5Hex())) { + next.put(name, prev); + } else { + if (prev != null) { + log.info("Reloading changed function {} from {} (previous archive {})", + name, archive, prev.getArchivePath()); + toClose.add(prev); + } + next.put(name, new FunctionArchive(archive, funcDef, narExtractionDirectory, + enableClassloading, md5Hex)); + } + } + } catch (Throwable t) { + log.warn("Failed to load function from {}", archive, t); + } + } + } + toClose.addAll(remaining.values()); + return new ReloadFunctionsResult(next, toClose); + } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/ReloadFunctionsResult.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/ReloadFunctionsResult.java new file mode 100644 index 0000000000000..2eb8c73644512 --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/ReloadFunctionsResult.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils.functions; + +import java.util.List; +import java.util.Map; + +/** + * Result of {@link FunctionUtils#reloadFunctions}: the new function map and function archives evicted from the + * active set that the caller must close. + */ +public record ReloadFunctionsResult(Map functions, List functionsToClose) { +} diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/functions/FunctionUtilsReloadTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/functions/FunctionUtilsReloadTest.java new file mode 100644 index 0000000000000..0dec5f1c613e5 --- /dev/null +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/functions/FunctionUtilsReloadTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils.functions; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotSame; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.testng.annotations.Test; + +@Test +public class FunctionUtilsReloadTest { + + private static void closeEvicted(ReloadFunctionsResult reload) throws Exception { + for (FunctionArchive functionArchive : reload.functionsToClose()) { + functionArchive.close(); + } + } + + private static void writeMinimalNar(Path narPath, FunctionDefinition def) throws IOException { + byte[] yaml = ObjectMapperFactory.getYamlMapper().getObjectMapper().writeValueAsBytes(def); + try (OutputStream os = Files.newOutputStream(narPath); + ZipOutputStream zos = new ZipOutputStream(os)) { + ZipEntry entry = new ZipEntry("META-INF/services/pulsar-io.yaml"); + zos.putNextEntry(entry); + zos.write(yaml); + zos.closeEntry(); + } + } + + private static FunctionDefinition sampleDefinition(String name) { + FunctionDefinition def = new FunctionDefinition(); + def.setName(name); + def.setFunctionClass("org.example.Function"); + return def; + } + + /** + * Historical {@code FunctionsManager} reload replaced the whole map and closed every prior + * {@link FunctionArchive}, even when NAR files were unchanged. A caller keeping a reference to the + * pre-reload archive would then hit {@link IllegalStateException} on lazy use. + *

+ * Incremental reload must evict nothing, reuse the same instance, and leave that instance usable + * after the caller closes only {@link ReloadFunctionsResult#functionsToClose()}. + */ + @Test + public void reloadUnchangedNarEvictsNothingAndKeepsSameFunctionArchiveUsable() throws Exception { + Path dir = Files.createTempDirectory("fn-reload-"); + Path nar = dir.resolve("f1.nar"); + writeMinimalNar(nar, sampleDefinition("f-one")); + + Map first = + FunctionUtils.searchForFunctions(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + FunctionArchive functionArchive = first.get("f-one"); + functionArchive.getFunctionPackage(); + + ReloadFunctionsResult reload = FunctionUtils.reloadFunctions( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + assertTrue(reload.functionsToClose().isEmpty()); + closeEvicted(reload); + Map second = reload.functions(); + + assertSame(second.get("f-one"), functionArchive); + functionArchive.getFunctionPackage(); + } + + @Test + public void reloadReopensFunctionArchiveWhenNarContentChanges() throws Exception { + Path dir = Files.createTempDirectory("fn-reload-"); + Path nar = dir.resolve("f1.nar"); + writeMinimalNar(nar, sampleDefinition("f-one")); + + Map first = + FunctionUtils.searchForFunctions(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + FunctionArchive before = first.get("f-one"); + + FunctionDefinition updated = sampleDefinition("f-one"); + updated.setDescription("changed"); + writeMinimalNar(nar, updated); + + ReloadFunctionsResult reload = FunctionUtils.reloadFunctions( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + closeEvicted(reload); + Map second = reload.functions(); + + assertNotSame(second.get("f-one"), before); + assertThrows(IllegalStateException.class, before::getFunctionPackage); + } + + @Test + public void reloadClosesFunctionArchivesRemovedFromDirectory() throws Exception { + Path dir = Files.createTempDirectory("fn-reload-"); + Path nar1 = dir.resolve("a.nar"); + Path nar2 = dir.resolve("b.nar"); + writeMinimalNar(nar1, sampleDefinition("fn-a")); + writeMinimalNar(nar2, sampleDefinition("fn-b")); + + Map first = + FunctionUtils.searchForFunctions(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + FunctionArchive removed = first.get("fn-b"); + Files.delete(nar2); + + ReloadFunctionsResult reload = FunctionUtils.reloadFunctions( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + closeEvicted(reload); + Map second = reload.functions(); + + assertEquals(second.size(), 1); + assertSame(second.get("fn-a"), first.get("fn-a")); + assertThrows(IllegalStateException.class, removed::getFunctionPackage); + } + + @Test + public void reloadClosesAllFunctionArchivesWhenDirectoryIsMissing() throws Exception { + Path dir = Files.createTempDirectory("fn-reload-"); + Path nar = dir.resolve("f1.nar"); + writeMinimalNar(nar, sampleDefinition("f-one")); + + Map first = + FunctionUtils.searchForFunctions(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + FunctionArchive removed = first.get("f-one"); + Files.delete(nar); + Files.delete(dir); + + ReloadFunctionsResult reload = FunctionUtils.reloadFunctions( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + closeEvicted(reload); + + assertTrue(reload.functions().isEmpty()); + assertThrows(IllegalStateException.class, removed::getFunctionPackage); + } +} From 63ac35b0675f2bcd03f5055e4b1de5845ce05639 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jun 2026 05:06:59 +0300 Subject: [PATCH 118/201] [fix][sec] Upgrade Netty to 4.1.135.Final to address several CVEs (#25918) (cherry picked from commit df953e9d7e95a052a44a906229750824338e1711) --- .../server/src/assemble/LICENSE.bin.txt | 40 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 38 +++++++++--------- pom.xml | 2 +- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index bddcd8f54588a..0dd4c17211200 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -293,26 +293,26 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-lang3-3.19.0.jar - org.apache.commons-commons-text-1.14.0.jar * Netty - - io.netty-netty-buffer-4.1.134.Final.jar - - io.netty-netty-codec-4.1.134.Final.jar - - io.netty-netty-codec-dns-4.1.134.Final.jar - - io.netty-netty-codec-http-4.1.134.Final.jar - - io.netty-netty-codec-http2-4.1.134.Final.jar - - io.netty-netty-codec-socks-4.1.134.Final.jar - - io.netty-netty-codec-haproxy-4.1.134.Final.jar - - io.netty-netty-common-4.1.134.Final.jar - - io.netty-netty-handler-4.1.134.Final.jar - - io.netty-netty-handler-proxy-4.1.134.Final.jar - - io.netty-netty-resolver-4.1.134.Final.jar - - io.netty-netty-resolver-dns-4.1.134.Final.jar - - io.netty-netty-resolver-dns-classes-macos-4.1.134.Final.jar - - io.netty-netty-resolver-dns-native-macos-4.1.134.Final-osx-aarch_64.jar - - io.netty-netty-resolver-dns-native-macos-4.1.134.Final-osx-x86_64.jar - - io.netty-netty-transport-4.1.134.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.134.Final.jar - - io.netty-netty-transport-native-epoll-4.1.134.Final-linux-aarch_64.jar - - io.netty-netty-transport-native-epoll-4.1.134.Final-linux-x86_64.jar - - io.netty-netty-transport-native-unix-common-4.1.134.Final.jar + - io.netty-netty-buffer-4.1.135.Final.jar + - io.netty-netty-codec-4.1.135.Final.jar + - io.netty-netty-codec-dns-4.1.135.Final.jar + - io.netty-netty-codec-http-4.1.135.Final.jar + - io.netty-netty-codec-http2-4.1.135.Final.jar + - io.netty-netty-codec-socks-4.1.135.Final.jar + - io.netty-netty-codec-haproxy-4.1.135.Final.jar + - io.netty-netty-common-4.1.135.Final.jar + - io.netty-netty-handler-4.1.135.Final.jar + - io.netty-netty-handler-proxy-4.1.135.Final.jar + - io.netty-netty-resolver-4.1.135.Final.jar + - io.netty-netty-resolver-dns-4.1.135.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.1.135.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.1.135.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.1.135.Final-osx-x86_64.jar + - io.netty-netty-transport-4.1.135.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.135.Final.jar + - io.netty-netty-transport-native-epoll-4.1.135.Final-linux-aarch_64.jar + - io.netty-netty-transport-native-epoll-4.1.135.Final-linux-x86_64.jar + - io.netty-netty-transport-native-unix-common-4.1.135.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-linux-aarch_64.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-linux-x86_64.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index f8076d313ba94..48b56ec4a6c5d 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -345,22 +345,22 @@ The Apache Software License, Version 2.0 - commons-text-1.14.0.jar - commons-compress-1.28.0.jar * Netty - - netty-buffer-4.1.134.Final.jar - - netty-codec-4.1.134.Final.jar - - netty-codec-dns-4.1.134.Final.jar - - netty-codec-http-4.1.134.Final.jar - - netty-codec-socks-4.1.134.Final.jar - - netty-codec-haproxy-4.1.134.Final.jar - - netty-common-4.1.134.Final.jar - - netty-handler-4.1.134.Final.jar - - netty-handler-proxy-4.1.134.Final.jar - - netty-resolver-4.1.134.Final.jar - - netty-resolver-dns-4.1.134.Final.jar - - netty-transport-4.1.134.Final.jar - - netty-transport-classes-epoll-4.1.134.Final.jar - - netty-transport-native-epoll-4.1.134.Final-linux-aarch_64.jar - - netty-transport-native-epoll-4.1.134.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.134.Final.jar + - netty-buffer-4.1.135.Final.jar + - netty-codec-4.1.135.Final.jar + - netty-codec-dns-4.1.135.Final.jar + - netty-codec-http-4.1.135.Final.jar + - netty-codec-socks-4.1.135.Final.jar + - netty-codec-haproxy-4.1.135.Final.jar + - netty-common-4.1.135.Final.jar + - netty-handler-4.1.135.Final.jar + - netty-handler-proxy-4.1.135.Final.jar + - netty-resolver-4.1.135.Final.jar + - netty-resolver-dns-4.1.135.Final.jar + - netty-transport-4.1.135.Final.jar + - netty-transport-classes-epoll-4.1.135.Final.jar + - netty-transport-native-epoll-4.1.135.Final-linux-aarch_64.jar + - netty-transport-native-epoll-4.1.135.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.135.Final.jar - netty-tcnative-boringssl-static-2.0.77.Final.jar - netty-tcnative-boringssl-static-2.0.77.Final-linux-aarch_64.jar - netty-tcnative-boringssl-static-2.0.77.Final-linux-x86_64.jar @@ -371,9 +371,9 @@ The Apache Software License, Version 2.0 - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - - netty-resolver-dns-classes-macos-4.1.134.Final.jar - - netty-resolver-dns-native-macos-4.1.134.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.134.Final-osx-x86_64.jar + - netty-resolver-dns-classes-macos-4.1.135.Final.jar + - netty-resolver-dns-native-macos-4.1.135.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.135.Final-osx-x86_64.jar * Prometheus client - simpleclient-0.16.0.jar - simpleclient_log4j2-0.16.0.jar diff --git a/pom.xml b/pom.xml index 6e9a41961ec3f..19d9ab09f42bf 100644 --- a/pom.xml +++ b/pom.xml @@ -189,7 +189,7 @@ flexible messaging model and an intuitive client API. 1.1.10.8 4.1.12.1 5.7.1 - 4.1.134.Final + 4.1.135.Final 0.0.26.Final 12.1.9 From dab45f6cd0a6e0c810743a13473c13d84d390afd Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Wed, 3 Jun 2026 16:52:06 +0800 Subject: [PATCH 119/201] [fix][client] Clean up unacked messages when unsubscribing a topic with ack timeout backoff (#25916) Signed-off-by: Dream95 (cherry picked from commit 756c03df2d3ef17ce60ee124bbef81be03fc0191) --- .../client/impl/MultiTopicsConsumerImpl.java | 6 +- .../UnAckedTopicMessageRedeliveryTracker.java | 5 +- ...ckedTopicMessageRedeliveryTrackerTest.java | 79 +++++++++++++++++++ 3 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 7865939d77df3..f1a543e95d456 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -1283,8 +1283,10 @@ public CompletableFuture unsubscribeAsync(String topicName) { }); removeTopic(topicName); - if (unAckedMessageTracker instanceof UnAckedTopicMessageTracker) { - ((UnAckedTopicMessageTracker) unAckedMessageTracker).removeTopicMessages(topicName); + if (unAckedMessageTracker instanceof UnAckedTopicMessageTracker tracker) { + tracker.removeTopicMessages(topicName); + } else if (unAckedMessageTracker instanceof UnAckedTopicMessageRedeliveryTracker tracker){ + tracker.removeTopicMessages(topicName); } unsubscribeFuture.complete(null); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java index 823dd4ad5f488..393557cd89adb 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java @@ -44,7 +44,6 @@ public int removeTopicMessages(String topicName) { MessageId messageId = messageIdWrapper.getMessageId(); if (messageId instanceof TopicMessageId && ((TopicMessageId) messageId).getOwnerTopic().contains(topicName)) { - HashSet exist = redeliveryMessageIdPartitionMap.get(messageIdWrapper); entry.getValue().remove(messageIdWrapper); iterator.remove(); messageIdWrapper.recycle(); @@ -53,11 +52,11 @@ public int removeTopicMessages(String topicName) { } Iterator iteratorAckTimeOut = ackTimeoutMessages.keySet().iterator(); - while (iterator.hasNext()) { + while (iteratorAckTimeOut.hasNext()) { MessageId messageId = iteratorAckTimeOut.next(); if (messageId instanceof TopicMessageId && ((TopicMessageId) messageId).getOwnerTopic().contains(topicName)) { - iterator.remove(); + iteratorAckTimeOut.remove(); removed++; } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java new file mode 100644 index 0000000000000..9fdab39145bd8 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timer; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.testng.annotations.Test; + +public class UnAckedTopicMessageRedeliveryTrackerTest { + + @Test + @SuppressWarnings("unchecked") + public void testRemoveTopicMessages() { + PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.instrumentProvider()).thenReturn(InstrumentProvider.NOOP); + when(client.getCnxPool()).thenReturn(connectionPool); + @Cleanup("stop") + Timer timer = new HashedWheelTimer( + new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()), + 1, TimeUnit.MILLISECONDS); + when(client.timer()).thenReturn(timer); + + ConsumerBase consumer = mock(ConsumerBase.class); + doNothing().when(consumer).onAckTimeoutSend(any()); + doNothing().when(consumer).redeliverUnacknowledgedMessages(any()); + + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setAckTimeoutMillis(1_000_000); + conf.setTickDurationMillis(100_000); + conf.setAckTimeoutRedeliveryBackoff(MultiplierRedeliveryBackoff.builder().build()); + + UnAckedTopicMessageRedeliveryTracker tracker = + new UnAckedTopicMessageRedeliveryTracker(client, consumer, conf); + + String ownerTopic = "persistent://public/default/my-topic-partition-0"; + TopicMessageIdImpl msgInPartition = + new TopicMessageIdImpl(ownerTopic, new MessageIdImpl(1L, 0L, -1)); + TopicMessageIdImpl msgInAckTimeout = + new TopicMessageIdImpl(ownerTopic, new MessageIdImpl(2L, 0L, -1)); + + assertTrue(tracker.add(msgInPartition)); + tracker.ackTimeoutMessages.put(msgInAckTimeout, System.currentTimeMillis() + 1_000_000L); + assertEquals(tracker.size(), 2); + + assertEquals(tracker.removeTopicMessages("my-topic"), 2); + assertTrue(tracker.isEmpty()); + + tracker.close(); + } + +} From e3b8c4a29c83095e804fb8a4553d2c5c3d711460 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jun 2026 11:54:30 +0300 Subject: [PATCH 120/201] [fix][proxy] Avoid intermittent 502 when admin proxy follows a broker redirect for a request with a body (#25919) (cherry picked from commit 2acee322ef8ace78759677b997a57be454ed2eca) --- .../proxy/server/AdminProxyHandler.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index 7992cd20d11af..ba49714b89bd6 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -46,6 +46,7 @@ import org.eclipse.jetty.client.ProtocolHandlers; import org.eclipse.jetty.client.RedirectProtocolHandler; import org.eclipse.jetty.client.Request; +import org.eclipse.jetty.client.Response; import org.eclipse.jetty.client.transport.HttpClientTransportOverHTTP; import org.eclipse.jetty.ee8.proxy.ProxyServlet; import org.eclipse.jetty.http.HttpField; @@ -121,7 +122,7 @@ protected void customizeHttpClient(HttpClient httpClient) { ProtocolHandlers protocolHandlers = httpClient.getProtocolHandlers(); if (protocolHandlers != null) { - protocolHandlers.put(new RedirectProtocolHandler(httpClient)); + protocolHandlers.put(new NonAbortingRedirectProtocolHandler(httpClient)); } httpClient.setIdleTimeout(config.getHttpProxyIdleTimeout()); @@ -129,6 +130,34 @@ protected void customizeHttpClient(HttpClient httpClient) { setTimeout(config.getHttpProxyTimeout()); } + /** + * A {@link RedirectProtocolHandler} that does not abort the in-flight request when a redirect + * response is received. + * + *

Jetty's default {@link RedirectProtocolHandler#onSuccess(Response)} aborts a request that + * still has a body to send when a redirect status is received, raising + * {@code HttpRequestException: "Aborting request after receiving a NNN response"}. When a broker + * returns a 307 (to redirect an admin request to the bundle-owner broker) before the proxy has + * finished streaming the request body, that abort can race ahead of the redirect continuation in + * {@link RedirectProtocolHandler#onComplete} and surface to the proxy as a spurious HTTP 502 Bad + * Gateway. The redirect itself is driven by {@code onComplete} from the response (its status and + * {@code Location} header) and does not depend on the abort, so skipping it lets the redirect + * always be followed on a fresh request (with the body replayed by + * {@link ReplayableProxyContentProvider}), which is the behavior this proxy needs. + */ + static class NonAbortingRedirectProtocolHandler extends RedirectProtocolHandler { + NonAbortingRedirectProtocolHandler(HttpClient client) { + super(client); + } + + @Override + public void onSuccess(Response response) { + // Intentionally do NOT abort the request here. The redirect is followed in onComplete(); + // aborting the in-flight request only races a 502 to the proxy when the broker returns a + // redirect before the request body has finished sending. + } + } + // This class allows the request body to be replayed, the default implementation // does not protected class ReplayableProxyContentProvider extends ProxyInputStreamRequestContent { From c3c3fe5cffa73229692f2ac88f5688ac230e9e36 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Thu, 4 Jun 2026 01:44:38 +0800 Subject: [PATCH 121/201] [fix][client] Prevent duplicate ServiceUrlProvider initialization (#25899) (cherry picked from commit 882946c59d37b2a4d7e474bfa3a7d8da4d7fa59d) --- ...thParamsLookupAutoClusterFailoverTest.java | 14 +++++++++++++ .../pulsar/client/api/ServiceUrlProvider.java | 11 +++++++++- .../client/impl/AutoClusterFailover.java | 15 +++++++++++-- .../impl/ControlledClusterFailover.java | 15 +++++++++++-- ...meAuthParamsLookupAutoClusterFailover.java | 16 +++++++++++--- .../client/impl/AutoClusterFailoverTest.java | 21 +++++++++++++++++++ .../impl/ControlledClusterFailoverTest.java | 17 +++++++++++++++ 7 files changed, 101 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java index 1314be12c9d0e..a2e587fef395f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -172,6 +172,20 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { dummyServer.close(); } + @Test + public void testInitializeCanOnlyBeCalledOnce() throws Exception { + setup(); + final SameAuthParamsLookupAutoClusterFailover failover = SameAuthParamsLookupAutoClusterFailover.builder() + .pulsarServiceUrlArray(new String[]{pulsar1.getBrokerServiceUrl()}) + .checkHealthyIntervalMs(1000) + .build(); + + try (PulsarClient client = PulsarClient.builder().serviceUrlProvider(failover).build()) { + Throwable error = Assert.expectThrows(IllegalStateException.class, () -> failover.initialize(client)); + Assert.assertEquals(error.getMessage(), "ServiceUrlProvider has already been initialized"); + } + } + @Override protected void cleanupPulsarResources() { // Nothing to do. diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java index e8b513b103f65..f95f650cbb731 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java @@ -27,6 +27,11 @@ *

This allows applications to retrieve the service URL from an external configuration provider and, * more importantly, to force the Pulsar client to reconnect if the service URL has been changed. * + *

Each provider instance is tied to the lifecycle of one {@link PulsarClient} instance. The client + * initializes the provider when the client is created and closes the provider when the owning client is + * closed. Applications that create multiple Pulsar clients should create a separate provider instance + * for each client instead of sharing one provider. + * *

It can be passed with {@link ClientBuilder#serviceUrlProvider(ServiceUrlProvider)} */ @InterfaceAudience.Public @@ -39,6 +44,9 @@ public interface ServiceUrlProvider extends AutoCloseable { *

This can be used by the provider to force the Pulsar client to reconnect whenever the service url might have * changed. See {@link PulsarClient#updateServiceUrl(String)}. * + *

This method is invoked by the Pulsar client and is expected to be called once for a provider + * instance. Implementations may reject repeated initialization. + * * @param client * created pulsar client. */ @@ -52,7 +60,8 @@ public interface ServiceUrlProvider extends AutoCloseable { String getServiceUrl(); /** - * Close the resource that the provider allocated. + * Close the resource that the provider allocated. The owning Pulsar client invokes this method when + * it is closed. * */ @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java index 7615ec9f810b8..5c95590a64a33 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java @@ -40,6 +40,14 @@ import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.util.ExecutorProvider; +/** + * A service URL provider that automatically fails over from the primary Pulsar service URL to one of + * the secondary service URLs and switches back after the primary service recovers. + * + *

Each instance is tied to the lifecycle of one {@link PulsarClient}. Once initialized by a + * Pulsar client, it must not be reused by another client. Create a new provider instance for each + * Pulsar client. + */ @Slf4j @Data public class AutoClusterFailover implements ServiceUrlProvider { @@ -86,7 +94,10 @@ private AutoClusterFailover(AutoClusterFailoverBuilderImpl builder) { } @Override - public void initialize(PulsarClient client) { + public synchronized void initialize(PulsarClient client) { + if (this.pulsarClient != null) { + throw new IllegalStateException("ServiceUrlProvider has already been initialized"); + } this.pulsarClient = (PulsarClientImpl) client; this.addressResolver = pulsarClient.getAddressResolver(); ClientConfigurationData config = pulsarClient.getConfiguration(); @@ -123,7 +134,7 @@ public String getServiceUrl() { } @Override - public void close() { + public synchronized void close() { this.executor.shutdown(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java index 7819f051e7581..aa4faaaf9c747 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java @@ -57,6 +57,14 @@ import org.asynchttpclient.channel.DefaultKeepAliveStrategy; import org.jspecify.annotations.Nullable; +/** + * A service URL provider that fetches controlled failover configuration from an external HTTP service + * and updates the Pulsar client when the returned configuration changes. + * + *

Each instance is tied to the lifecycle of one {@link PulsarClient}. Once initialized by a + * Pulsar client, it must not be reused by another client. Create a new provider instance for each + * Pulsar client. + */ @Slf4j public class ControlledClusterFailover implements ServiceUrlProvider { private static final int DEFAULT_CONNECT_TIMEOUT_IN_SECONDS = 10; @@ -111,7 +119,10 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, } @Override - public void initialize(PulsarClient client) { + public synchronized void initialize(PulsarClient client) { + if (this.pulsarClient != null) { + throw new IllegalStateException("ServiceUrlProvider has already been initialized"); + } this.pulsarClient = (PulsarClientImpl) client; this.httpClient = buildHttpClient(); this.requestBuilder = httpClient.prepareGet(urlProvider) @@ -220,7 +231,7 @@ public String getServiceUrl() { } @Override - public void close() { + public synchronized void close() { this.executor.shutdown(); if (httpClient != null) { try { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java index 448dd95c4fdd9..69c74bd28d92a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java @@ -37,6 +37,14 @@ import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.netty.EventLoopUtil; +/** + * A service URL provider that probes multiple Pulsar service URLs with the same authentication + * parameters and fails over according to service health. + * + *

Each instance is tied to the lifecycle of one {@link PulsarClient}. Once initialized by a + * Pulsar client, it must not be reused by another client. Create a new provider instance for each + * Pulsar client. + */ @Slf4j @SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) public class SameAuthParamsLookupAutoClusterFailover implements ServiceUrlProvider { @@ -65,7 +73,10 @@ public class SameAuthParamsLookupAutoClusterFailover implements ServiceUrlProvid private SameAuthParamsLookupAutoClusterFailover() {} @Override - public void initialize(PulsarClient client) { + public synchronized void initialize(PulsarClient client) { + if (this.pulsarClient != null) { + throw new IllegalStateException("ServiceUrlProvider has already been initialized"); + } this.currentPulsarServiceIndex = 0; this.pulsarClient = (PulsarClientImpl) client; this.executor = EventLoopUtil.newEventLoopGroup(1, false, @@ -111,7 +122,7 @@ public String getServiceUrl() { } @Override - public void close() throws Exception { + public synchronized void close() throws Exception { if (closed) { return; } @@ -371,4 +382,3 @@ public SameAuthParamsLookupAutoClusterFailover build() { } } } - diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java index b275ffb6012ca..00688db800fe3 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java @@ -31,10 +31,12 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.ServiceUrlProvider; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.awaitility.Awaitility; import org.mockito.Mockito; +import org.testng.Assert; import org.testng.annotations.Test; @Test(groups = "broker-impl") @@ -148,6 +150,25 @@ public void testInitialize() throws Exception { } } + @Test + public void testInitializeCanOnlyBeCalledOnce() throws Exception { + String primary = "pulsar://localhost:6650"; + String secondary = "pulsar://localhost:6651"; + + ServiceUrlProvider provider = AutoClusterFailover.builder() + .primary(primary) + .secondary(Collections.singletonList(secondary)) + .failoverDelay(1, TimeUnit.SECONDS) + .switchBackDelay(1, TimeUnit.SECONDS) + .checkInterval(30, TimeUnit.SECONDS) + .build(); + + try (PulsarClient client = PulsarClient.builder().serviceUrlProvider(provider).build()) { + Throwable error = Assert.expectThrows(IllegalStateException.class, () -> provider.initialize(client)); + assertEquals(error.getMessage(), "ServiceUrlProvider has already been initialized"); + } + } + @Test public void testAutoClusterFailoverSwitchWithoutAuthentication() throws Exception { String primary = "pulsar://localhost:6650"; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java index 86b2fa7cb4f9b..cc47841ce0a11 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java @@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.ServiceUrlProvider; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.asynchttpclient.Request; @@ -73,6 +74,22 @@ public void testBuildControlledClusterFailoverInstance() throws Exception { Assert.assertEquals(request.getHeaders().get(keyB), valueB); } + @Test + public void testInitializeCanOnlyBeCalledOnce() throws Exception { + String defaultServiceUrl = "pulsar://localhost:6650"; + String urlProvider = "http://localhost:8080/test"; + + ServiceUrlProvider provider = ControlledClusterFailover.builder() + .defaultServiceUrl(defaultServiceUrl) + .urlProvider(urlProvider) + .build(); + + try (PulsarClient client = PulsarClient.builder().serviceUrlProvider(provider).build()) { + Throwable error = Assert.expectThrows(IllegalStateException.class, () -> provider.initialize(client)); + Assert.assertEquals(error.getMessage(), "ServiceUrlProvider has already been initialized"); + } + } + @Test public void testControlledClusterFailoverSwitch() throws Exception { String defaultServiceUrl = "pulsar+ssl://localhost:6651"; From a640579a55ab64c006aaeee0e17d789c3ff9d823 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Thu, 4 Jun 2026 02:16:26 +0800 Subject: [PATCH 122/201] [improve][offload] Coalesce automatic offload triggers to reduce retry loops and ledger scans (#25793) (cherry picked from commit 7ecedb8265b1d802ef1a571d7780f4a73141251b) --- .../AutomaticOffloadTriggerController.java | 86 ++++++++++++ .../impl/ManagedLedgerFactoryImpl.java | 4 +- .../mledger/impl/ManagedLedgerImpl.java | 90 +++++++++++-- ...AutomaticOffloadTriggerControllerTest.java | 85 ++++++++++++ .../mledger/impl/OffloadPrefixTest.java | 124 ++++++++++++++++++ 5 files changed, 375 insertions(+), 14 deletions(-) create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java new file mode 100644 index 0000000000000..35f5a26706371 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Coalesces repeated automatic offload triggers into at most one active run and one follow-up run. + */ +final class AutomaticOffloadTriggerController { + private static final int IDLE = 0; + private static final int RUNNING = 1; + private static final int RUNNING_WITH_PENDING_TRIGGER = 2; + + private final AtomicInteger state = new AtomicInteger(IDLE); + + /** + * Records an automatic offload trigger. + * + * @return true when the caller must start a new automatic offload run + */ + boolean requestRun() { + while (true) { + int current = state.get(); + switch (current) { + case IDLE: + if (state.compareAndSet(IDLE, RUNNING)) { + return true; + } + break; + case RUNNING: + if (state.compareAndSet(RUNNING, RUNNING_WITH_PENDING_TRIGGER)) { + return false; + } + break; + case RUNNING_WITH_PENDING_TRIGGER: + return false; + default: + throw new IllegalStateException("Unknown automatic offload trigger state: " + current); + } + } + } + + /** + * Records completion of the current automatic offload run. + * + * @return true when the caller must immediately start one coalesced follow-up run + */ + boolean completeRun() { + while (true) { + int current = state.get(); + switch (current) { + case IDLE: + return false; + case RUNNING: + if (state.compareAndSet(RUNNING, IDLE)) { + return false; + } + break; + case RUNNING_WITH_PENDING_TRIGGER: + if (state.compareAndSet(RUNNING_WITH_PENDING_TRIGGER, RUNNING)) { + return true; + } + break; + default: + throw new IllegalStateException("Unknown automatic offload trigger state: " + current); + } + } + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index 4e530a185cce1..a719d484beb27 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -19,7 +19,7 @@ package org.apache.bookkeeper.mledger.impl; import static org.apache.bookkeeper.mledger.ManagedLedgerException.getManagedLedgerException; -import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.NULL_OFFLOAD_PROMISE; +import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.AUTOMATIC_OFFLOAD_TRIGGER; import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicates; @@ -487,7 +487,7 @@ public void initializeComplete() { future.complete(newledger); // May need to trigger offloading if (config.isTriggerOffloadOnTopicLoad()) { - newledger.maybeOffloadInBackground(NULL_OFFLOAD_PROMISE); + newledger.maybeOffloadInBackground(AUTOMATIC_OFFLOAD_TRIGGER); } }); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 54be0723d4742..b1a7e199740c7 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -224,8 +224,20 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback { protected final CallbackMutex trimmerMutex = new CallbackMutex(); protected final CallbackMutex offloadMutex = new CallbackMutex(); - public static final CompletableFuture NULL_OFFLOAD_PROMISE = CompletableFuture + private final AutomaticOffloadTriggerController automaticOffloadTriggerController = + new AutomaticOffloadTriggerController(); + // Identity sentinel for automatic offload requests. The completed Position value is not used. + public static final CompletableFuture AUTOMATIC_OFFLOAD_TRIGGER = CompletableFuture .completedFuture(PositionFactory.LATEST); + + private enum OffloadRequestSource { + AUTOMATIC, + EXPLICIT + } + + private record OffloadThresholds(long thresholdInBytes, long thresholdInSeconds) { + } + @VisibleForTesting @Getter protected volatile LedgerHandle currentLedger; @@ -1970,7 +1982,7 @@ synchronized void ledgerClosed(final LedgerHandle lh, Long lastAddConfirmed) { trimConsumedLedgersInBackground(); - maybeOffloadInBackground(NULL_OFFLOAD_PROMISE); + maybeOffloadInBackground(AUTOMATIC_OFFLOAD_TRIGGER); createLedgerAfterClosed(); } @@ -2803,22 +2815,73 @@ private void scheduleDeferredTrimming(boolean isTruncate, CompletableFuture p } public void maybeOffloadInBackground(CompletableFuture promise) { - if (getOffloadPoliciesIfAppendable().isEmpty()) { + if (promise == AUTOMATIC_OFFLOAD_TRIGGER) { + if (automaticOffloadTriggerController.requestRun()) { + startAutomaticOffload(); + } + return; + } + + maybeOffloadInBackground(promise, OffloadRequestSource.EXPLICIT); + } + + private void startAutomaticOffload() { + CompletableFuture automaticOffloadCompletion = new CompletableFuture<>(); + automaticOffloadCompletion.whenComplete((res, ex) -> finishAutomaticOffload(ex)); + try { + maybeOffloadInBackground(automaticOffloadCompletion, OffloadRequestSource.AUTOMATIC); + } catch (RuntimeException e) { + automaticOffloadCompletion.completeExceptionally(e); + } + } + + private void maybeOffloadInBackground(CompletableFuture promise, OffloadRequestSource source) { + Optional offloadThresholds = getOffloadThresholds(); + if (offloadThresholds.isEmpty()) { + if (source == OffloadRequestSource.AUTOMATIC) { + promise.complete(PositionFactory.LATEST); + } return; } - final OffloadPolicies policies = config.getLedgerOffloader().getOffloadPolicies(); + OffloadThresholds thresholds = offloadThresholds.get(); + try { + executor.execute(() -> maybeOffload(thresholds.thresholdInBytes(), thresholds.thresholdInSeconds(), + promise, source)); + } catch (RuntimeException e) { + promise.completeExceptionally(e); + } + } + + private Optional getOffloadThresholds() { + Optional optionalOffloadPolicies = getOffloadPoliciesIfAppendable(); + if (optionalOffloadPolicies.isEmpty()) { + return Optional.empty(); + } + + final OffloadPolicies policies = optionalOffloadPolicies.get(); final long offloadThresholdInBytes = Optional.ofNullable(policies.getManagedLedgerOffloadThresholdInBytes()).orElse(-1L); final long offloadThresholdInSeconds = Optional.ofNullable(policies.getManagedLedgerOffloadThresholdInSeconds()).orElse(-1L); if (offloadThresholdInBytes >= 0 || offloadThresholdInSeconds >= 0) { - executor.execute(() -> maybeOffload(offloadThresholdInBytes, offloadThresholdInSeconds, promise)); + return Optional.of(new OffloadThresholds(offloadThresholdInBytes, offloadThresholdInSeconds)); + } + + return Optional.empty(); + } + + private void finishAutomaticOffload(Throwable exception) { + if (exception != null && log.isDebugEnabled()) { + log.debug("Failed to automatically offload ledgers", exception); + } + if (automaticOffloadTriggerController.completeRun()) { + startAutomaticOffload(); } } private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInSeconds, - CompletableFuture finalPromise) { + CompletableFuture finalPromise, OffloadRequestSource source) { if (getOffloadPoliciesIfAppendable().isEmpty()) { String msg = String.format("[%s] Nothing to offload due to offloader or offloadPolicies is NULL", name); finalPromise.completeExceptionally(new IllegalArgumentException(msg)); @@ -2833,8 +2896,12 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS } if (!offloadMutex.tryLock()) { - scheduledExecutor.schedule(() -> maybeOffloadInBackground(finalPromise), - 100, TimeUnit.MILLISECONDS); + try { + scheduledExecutor.schedule(() -> maybeOffloadInBackground(finalPromise, source), + 100, TimeUnit.MILLISECONDS); + } catch (RuntimeException e) { + finalPromise.completeExceptionally(e); + } return; } @@ -2924,12 +2991,11 @@ void internalTrimConsumedLedgers(CompletableFuture promise) { private Optional getOffloadPoliciesIfAppendable() { LedgerOffloader ledgerOffloader = config.getLedgerOffloader(); - if (ledgerOffloader == null - || !ledgerOffloader.isAppendable() - || ledgerOffloader.getOffloadPolicies() == null) { + if (ledgerOffloader == null || !ledgerOffloader.isAppendable()) { return Optional.empty(); } - return Optional.ofNullable(ledgerOffloader.getOffloadPolicies()); + OffloadPolicies offloadPolicies = ledgerOffloader.getOffloadPolicies(); + return Optional.ofNullable(offloadPolicies); } @VisibleForTesting diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java new file mode 100644 index 0000000000000..9aefc7e3e1e1f --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import static org.assertj.core.api.Assertions.assertThat; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.testng.annotations.Test; + +public class AutomaticOffloadTriggerControllerTest { + + @Test + public void triggersCoalesceWhileRunIsActive() { + AutomaticOffloadTriggerController controller = new AutomaticOffloadTriggerController(); + + assertThat(controller.requestRun()).isTrue(); + assertThat(controller.requestRun()).isFalse(); + assertThat(controller.requestRun()).isFalse(); + } + + @Test + public void pendingTriggerSchedulesOneFollowUpRun() { + AutomaticOffloadTriggerController controller = new AutomaticOffloadTriggerController(); + + assertThat(controller.requestRun()).isTrue(); + assertThat(controller.requestRun()).isFalse(); + + assertThat(controller.completeRun()).isTrue(); + assertThat(controller.completeRun()).isFalse(); + assertThat(controller.requestRun()).isTrue(); + } + + @Test(timeOut = 30000) + public void concurrentTriggerAndCompletionAlwaysReserveOneFollowUpRun() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + for (int i = 0; i < 1000; i++) { + AutomaticOffloadTriggerController controller = new AutomaticOffloadTriggerController(); + assertThat(controller.requestRun()).isTrue(); + + // Completion and a new trigger can race; exactly one side must reserve the follow-up run. + CyclicBarrier barrier = new CyclicBarrier(3); + Future completeResult = executor.submit(() -> { + barrier.await(5, TimeUnit.SECONDS); + return controller.completeRun(); + }); + Future triggerResult = executor.submit(() -> { + barrier.await(5, TimeUnit.SECONDS); + return controller.requestRun(); + }); + + barrier.await(5, TimeUnit.SECONDS); + boolean followUpReservedByComplete = completeResult.get(5, TimeUnit.SECONDS); + boolean followUpReservedByTrigger = triggerResult.get(5, TimeUnit.SECONDS); + + assertThat(followUpReservedByComplete) + .as("iteration %s must reserve exactly one follow-up run", i) + .isNotEqualTo(followUpReservedByTrigger); + assertThat(controller.completeRun()).isFalse(); + } + } finally { + executor.shutdownNow(); + executor.awaitTermination(5, TimeUnit.SECONDS); + } + } +} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java index b9c95f445cc33..76c223b0405d3 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java @@ -1184,6 +1184,130 @@ public CompletableFuture offload(ReadHandle ledger, } } + @Test + public void automaticOffloadTriggersAreCoalescedWhileOffloadInProgress() throws Exception { + CompletableFuture slowOffload = new CompletableFuture<>(); + CountDownLatch offloadRunning = new CountDownLatch(1); + AtomicInteger offloadPolicyCalls = new AtomicInteger(); + MockLedgerOffloader offloader = new MockLedgerOffloader() { + @Override + public CompletableFuture offload(ReadHandle ledger, + UUID uuid, + Map extraMetadata) { + offloadRunning.countDown(); + return slowOffload.thenCompose((res) -> super.offload(ledger, uuid, extraMetadata)); + } + + @Override + public OffloadPoliciesImpl getOffloadPolicies() { + offloadPolicyCalls.incrementAndGet(); + return super.getOffloadPolicies(); + } + }; + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(10); + config.setRetentionTime(10, TimeUnit.MINUTES); + config.setRetentionSizeInMB(10); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInBytes(0L); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInSeconds(null); + config.setLedgerOffloader(offloader); + + ManagedLedgerImpl ledger = + (ManagedLedgerImpl) factory.open("my_test_ledger" + UUID.randomUUID(), config); + + for (int i = 0; i < 25; i++) { + ledger.addEntry(buildEntry(10, "entry-" + i)); + } + assertTrue(offloadRunning.await(5, TimeUnit.SECONDS)); + + // Repeated automatic triggers should stop at the controller and avoid another policy lookup. + int callsBeforeRepeatedTriggers = offloadPolicyCalls.get(); + for (int i = 0; i < 20; i++) { + ledger.maybeOffloadInBackground(ManagedLedgerImpl.AUTOMATIC_OFFLOAD_TRIGGER); + } + + assertEquals(offloadPolicyCalls.get(), callsBeforeRepeatedTriggers); + + slowOffload.complete(null); + + assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2); + List allLedgerIds = ledger.getLedgersInfoAsList().stream().map(LedgerInfo::getLedgerId).toList(); + assertEquals(offloader.offloadedLedgers(), Set.of(allLedgerIds.get(0), allLedgerIds.get(1))); + } + + @Test + public void automaticOffloadRunsAgainForCoalescedTrigger() throws Exception { + CompletableFuture slowOffload = new CompletableFuture<>(); + CountDownLatch offloadRunning = new CountDownLatch(1); + MockLedgerOffloader offloader = new MockLedgerOffloader() { + @Override + public CompletableFuture offload(ReadHandle ledger, + UUID uuid, + Map extraMetadata) { + offloadRunning.countDown(); + return slowOffload.thenCompose((res) -> super.offload(ledger, uuid, extraMetadata)); + } + }; + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(10); + config.setRetentionTime(10, TimeUnit.MINUTES); + config.setRetentionSizeInMB(10); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInBytes(0L); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInSeconds(null); + config.setLedgerOffloader(offloader); + + ManagedLedgerImpl ledger = + (ManagedLedgerImpl) factory.open("my_test_ledger" + UUID.randomUUID(), config); + + for (int i = 0; i < 11; i++) { + ledger.addEntry(buildEntry(10, "entry-" + i)); + } + assertTrue(offloadRunning.await(5, TimeUnit.SECONDS)); + + // The next ledger closes after the first automatic scan, so it depends on the coalesced rerun. + for (int i = 11; i < 21; i++) { + ledger.addEntry(buildEntry(10, "entry-" + i)); + } + assertEquals(offloader.offloadedLedgers().size(), 0); + + slowOffload.complete(null); + + assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2); + List allLedgerIds = ledger.getLedgersInfoAsList().stream().map(LedgerInfo::getLedgerId).toList(); + assertEquals(offloader.offloadedLedgers(), Set.of(allLedgerIds.get(0), allLedgerIds.get(1))); + } + + @Test + public void automaticOffloadWithoutThresholdDoesNotBlockLaterTriggers() throws Exception { + MockLedgerOffloader offloader = new MockLedgerOffloader(); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(10); + config.setRetentionTime(10, TimeUnit.MINUTES); + config.setRetentionSizeInMB(10); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInBytes(-1L); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInSeconds(null); + config.setLedgerOffloader(offloader); + + ManagedLedgerImpl ledger = + (ManagedLedgerImpl) factory.open("my_test_ledger" + UUID.randomUUID(), config); + + for (int i = 0; i < 25; i++) { + ledger.addEntry(buildEntry(10, "entry-" + i)); + } + ledger.maybeOffloadInBackground(ManagedLedgerImpl.AUTOMATIC_OFFLOAD_TRIGGER); + assertEquals(offloader.offloadedLedgers().size(), 0); + + // A disabled automatic trigger must complete internally so a later valid trigger can run. + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInBytes(0L); + ledger.maybeOffloadInBackground(ManagedLedgerImpl.AUTOMATIC_OFFLOAD_TRIGGER); + + assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2); + List allLedgerIds = ledger.getLedgersInfoAsList().stream().map(LedgerInfo::getLedgerId).toList(); + assertEquals(offloader.offloadedLedgers(), Set.of(allLedgerIds.get(0), allLedgerIds.get(1))); + } + @DataProvider(name = "offloadAsSoonAsClosed") public Object[][] offloadAsSoonAsClosedProvider() { return new Object[][]{ From edac8b150876220669d408e560830664199830a3 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Thu, 4 Jun 2026 03:50:24 +0800 Subject: [PATCH 123/201] [fix][test] Stabilize testSecondaryIsolationGroupsBookiesNegative() test (#25900) (cherry picked from commit 7ab09413c8bccaa759ce5efcd187cd8e7bd81580) --- .../IsolatedBookieEnsemblePlacementPolicyTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java index fc67395c813c4..d4785d6ca83d5 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java @@ -595,6 +595,12 @@ public void testSecondaryIsolationGroupsBookiesNegative() throws Exception { NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies); + // Wait for the async cache load triggered by initialize() to complete; otherwise the + // first newEnsemble call can race with an empty cachedRackConfiguration and skip isolation. + Awaitility.await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> + assertNotNull(isolationPolicy.getBookieMappingCache() + .getIfCached(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH))); + try { isolationPolicy .newEnsemble(3, 3, 2, Collections.emptyMap(), new HashSet<>()).getResult(); From fa9865cd8920370457c034fdae5a16724987bbd9 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 4 Jun 2026 09:21:18 +0300 Subject: [PATCH 124/201] [improve][misc] Upgrade vert.x to 4.5.28 (#25924) (cherry picked from commit 1bbe964056669315507b0c12f828ec6d3c671f1e) --- distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++----- pom.xml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 0dd4c17211200..4eb3abb6c12c4 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -508,11 +508,11 @@ The Apache Software License, Version 2.0 * JCTools - Java Concurrency Tools for the JVM - org.jctools-jctools-core-4.0.5.jar * Vertx - - io.vertx-vertx-auth-common-4.5.27.jar - - io.vertx-vertx-bridge-common-4.5.27.jar - - io.vertx-vertx-core-4.5.27.jar - - io.vertx-vertx-web-4.5.27.jar - - io.vertx-vertx-web-common-4.5.27.jar + - io.vertx-vertx-auth-common-4.5.28.jar + - io.vertx-vertx-bridge-common-4.5.28.jar + - io.vertx-vertx-core-4.5.28.jar + - io.vertx-vertx-web-4.5.28.jar + - io.vertx-vertx-web-common-4.5.28.jar * Apache ZooKeeper - org.apache.zookeeper-zookeeper-jute-3.9.5.jar * Snappy Java diff --git a/pom.xml b/pom.xml index 19d9ab09f42bf..d722e39e3a133 100644 --- a/pom.xml +++ b/pom.xml @@ -198,7 +198,7 @@ flexible messaging model and an intuitive client API. 2.42 1.10.62 0.16.0 - 4.5.27 + 4.5.28 7.9.2 2.0.17 4.5.0 From 9ce25e3ce38cecce3e07a6e985eb239bdd17d742 Mon Sep 17 00:00:00 2001 From: iantowey <44482548+iantowey@users.noreply.github.com> Date: Wed, 3 Jun 2026 21:00:08 +0100 Subject: [PATCH 125/201] [improve][functions] Allow customizing Kubernetes service domain suffix in Function Worker (#25872) Co-authored-by: Ian (cherry picked from commit cc9fddcd02939ba6c735d88a2bfe702a22fb2ff4) --- conf/functions_worker.yml | 3 +++ .../runtime/kubernetes/KubernetesRuntime.java | 12 ++++++---- .../kubernetes/KubernetesRuntimeFactory.java | 4 +++- .../KubernetesRuntimeFactoryConfig.java | 7 +++++- .../kubernetes/KubernetesRuntimeTest.java | 23 +++++++++++++++++++ 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 6f995576ebd64..794cfba1d519b 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -198,6 +198,9 @@ functionRuntimeFactoryConfigs: # # The Kubernetes pod name to run the function instances. It is set to # # `pf----` if this setting is left to be empty # jobName: +# # Optional domain suffix to use when the Function Worker constructs the gRPC address to connect to function instances. +# # If left blank, it defaults to `.svc.cluster.local`. Set this if your Function Worker is outside the cluster and connects via an external Gateway/Ingress. +# kubernetesServiceDomainSuffix: # # the docker image to run function instance. by default it is `apachepulsar/pulsar` # pulsarDockerImageName: # # the docker image to run function instance according to different configurations provided by users. diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index 80d1a7af89cd4..7cefda12bb97a 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -151,11 +151,13 @@ public class KubernetesRuntime implements Runtime { private final Optional manifestCustomizer; private String functionInstanceClassPath; private String downloadDirectory; + private final String kubernetesServiceDomainSuffix; KubernetesRuntime(AppsV1Api appsClient, CoreV1Api coreClient, String jobNamespace, String jobName, + String kubernetesServiceDomainSuffix, Map customLabels, Boolean installUserCodeDependencies, String pythonDependencyRepository, @@ -194,6 +196,7 @@ public class KubernetesRuntime implements Runtime { this.instanceConfig = instanceConfig; this.jobNamespace = jobNamespace; this.jobName = jobName; + this.kubernetesServiceDomainSuffix = kubernetesServiceDomainSuffix; this.customLabels = customLabels; this.functionDockerImages = functionDockerImages; this.pulsarDockerImageName = pulsarDockerImageName; @@ -318,7 +321,6 @@ private synchronized void setupGrpcChannelIfNeeded() { channel = new ManagedChannel[instanceConfig.getFunctionDetails().getParallelism()]; stub = new InstanceControlGrpc.InstanceControlFutureStub[instanceConfig.getFunctionDetails() .getParallelism()]; - String jobName = createJobName(instanceConfig.getFunctionDetails(), this.jobName); for (int i = 0; i < instanceConfig.getFunctionDetails().getParallelism(); ++i) { String address = getServiceUrl(jobName, jobNamespace, i); @@ -1144,11 +1146,11 @@ private static String createJobName(String tenant, String namespace, String func final String shortHash = DigestUtils.sha1Hex(jobNameBase).toLowerCase().substring(0, 8); return convertedJobName + "-" + shortHash; } - - private static String getServiceUrl(String jobName, String jobNamespace, int instanceId) { - return String.format("%s-%d.%s.%s.svc.cluster.local", jobName, instanceId, jobName, jobNamespace); + @VisibleForTesting + String getServiceUrl(String jobName, String jobNamespace, int instanceId) { + String suffix = isNotBlank(kubernetesServiceDomainSuffix) ? kubernetesServiceDomainSuffix : "svc.cluster.local"; + return String.format("%s-%d.%s.%s.%s", jobName, instanceId, jobName, jobNamespace, suffix); } - public static void doChecks(Function.FunctionDetails functionDetails, String overridenJobName) { final String jobName = createJobName(functionDetails, overridenJobName); if (!jobName.equals(jobName.toLowerCase())) { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index 3f0a2ae9cc351..b15cc587b2f86 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -101,6 +101,7 @@ public class KubernetesRuntimeFactory implements RuntimeFactory { private String functionInstanceClassPath; private String downloadDirectory; private int gracePeriodSeconds; + private String kubernetesServiceDomainSuffix; @ToString.Exclude @EqualsAndHashCode.Exclude @@ -178,7 +179,7 @@ public void initialize(WorkerConfig workerConfig, AuthenticationConfig authentic if (!Paths.get(this.downloadDirectory).isAbsolute()) { this.downloadDirectory = this.pulsarRootDir + "/" + this.downloadDirectory; } - + this.kubernetesServiceDomainSuffix = factoryConfig.getKubernetesServiceDomainSuffix(); this.submittingInsidePod = factoryConfig.getSubmittingInsidePod(); this.installUserCodeDependencies = factoryConfig.getInstallUserCodeDependencies(); this.pythonDependencyRepository = factoryConfig.getPythonDependencyRepository(); @@ -318,6 +319,7 @@ public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String c // get the namespace for this function overriddenNamespace, overriddenName, + kubernetesServiceDomainSuffix, customLabels, installUserCodeDependencies, pythonDependencyRepository, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java index 43cdc035076c3..ea19692349794 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java @@ -47,7 +47,12 @@ public class KubernetesRuntimeFactoryConfig { doc = "The docker image used to run function instance. By default it is `apachepulsar/pulsar`" ) protected String pulsarDockerImageName; - + @FieldContext( + doc = "Optional domain suffix to use when the Function Worker constructs the gRPC address " + + "to connect to function instances. If left blank, it defaults to `.svc.cluster.local`. " + + "Set this if your Function Worker is outside the cluster and connects via an external Gateway/Ingress." + ) + protected String kubernetesServiceDomainSuffix; @FieldContext( doc = "The function docker images used to run function instance according to different " + "configurations provided by users. By default it is `apachepulsar/pulsar`" diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index 2be701c832a8a..161a029c1c63b 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -338,6 +338,29 @@ InstanceConfig createJavaInstanceConfig(FunctionDetails.Runtime runtime, boolean return config; } + @Test + public void testGetServiceUrl() throws Exception { + factory = createKubernetesRuntimeFactory(null, 10, 1.0, 1.0); + InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, true); + + KubernetesRuntime container1 = factory.createContainer( + config, userJarFile, userJarFile, null, null, 30L); + assertEquals(container1.getServiceUrl("my-job", "my-namespace", 0), + "my-job-0.my-job.my-namespace.svc.cluster.local"); + + KubernetesRuntimeFactory factory2 = createKubernetesRuntimeFactory(null, 10, 1.0, 1.0); + java.lang.reflect.Field field = KubernetesRuntimeFactory.class.getDeclaredField( + "kubernetesServiceDomainSuffix"); + field.setAccessible(true); + field.set(factory2, "custom.gateway.internal"); + + KubernetesRuntime container2 = factory2.createContainer( + config, userJarFile, userJarFile, null, null, 30L); + assertEquals(container2.getServiceUrl("my-job", "my-namespace", 0), + "my-job-0.my-job.my-namespace.custom.gateway.internal"); + } + + @Test public void testRamPadding() throws Exception { verifyRamPadding(0, 1000, 1000); From b91e2ee3a0fc332afe461090f484d93ad798a67f Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Fri, 5 Jun 2026 18:33:10 +0800 Subject: [PATCH 126/201] [fix][client] Match logical topic when removing unacked messages (#25921) Signed-off-by: Dream95 (cherry picked from commit 4e509574e5c50c3edc872dd5a2c65c1b0c35909b) --- .../pulsar/client/api/TopicMessageId.java | 13 ++++++ .../client/impl/TopicMessageIdImpl.java | 7 ++++ .../UnAckedTopicMessageRedeliveryTracker.java | 8 ++-- .../impl/UnAckedTopicMessageTracker.java | 4 +- .../client/impl/TopicMessageIdImplTest.java | 17 ++++++++ ...ckedTopicMessageRedeliveryTrackerTest.java | 40 ++++++++++++++++++- 6 files changed, 82 insertions(+), 7 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java index 4d02a7f4096d6..e2791ebf26a9f 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java @@ -41,6 +41,19 @@ public interface TopicMessageId extends MessageId { */ String getOwnerTopic(); + /** + * Checks if this message's owner topic and the given topic refer to the same base + * partitioned topic by comparing their base partitioned topic names. + * + *

For example, {@code persistent://public/default/my-topic-partition-0} matches + * {@code persistent://public/default/my-topic} or any other partition of that topic. + * Topics sharing only a name prefix (e.g., {@code my-topic} vs {@code my-topic-v2}) do not match. + * + * @param topicName a full topic name (non-partitioned, partitioned, or specific partition) + * @return {@code true} if both topics resolve to the same base partitioned topic name + */ + boolean hasSameBasePartitionedTopic(String topicName); + static TopicMessageId create(String topic, MessageId messageId) { if (messageId instanceof TopicMessageId) { return (TopicMessageId) messageId; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java index 872fe283fb9b6..c9a444bf9fd01 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java @@ -23,6 +23,7 @@ import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.api.TraceableMessageId; +import org.apache.pulsar.common.naming.TopicName; public class TopicMessageIdImpl implements MessageIdAdv, TopicMessageId, TraceableMessageId { @@ -91,6 +92,12 @@ public String getOwnerTopic() { return ownerTopic; } + @Override + public boolean hasSameBasePartitionedTopic(String topicName) { + return TopicName.get(getOwnerTopic()).getPartitionedTopicName() + .equals(TopicName.get(topicName).getPartitionedTopicName()); + } + @Override public long getLedgerId() { return msgId.getLedgerId(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java index 393557cd89adb..6d142c1cfde16 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java @@ -42,8 +42,8 @@ public int removeTopicMessages(String topicName) { Entry> entry = iterator.next(); UnackMessageIdWrapper messageIdWrapper = entry.getKey(); MessageId messageId = messageIdWrapper.getMessageId(); - if (messageId instanceof TopicMessageId - && ((TopicMessageId) messageId).getOwnerTopic().contains(topicName)) { + if (messageId instanceof TopicMessageId topicMessageId + && topicMessageId.hasSameBasePartitionedTopic(topicName)) { entry.getValue().remove(messageIdWrapper); iterator.remove(); messageIdWrapper.recycle(); @@ -54,8 +54,8 @@ public int removeTopicMessages(String topicName) { Iterator iteratorAckTimeOut = ackTimeoutMessages.keySet().iterator(); while (iteratorAckTimeOut.hasNext()) { MessageId messageId = iteratorAckTimeOut.next(); - if (messageId instanceof TopicMessageId - && ((TopicMessageId) messageId).getOwnerTopic().contains(topicName)) { + if (messageId instanceof TopicMessageId topicMessageId + && topicMessageId.hasSameBasePartitionedTopic(topicName)) { iteratorAckTimeOut.remove(); removed++; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageTracker.java index 1cbab5844046f..bac68a9c5dd69 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageTracker.java @@ -40,8 +40,8 @@ public int removeTopicMessages(String topicName) { while (iterator.hasNext()) { Entry> entry = iterator.next(); MessageId messageId = entry.getKey(); - if (messageId instanceof TopicMessageId - && ((TopicMessageId) messageId).getOwnerTopic().contains(topicName)) { + if (messageId instanceof TopicMessageId topicMessageId + && topicMessageId.hasSameBasePartitionedTopic(topicName)) { entry.getValue().remove(messageId); iterator.remove(); removed++; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java index 59f15266c1091..26f4335b75a20 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java @@ -19,8 +19,10 @@ package org.apache.pulsar.client.impl; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; import org.testng.annotations.Test; public class TopicMessageIdImplTest { @@ -54,6 +56,21 @@ public void equalsTest() { assertNotEquals(topicMsgId1, topicMsgId2); } + @Test + public void testHasSameBasePartitionedTopic() { + MessageIdImpl msgId = new MessageIdImpl(0, 0, 0); + TopicMessageIdImpl partitionMsgId = new TopicMessageIdImpl( + "persistent://public/default/my-topic-partition-0", msgId); + assertTrue(partitionMsgId.hasSameBasePartitionedTopic( + "persistent://public/default/my-topic-partition-1")); + assertTrue(partitionMsgId.hasSameBasePartitionedTopic( + "persistent://public/default/my-topic")); + assertFalse(partitionMsgId.hasSameBasePartitionedTopic( + "persistent://public/default/my-topic-v2")); + assertFalse(partitionMsgId.hasSameBasePartitionedTopic( + "persistent://public/default/my-topic-v2-partition-0")); + } + @Test public void testDeprecatedMethods() { BatchMessageIdImpl msgId = new BatchMessageIdImpl(1, 2, 3, 4); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java index 9fdab39145bd8..e0ad1f9111b7e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java @@ -70,10 +70,48 @@ public void testRemoveTopicMessages() { tracker.ackTimeoutMessages.put(msgInAckTimeout, System.currentTimeMillis() + 1_000_000L); assertEquals(tracker.size(), 2); - assertEquals(tracker.removeTopicMessages("my-topic"), 2); + assertEquals(tracker.removeTopicMessages("persistent://public/default/my-topic-partition-0"), 2); assertTrue(tracker.isEmpty()); tracker.close(); } + @Test + @SuppressWarnings("unchecked") + public void testRemoveTopicMessagesDoesNotMatchPrefixTopic() { + PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.instrumentProvider()).thenReturn(InstrumentProvider.NOOP); + when(client.getCnxPool()).thenReturn(connectionPool); + @Cleanup("stop") + Timer timer = new HashedWheelTimer( + new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()), + 1, TimeUnit.MILLISECONDS); + when(client.timer()).thenReturn(timer); + + ConsumerBase consumer = mock(ConsumerBase.class); + doNothing().when(consumer).onAckTimeoutSend(any()); + doNothing().when(consumer).redeliverUnacknowledgedMessages(any()); + + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setAckTimeoutMillis(1_000_000); + conf.setTickDurationMillis(100_000); + conf.setAckTimeoutRedeliveryBackoff(MultiplierRedeliveryBackoff.builder().build()); + + UnAckedTopicMessageRedeliveryTracker tracker = + new UnAckedTopicMessageRedeliveryTracker(client, consumer, conf); + + String ownerTopic = "persistent://public/default/my-topic-v2-partition-0"; + TopicMessageIdImpl msgInPartition = + new TopicMessageIdImpl(ownerTopic, new MessageIdImpl(1L, 0L, -1)); + + assertTrue(tracker.add(msgInPartition)); + assertEquals(tracker.size(), 1); + + assertEquals(tracker.removeTopicMessages("persistent://public/default/my-topic"), 0); + assertEquals(tracker.size(), 1); + + tracker.close(); + } + } From 45cd065d32f09eb85cf71a6fde4ffbc562a34cce Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Fri, 5 Jun 2026 18:33:24 +0800 Subject: [PATCH 127/201] [improve][client] Clean up unacked message tracker when topics are removed in multi-topic consumers (#25923) Signed-off-by: Dream95 (cherry picked from commit 26cf550bb9635a8107e5628800b465778c847a91) --- .../client/impl/MultiTopicsConsumerImpl.java | 17 ++++--- .../impl/PatternMultiTopicsConsumerImpl.java | 1 + .../impl/MultiTopicsConsumerImplTest.java | 51 +++++++++++++++++++ .../PatternMultiTopicsConsumerImplTest.java | 41 +++++++++++++++ 4 files changed, 104 insertions(+), 6 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index f1a543e95d456..f611abb179f68 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -974,6 +974,14 @@ private void removeTopic(String topic) { } } + protected void removeTopicMessagesFromUnackedTracker(String topicName) { + if (unAckedMessageTracker instanceof UnAckedTopicMessageTracker tracker) { + tracker.removeTopicMessages(topicName); + } else if (unAckedMessageTracker instanceof UnAckedTopicMessageRedeliveryTracker tracker) { + tracker.removeTopicMessages(topicName); + } + } + /*** * Subscribe one more given topic. * @param topicName topic name without the partition suffix. @@ -1283,11 +1291,7 @@ public CompletableFuture unsubscribeAsync(String topicName) { }); removeTopic(topicName); - if (unAckedMessageTracker instanceof UnAckedTopicMessageTracker tracker) { - tracker.removeTopicMessages(topicName); - } else if (unAckedMessageTracker instanceof UnAckedTopicMessageRedeliveryTracker tracker){ - tracker.removeTopicMessages(topicName); - } + removeTopicMessagesFromUnackedTracker(topicName); unsubscribeFuture.complete(null); log.info("[{}] [{}] [{}] Unsubscribed Topics Consumer, allTopicPartitionsNumber: {}", @@ -1414,7 +1418,8 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa } } - return FutureUtil.waitForAll(futures); + return FutureUtil.waitForAll(futures) + .thenRun(() -> removeTopicMessagesFromUnackedTracker(topicName)); } else if (oldPartitionNumber < currentPartitionNumber) { allTopicPartitionsNumber.addAndGet(currentPartitionNumber - oldPartitionNumber); partitionedTopics.put(topicName, currentPartitionNumber); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java index 3db3786498660..cbfe483290704 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java @@ -367,6 +367,7 @@ public CompletableFuture onTopicsRemoved(Collection removedTopics) removedPartitionedTopicsForLog.add(String.format("%s with %s partitions", groupedTopicRemoved, partitions)); partitionedTopics.remove(groupedTopicRemoved, partitions); + removeTopicMessagesFromUnackedTracker(groupedTopicRemoved); } } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java index 54175613e3bb8..b4d007d855f88 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java @@ -37,6 +37,7 @@ import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -266,4 +267,54 @@ public void testDontCheckForPartitionsUpdatesOnNonPartitionedTopics() throws Exc verify(clientMock, times(3)).getPartitionedTopicMetadata(any(), anyBoolean(), anyBoolean()); } + @Test + @SuppressWarnings("unchecked") + public void testOnTopicsExtendedRemovedTopicCleansUnackedMessages() { + String topicName = "persistent://public/default/deleted-topic"; + String topicPartition0 = topicName + "-partition-0"; + String topicPartition1 = topicName + "-partition-1"; + String otherTopicPartition = "persistent://public/default/other-topic-partition-0"; + + ConsumerConfigurationData consumerConfData = new ConsumerConfigurationData<>(); + consumerConfData.setSubscriptionName("subscriptionName"); + consumerConfData.setAutoUpdatePartitions(true); + consumerConfData.setAutoUpdatePartitionsIntervalSeconds(60); + consumerConfData.setAckTimeoutMillis(1000); + + MultiTopicsConsumerImpl impl = createMultiTopicsConsumer(consumerConfData); + + impl.partitionedTopics.put(topicName, 2); + impl.allTopicPartitionsNumber.set(2); + + ConsumerImpl partitionConsumer0 = (ConsumerImpl) mock(ConsumerImpl.class); + ConsumerImpl partitionConsumer1 = (ConsumerImpl) mock(ConsumerImpl.class); + when(partitionConsumer0.getTopic()).thenReturn(topicPartition0); + when(partitionConsumer1.getTopic()).thenReturn(topicPartition1); + when(partitionConsumer0.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + when(partitionConsumer1.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + impl.consumers.put(topicPartition0, partitionConsumer0); + impl.consumers.put(topicPartition1, partitionConsumer1); + + TopicMessageIdImpl removedTopicMessageId = new TopicMessageIdImpl(topicPartition0, new MessageIdImpl(1, 1, 0)); + TopicMessageIdImpl otherTopicMessageId = + new TopicMessageIdImpl(otherTopicPartition, new MessageIdImpl(2, 2, 0)); + impl.getUnAckedMessageTracker().add(removedTopicMessageId); + impl.getUnAckedMessageTracker().add(otherTopicMessageId); + assertEquals(impl.getUnAckedMessageTracker().size(), 2); + + when(impl.client.getPartitionsForTopic(topicName, false)).thenReturn(CompletableFuture.completedFuture( + Collections.emptyList())); + + PartitionsChangedListener listener = impl.topicsPartitionChangedListener; + listener.onTopicsExtended(Collections.singleton(topicName)).join(); + + assertTrue(impl.getConsumers().isEmpty()); + assertEquals(impl.partitionedTopics.get(topicName), Integer.valueOf(0)); + assertEquals(impl.allTopicPartitionsNumber.get(), 0); + assertEquals(impl.getUnAckedMessageTracker().size(), 1); + assertTrue(impl.getUnAckedMessageTracker().remove(otherTopicMessageId)); + verify(partitionConsumer0).closeAsync(); + verify(partitionConsumer1).closeAsync(); + } + } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java index cd799edb3bdac..84fdca77febb1 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java @@ -310,6 +310,47 @@ public void testPatternSubscribeAndHashHandlingWithChanges() throws Exception { assertThat(invocationCount.get()).isEqualTo(5); } + @Test + @SuppressWarnings("unchecked") + public void testOnTopicsRemovedCleansUnackedMessagesForRemovedPartitionedTopic() { + String partitionedTopic = "persistent://tenant/namespace/deleted-topic"; + String partition0 = partitionedTopic + "-partition-0"; + String partition1 = partitionedTopic + "-partition-1"; + String otherTopicPartition = "persistent://tenant/namespace/other-topic-partition-0"; + TopicsPattern topicsPattern = + TopicsPatternFactory.create("persistent://tenant/namespace/.*", TopicsPattern.RegexImplementation.JDK); + ConsumerConfigurationData consumerConfData = createConsumerConfigurationData(); + consumerConfData.setAckTimeoutMillis(1000); + + PatternMultiTopicsConsumerImpl consumer = + createPatternMultiTopicsConsumer(consumerConfData, topicsPattern); + + consumer.partitionedTopics.put(partitionedTopic, 2); + + ConsumerImpl partitionConsumer0 = (ConsumerImpl) mock(ConsumerImpl.class); + ConsumerImpl partitionConsumer1 = (ConsumerImpl) mock(ConsumerImpl.class); + when(partitionConsumer0.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + when(partitionConsumer1.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + consumer.consumers.put(partition0, partitionConsumer0); + consumer.consumers.put(partition1, partitionConsumer1); + + TopicMessageIdImpl removedTopicMessageId = new TopicMessageIdImpl(partition0, new MessageIdImpl(1, 1, 0)); + TopicMessageIdImpl otherTopicMessageId = + new TopicMessageIdImpl(otherTopicPartition, new MessageIdImpl(2, 2, 0)); + consumer.getUnAckedMessageTracker().add(removedTopicMessageId); + consumer.getUnAckedMessageTracker().add(otherTopicMessageId); + assertThat(consumer.getUnAckedMessageTracker().size()).isEqualTo(2); + + consumer.topicsChangeListener.onTopicsRemoved(Arrays.asList(partition0, partition1)).join(); + + assertThat(consumer.partitionedTopics.containsKey(partitionedTopic)).isFalse(); + assertThat(consumer.consumers).doesNotContainKeys(partition0, partition1); + assertThat(consumer.getUnAckedMessageTracker().size()).isEqualTo(1); + assertThat(consumer.getUnAckedMessageTracker().remove(otherTopicMessageId)).isTrue(); + verify(partitionConsumer0).closeAsync(); + verify(partitionConsumer1).closeAsync(); + } + private static void runTimerTasks(Deque tasks) throws Exception { // first drain the queue to a list to avoid an infinite loop List taskList = new ArrayList<>(); From ae5f410189eea30b883b6d78845cce630f08c1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AE=B6=E5=90=8D?= Date: Fri, 5 Jun 2026 18:39:05 +0800 Subject: [PATCH 128/201] [fix][client] Preserve equals in FieldParser map values (#25907) (cherry picked from commit f0a3149abedaf0bf0497a9e15134444569de2b16) --- .../org/apache/pulsar/common/util/FieldParser.java | 2 +- .../apache/pulsar/common/util/FieldParserTest.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java index 10c1951ab208b..d6aa2c876b1a7 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java @@ -346,7 +346,7 @@ private static Map stringToMap(String strValue, Class keyType, C String[] tokens = trim(strValue).split(","); Map map = new HashMap<>(); for (String token : tokens) { - String[] keyValue = trim(token).split("="); + String[] keyValue = trim(token).split("=", 2); checkArgument(keyValue.length == 2, strValue + " map-value is not in correct format key1=value,key2=value2"); map.put(convert(trim(keyValue[0]), keyType), convert(trim(keyValue[1]), valueType)); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java index b22170fa46505..1f9a1e2688267 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java @@ -95,6 +95,18 @@ public static class MyConfig { public Set stringSet; } + @Test + public void testMapWithEqualsSignAndEmptyValue() { + Map properties = new HashMap<>(); + properties.put("stringStringMap", "key1=value=1,key2="); + + MyConfig config = new MyConfig(); + FieldParser.update(properties, config); + + assertEquals(config.stringStringMap.get("key1"), "value=1"); + assertEquals(config.stringStringMap.get("key2"), ""); + } + @Test public void testNullStrValue() throws Exception { class TestMap { From 65e999cbff9e2c21c98adfad889b45d1dc8d04e3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 5 Jun 2026 16:35:22 +0300 Subject: [PATCH 129/201] [fix][fn] Fix orphan exclusive producer on creation timeout in WorkerUtils.createExclusiveProducerWithRetry (#25942) (cherry picked from commit 2177b0e44d96e87a698c40d47ebb763a9523f2ee) --- .../pulsar/functions/worker/WorkerUtils.java | 11 ++- .../functions/worker/WorkerUtilsTest.java | 69 +++++++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java index af1edf5c8e80d..c86c6e9a193cc 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java @@ -30,8 +30,8 @@ import java.net.URI; import java.nio.file.Files; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -60,6 +60,7 @@ import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.FunctionInstanceStatsDataImpl; import org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.runtime.Runtime; import org.apache.pulsar.functions.runtime.RuntimeSpawner; @@ -400,13 +401,17 @@ public static Producer createExclusiveProducerWithRetry(PulsarClient cli int tries = 0; do { try { - return client.newProducer().topic(topic) + CompletableFuture> producerFuture = client.newProducer().topic(topic) .accessMode(ProducerAccessMode.Exclusive) .enableBatching(false) .blockIfQueueFull(true) .compressionType(CompressionType.LZ4) .producerName(producerName) - .createAsync().get(10, TimeUnit.SECONDS); + .createAsync(); + return FutureUtil.getAndCleanupOnInterrupt(producerFuture, Producer::closeAsync); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw e; } catch (Exception e) { log.info("Encountered exception while at creating exclusive producer to topic {}", topic, e); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java index 0f5fca4a8a5a3..823b645c672a3 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java @@ -18,26 +18,33 @@ */ package org.apache.pulsar.functions.worker; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; import java.util.HashSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import lombok.Cleanup; import org.apache.distributedlog.DistributedLogConfiguration; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.client.api.CompressionType; @@ -108,6 +115,68 @@ public Boolean get() { } } + @Test + @SuppressWarnings("unchecked") + public void testCreateExclusiveProducerWithRetryClosesProducerOnInterrupt() throws Exception { + Producer producer = mock(Producer.class); + when(producer.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + + // producer creation stays pending until the test completes it explicitly + CompletableFuture> producerFuture = new CompletableFuture<>(); + CountDownLatch createAsyncCalled = new CountDownLatch(1); + + ProducerBuilder builder = mock(ProducerBuilder.class); + when(builder.topic(anyString())).thenReturn(builder); + when(builder.producerName(anyString())).thenReturn(builder); + when(builder.enableBatching(anyBoolean())).thenReturn(builder); + when(builder.blockIfQueueFull(anyBoolean())).thenReturn(builder); + when(builder.compressionType(any(CompressionType.class))).thenReturn(builder); + when(builder.accessMode(any())).thenReturn(builder); + when(builder.createAsync()).thenAnswer(invocation -> { + createAsyncCalled.countDown(); + return producerFuture; + }); + + PulsarClient pulsarClient = mock(PulsarClient.class); + when(pulsarClient.newProducer()).thenReturn(builder); + + AtomicReference thrown = new AtomicReference<>(); + AtomicBoolean interruptStatusPreserved = new AtomicBoolean(); + @Cleanup("interrupt") + Thread caller = new Thread(() -> { + try { + WorkerUtils.createExclusiveProducerWithRetry(pulsarClient, "test-topic", "test-producer", + () -> true, 0); + } catch (Throwable t) { + thrown.set(t); + interruptStatusPreserved.set(Thread.currentThread().isInterrupted()); + } + }); + caller.setDaemon(true); + caller.start(); + assertTrue(createAsyncCalled.await(10, TimeUnit.SECONDS)); + + // interrupt the caller while it is waiting for the producer to be created + caller.interrupt(); + caller.join(TimeUnit.SECONDS.toMillis(10)); + assertThat(caller.isAlive()) + .as("Interrupt should abort the retry loop instead of retrying") + .isFalse(); + + assertThat(thrown.get()) + .isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(InterruptedException.class); + assertThat(interruptStatusPreserved) + .as("Interrupt status should be restored") + .isTrue(); + + // when the pending creation completes after the interrupt, the producer must be closed so that + // the exclusive producer doesn't leak + verify(producer, never()).closeAsync(); + producerFuture.complete(producer); + verify(producer, times(1)).closeAsync(); + } + @Test public void testDLogConfiguration() throws URISyntaxException, IOException { // The config yml is seeded with a fake bookie config. From 0bd2b6e66945366232da3d797c6eac4f3455b254 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 5 Jun 2026 18:47:51 +0300 Subject: [PATCH 130/201] Release 4.2.2 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- jetcd-core-shaded/pom.xml | 2 +- jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml | 2 +- jetty-upgrade/pom.xml | 2 +- jetty-upgrade/zookeeper-prometheus-metrics/pom.xml | 2 +- jetty-upgrade/zookeeper-with-patched-admin/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- microbench/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-bom/pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-cli-utils/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-dependencies-minimized/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-docs-tools/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/azure-data-explorer/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis-kpl-shaded/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-opentelemetry/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 135 files changed, 138 insertions(+), 138 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index dca920b94c5fd..45eabb7e3e004 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.2-SNAPSHOT + 4.2.2 bouncy-castle-bc diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 1c4428d1ea4c3..1114969543a93 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.2-SNAPSHOT + 4.2.2 bcfips-include-test diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 2302bdf08b2f3..fc9d055b5f2ff 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.2-SNAPSHOT + 4.2.2 bouncy-castle-bcfips diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 4b0b2a1599558..b135c12cd8bff 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.2-SNAPSHOT + 4.2.2 diff --git a/buildtools/pom.xml b/buildtools/pom.xml index ac6e4c34d1aeb..7652d26aff9ff 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 4.2.2-SNAPSHOT + 4.2.2 jar Pulsar Build Tools @@ -43,7 +43,7 @@ - 2026-04-27T13:54:53Z + 2026-06-05T15:47:49Z 1.8 1.8 8 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 691e317aa861d..0a6604701c29d 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.2-SNAPSHOT + 4.2.2 pulsar-io-distribution diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 9f0af92bb3bcb..bc1c7dcff886c 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.2-SNAPSHOT + 4.2.2 pulsar-offloader-distribution diff --git a/distribution/pom.xml b/distribution/pom.xml index f70857b87cf0f..3c1948acea233 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.2-SNAPSHOT + 4.2.2 distribution diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 1018c1276f218..128744a8ed97e 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.2-SNAPSHOT + 4.2.2 pulsar-server-distribution diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 3698f214d2eb4..94a43e65297a1 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.2-SNAPSHOT + 4.2.2 pulsar-shell-distribution diff --git a/docker/pom.xml b/docker/pom.xml index 9102dc1446ee4..1b26a7b42cbb0 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.2-SNAPSHOT + 4.2.2 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 072248b02fbcd..f3b25777f66f1 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.2-SNAPSHOT + 4.2.2 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index d350aed14d831..1d15f1b3de1a0 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.2-SNAPSHOT + 4.2.2 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index f3af75b637b65..cd52704753694 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.2-SNAPSHOT + 4.2.2 jclouds-shaded diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml index 0a05b3d6d88e7..407228ca19ef2 100644 --- a/jetcd-core-shaded/pom.xml +++ b/jetcd-core-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.2-SNAPSHOT + 4.2.2 jetcd-core-shaded diff --git a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml index f707287b031c8..f2221382d425e 100644 --- a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml +++ b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.2-SNAPSHOT + 4.2.2 pulsar-bookkeeper-prometheus-metrics-provider Apache Pulsar :: BookKeeper Stats Providers :: Prometheus diff --git a/jetty-upgrade/pom.xml b/jetty-upgrade/pom.xml index d1277ddd06b74..0d7618069d346 100644 --- a/jetty-upgrade/pom.xml +++ b/jetty-upgrade/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.2-SNAPSHOT + 4.2.2 jetty-upgrade diff --git a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml index 4473e496f41b6..29a3009d9ab80 100755 --- a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml +++ b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.2-SNAPSHOT + 4.2.2 pulsar-zookeeper-prometheus-metrics diff --git a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml index c233324f93199..83d6b819c9746 100644 --- a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml +++ b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar jetty-upgrade - 4.2.2-SNAPSHOT + 4.2.2 zookeeper-with-patched-admin diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 793bc1cb76700..15d1542c53e13 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.2-SNAPSHOT + 4.2.2 managed-ledger diff --git a/microbench/pom.xml b/microbench/pom.xml index 1c8a2b649f6a6..637c038eb365d 100644 --- a/microbench/pom.xml +++ b/microbench/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.2-SNAPSHOT + 4.2.2 microbench diff --git a/pom.xml b/pom.xml index d722e39e3a133..fdc1bb45f36e8 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 4.2.2-SNAPSHOT + 4.2.2 Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -94,7 +94,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - 2026-04-27T13:54:47Z + 2026-06-05T15:47:43Z true 9.4.58.v20250814 2.5.2 From adc4fe4d3d15343611df464ffbf0b9ebf6e901be Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 27 Apr 2026 14:43:00 +0800 Subject: [PATCH 132/201] [feat][pip] PIP-469: Legacy-aware topic policies backend routing and metadata-store topic policies (#25547) (cherry picked from commit 33fe7559b4d1a2cfb9d35756d38f7bad72b54309) --- pip/pip-469.md | 270 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 pip/pip-469.md diff --git a/pip/pip-469.md b/pip/pip-469.md new file mode 100644 index 0000000000000..69036381f0f01 --- /dev/null +++ b/pip/pip-469.md @@ -0,0 +1,270 @@ +# PIP-469: Legacy-aware topic policies backend routing and metadata-store topic policies + +# Background knowledge + +Apache Pulsar introduced topic-level policies in [PIP-39](pip-39.md). A broker reads and writes these policies through +`TopicPoliciesService`. The default implementation, +`SystemTopicBasedTopicPoliciesService`, persists topic policy changes in the namespace `__change_events` system topic and +keeps an in-memory cache on brokers that own bundles for that namespace. + +[PIP-92](pip-92.md) extended topic policies with the distinction between local and global policies. Any +`TopicPoliciesService` implementation therefore needs to handle two independent values for the same topic: the +cluster-local policy state and the globally visible policy state. + +[PIP-376](pip-376.md) made `TopicPoliciesService` pluggable through the broker configuration +`topicPoliciesServiceClassName`. That change removed the hard coupling between topic policies and system topics, but the +backend choice is still broker-wide. During upgrade from the default system-topic backend, brokers still need a way to +recognize namespaces that already have topic-policies state in `__change_events`, so those namespaces do not silently +move to another backend. + +# Motivation + +The system-topic-based topic policies implementation works by appending topic policy changes to a `__change_events` +topic in each namespace. It works well when this topic has already been loaded by a broker, then all topic policies +operations just access the in-memory cache. However, in cold start scenarios, for example when the owner broker is down +during a restart, the new owner broker has to create a reader on the `__change_events` topic and wait for it to catch +up before it can read any topic policies, which is required in the path of loading a topic in the same namespace. This +adds significant latency to the topic load path, especially before the topic is compacted. + +Things become worse when many `__change_events` topics move to a restarting broker. The new owner broker has to create +many readers and replay all messages on these topics. This leads to high pressure on BookKeeper and can cause +`Too many requests on the same bookie` errors in `GetLastMessageId` RPCs. + +A metadata-store-backed topic policies backend is attractive because it removes the extra lifecycle and operational +dependency of a dedicated `__change_events` topic. A metadata-cache-based implementation can still provide caching and +change notifications, while avoiding the cold-start latency of waiting for a system-topic reader to initialize and +catch up. + +There is a second operational requirement: operators need a safe gradual rollout path. Existing namespaces that already +have topic-policies state in `__change_events` must stay on the system-topic backend, while newly created namespaces +should be able to use the broker-configured backend. This does not require a new namespace policy. For the upgrade case +from the default configuration, the existence of `__change_events` is already a conservative legacy marker. + +# Goals + +## In Scope + +- Add a metadata-store-backed `TopicPoliciesService` implementation that does not depend on system topics. +- Add routing logic that forces the system-topic backend for namespaces that already have `__change_events`. +- Keep using the broker-level `topicPoliciesServiceClassName` for namespaces that do not have `__change_events`, + including newly created namespaces. + +## Out of Scope + +- Adding a migration framework that moves topic policies data between backends automatically. + +# High Level Design + +When topic-level policies are enabled, the broker instantiates a `LegacyAwareTopicPoliciesService` instead of using the +configured implementation directly. + +The wrapper always has access to two backends: + +- `SystemTopicBasedTopicPoliciesService` +- The broker-configured `topicPoliciesServiceClassName` + +For each namespace, the wrapper checks whether the topic-policies system topic `persistent://{tenant}/{namespace}/__change_events` +already exists: + +- If it exists, the namespace is treated as a legacy system-topic namespace and all topic-policies operations are + routed to `SystemTopicBasedTopicPoliciesService`. +- If it does not exist, the namespace uses the broker-configured `topicPoliciesServiceClassName`. + +This rule is intentionally conservative. If `__change_events` exists, the broker assumes that namespace may already +contain topic-policies state in the system-topic backend and therefore must not be moved implicitly. + +This proposal also introduces `MetadataStoreTopicPoliciesService`, a concrete `TopicPoliciesService` implementation +that stores topic policies in dedicated metadata-store paths: + +- Global topic policies are stored in the configuration metadata store. +- Local topic policies are stored in the local metadata store. + +This keeps the storage scope aligned with the semantics introduced by PIP-92 and avoids writing topic policies through +managed-ledger metadata side effects. + +# Detailed Design + +## Design & Implementation Details + +### Startup and validation + +`PulsarService#initTopicPoliciesService()` continues to respect `topicLevelPoliciesEnabled`. When topic-level policies +are disabled, behavior is unchanged and `TopicPoliciesService.DISABLED` is used. + +When topic-level policies are enabled, the broker constructs: + +```java +new LegacyAwareTopicPoliciesService( + this, + new SystemTopicBasedTopicPoliciesService(this), + configuredTopicPoliciesService) +``` + +Broker startup validates both backends: + +- `SystemTopicBasedTopicPoliciesService` must be instantiable. +- The configured `topicPoliciesServiceClassName` must be instantiable. + +If either backend cannot be instantiated or started, broker startup fails. There is no per-request fallback from one +backend to another. + +### Namespace-scoped service routing + +`LegacyAwareTopicPoliciesService` is responsible for: + +- Checking whether `__change_events` exists for the namespace by using + `NamespaceEventsSystemTopicFactory.checkSystemTopicExists(namespace, EventType.TOPIC_POLICY, pulsarService)`. +- Routing `getTopicPoliciesAsync`, `updateTopicPoliciesAsync`, `deleteTopicPoliciesAsync`, and listener operations to + the system-topic backend when the system topic exists. +- Routing the same operations to the configured backend when the system topic does not exist. + +The system-topic existence check can be cached per namespace in memory, but the routing rule is defined by actual topic +existence rather than by new namespace metadata. + +This means: + +- Existing namespaces that already materialized `__change_events` continue to use the system-topic backend. +- Namespaces that never created `__change_events` use the broker-configured backend. +- Newly created namespaces use the broker-configured backend because `__change_events` does not exist yet. + +If `__change_events` is later deleted, the namespace falls back to the broker-configured backend on subsequent +resolution. This matches current system-topic behavior, which already treats a missing `__change_events` topic as +meaning the system-topic-backed topic-policies state is gone. + +### Metadata-backed topic policies service + +`MetadataStoreTopicPoliciesService` implements `TopicPoliciesService` with the following storage model: + +- Topic names are normalized to the partitioned topic name, so all partitions share the same topic-policies record. +- Global policies are stored in the configuration metadata store path: + `/admin/topic-policies/{tenant}/{namespace}/{domain}/{encodedTopic}`. +- Local policies are stored in the local metadata store path: + `/admin/local-policies/topic-policies/{tenant}/{namespace}/{domain}/{encodedTopic}`. + +Each node stores a serialized `TopicPolicies` document. The backend writes and reads the two scopes independently: + +- Reads with `GetType.GLOBAL_ONLY` only touch the global path and return a `TopicPolicies` object whose `isGlobal` + flag is `true`. +- Reads with `GetType.LOCAL_ONLY` only touch the local path and return a `TopicPolicies` object whose `isGlobal` flag + is `false`. +- Updates with `isGlobalPolicy=true` only modify the global path. +- Updates with `isGlobalPolicy=false` only modify the local path. + +Deletes remove the local record and, unless `keepGlobalPoliciesAfterDeleting` is set, also remove the global record. +This matches the existing `TopicPoliciesService` deletion contract. + +This design intentionally uses dedicated metadata nodes instead of piggybacking on `PartitionedTopicMetadata` or +`ManagedLedgerInfo`. That keeps local/global visibility correct and avoids losing topic policies during normal +managed-ledger metadata updates. + +### Listener behavior + +The backend registers watchers on both metadata stores: + +- A change on the local path re-reads the local node and notifies listeners with the latest local `TopicPolicies` or + `null` if the local node was removed. +- A change on the global path re-reads the global node and notifies listeners with the latest global `TopicPolicies` + or `null` if the global node was removed. + +This preserves runtime updates for already loaded topics, including global topic policies. The backend does not add an +append-only replay log; it relies on metadata-store notifications and read-after-notify refresh. + +## Public-facing Changes + +### Public API + +No new namespace policy field is introduced. + +No new namespace admin REST endpoint or Java admin client method is introduced. + +Changing the topic-policies backend for a namespace is not a public operation in this proposal. The routing rule is +derived from `__change_events` existence plus the broker-level configuration. + +### Binary protocol + +No binary protocol changes. + +### Configuration + +- `topicPoliciesServiceClassName` + - Continues to define the broker-configured `TopicPoliciesService` implementation. + - Namespaces that do not have `__change_events` use this backend. + - Namespaces that already have `__change_events` keep using `SystemTopicBasedTopicPoliciesService` regardless of + this value. + +### CLI + +No CLI change in this proposal. + +### Metrics + +No new metric is required. + +# Backward & Forward Compatibility + +## Upgrade + +The intended upgrade flow is: + +1. Upgrade brokers to a version that understands legacy-aware backend routing. +2. Change `topicPoliciesServiceClassName` to the alternate backend if newly created namespaces should use it. +3. Existing namespaces that already have `__change_events` continue to use `SystemTopicBasedTopicPoliciesService`. +4. Namespaces that do not have `__change_events`, including newly created namespaces, use the configured backend. + +No namespace metadata backfill is required. + +This upgrade rule is intentionally conservative: + +- If `__change_events` exists, the namespace stays on the system-topic backend. +- If `__change_events` does not exist, the namespace uses the configured backend. + +This means some namespaces with an empty but already-created `__change_events` topic may continue using the +system-topic backend. That is acceptable because it avoids missing legacy state. + +## Downgrade / Rollback + +Rolling back to a broker version that does not understand legacy-aware routing returns topic-policies backend +selection to pure broker-wide behavior. + +- The older broker will no longer special-case namespaces that have `__change_events`. +- Operators will need to choose one broker-wide backend for the rollback cluster, or migrate data before rollback if + both legacy system-topic namespaces and metadata-store namespaces must coexist. + +## Pulsar Geo-Replication Upgrade & Downgrade/Rollback Considerations + +This proposal does not introduce a new geo-replication protocol for topic policies. + +- Global topic policies stay in the configuration metadata store and therefore keep global visibility semantics. +- Local topic policies stay in the local metadata store and therefore keep cluster-local visibility semantics. +- Legacy namespaces are recognized by the existence of `__change_events`, which is already shared broker-visible topic + metadata. + +# Alternatives + +## Keep a single broker-wide topic policies backend + +This keeps the implementation simpler, but it does not solve the operational requirement to keep existing namespaces on +their current backend while directing newly created namespaces to a different one. + +## Persist an explicit namespace backend marker + +This would also solve the upgrade problem, but it introduces new namespace-scoped metadata changes that are not +necessary for the default-system-topic upgrade path. The proposal prefers to reuse the already existing +`__change_events` artifact as the legacy marker. + +## Add a user-managed namespace override API + +This provides more flexibility than needed, but it also reintroduces runtime switching, rollback ambiguity, and the +risk of one namespace being served by different backends if brokers do not resolve the override identically. The +proposal intentionally avoids this surface. + +# General Notes + +This proposal is a follow-up to [PIP-376](pip-376.md). It keeps backend selection pluggable, but handles upgrade from +the legacy system-topic backend by reusing `__change_events` as the compatibility marker instead of introducing a new +namespace-level policy or namespace-level metadata field. + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/sn2pyyl9p1vm5vr8j8qssxbbksm2bzfr +* Mailing List voting thread: https://lists.apache.org/thread/b5mfqrmxcwwzjbkhzv6t6t12gtvjz1so From 712372f9ee18ae5d5bd8e35ca6ef45473c0a3bb5 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 18 May 2026 19:16:47 +0800 Subject: [PATCH 133/201] [feat][broker] PIP-469: Legacy-aware topic policies backend routing and metadata-store topic policies (#25707) (cherry picked from commit 8652efa4d60b791a5f1ee4e52f7ffda6ebbbb256) --- pip/pip-469.md | 37 ++- .../pulsar/broker/ServiceConfiguration.java | 12 +- .../apache/pulsar/broker/PulsarService.java | 10 +- .../pulsar/broker/service/AbstractTopic.java | 2 +- .../LegacyAwareTopicPoliciesService.java | 142 +++++++++ .../MetadataStoreTopicPoliciesService.java | 277 ++++++++++++++++++ .../SystemTopicBasedTopicPoliciesService.java | 2 +- .../broker/service/TopicPoliciesService.java | 15 +- .../service/persistent/PersistentTopic.java | 8 +- .../admin/MetadataStoreTopicPoliciesTest.java | 72 +++++ .../broker/admin/TopicPoliciesTest.java | 112 +++++-- .../LegacyAwareTopicPoliciesServiceTest.java | 190 ++++++++++++ ...temTopicBasedTopicPoliciesServiceTest.java | 4 +- .../broker/service/TopicPolicyTestUtils.java | 7 + 14 files changed, 846 insertions(+), 44 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MetadataStoreTopicPoliciesService.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesServiceTest.java diff --git a/pip/pip-469.md b/pip/pip-469.md index 69036381f0f01..4734adc127d4c 100644 --- a/pip/pip-469.md +++ b/pip/pip-469.md @@ -105,8 +105,15 @@ Broker startup validates both backends: - `SystemTopicBasedTopicPoliciesService` must be instantiable. - The configured `topicPoliciesServiceClassName` must be instantiable. -If either backend cannot be instantiated or started, broker startup fails. There is no per-request fallback from one -backend to another. +`LegacyAwareTopicPoliciesService#start` starts only the configured backend. It intentionally does not call +`SystemTopicBasedTopicPoliciesService#start`, because that start path registers a namespace-bundle ownership listener +whose only purpose is to eagerly create a reader on `/__change_events` when a namespace bundle is loaded. +Under legacy-aware routing, that eager optimization would be counterproductive because it can create readers for +namespaces that do not have topic policies in `__change_events`. For legacy namespaces, the system-topic reader and +policy cache are initialized lazily by the routed system-topic backend operations. + +If either backend cannot be instantiated, or if the configured backend cannot be started, broker startup fails. There is +no per-request fallback from one backend to another. ### Namespace-scoped service routing @@ -118,6 +125,10 @@ backend to another. the system-topic backend when the system topic exists. - Routing the same operations to the configured backend when the system topic does not exist. +Listener registration is routed through `TopicPoliciesService#registerListenerAsync`. This lets the wrapper resolve the +namespace backend before registering the listener, and the listener is registered only on the selected backend instead +of being registered on both backends. + The system-topic existence check can be cached per namespace in memory, but the routing rule is defined by actual topic existence rather than by new namespace metadata. @@ -137,9 +148,13 @@ meaning the system-topic-backed topic-policies state is gone. - Topic names are normalized to the partitioned topic name, so all partitions share the same topic-policies record. - Global policies are stored in the configuration metadata store path: - `/admin/topic-policies/{tenant}/{namespace}/{domain}/{encodedTopic}`. + `/admin/topic-policies/global/{tenant}/{namespace}/{domain}/{encodedTopic}`. - Local policies are stored in the local metadata store path: - `/admin/local-policies/topic-policies/{tenant}/{namespace}/{domain}/{encodedTopic}`. + `/admin/topic-policies/local/{tenant}/{namespace}/{domain}/{encodedTopic}`. + +To avoid possible conflicts like the listener registered on the `/admin/local-policies` path from +`BrokerService#handleMetadataChanges`, these two paths share the same root path `/admin/topic-policies`, which is not +used by any other component. Each node stores a serialized `TopicPolicies` document. The backend writes and reads the two scopes independently: @@ -159,6 +174,11 @@ managed-ledger metadata updates. ### Listener behavior +`TopicPoliciesService` adds `registerListenerAsync(TopicName, TopicPolicyListener)` for listener registration. The +existing synchronous `registerListener(TopicName, TopicPolicyListener)` method is retained as a deprecated compatibility +hook for existing custom implementations, and the default async method delegates to it. Implementations that need async +routing or initialization, such as `LegacyAwareTopicPoliciesService`, override `registerListenerAsync` directly. + The backend registers watchers on both metadata stores: - A change on the local path re-reads the local node and notifies listeners with the latest local `TopicPolicies` or @@ -173,6 +193,11 @@ append-only replay log; it relies on metadata-store notifications and read-after ### Public API +The `TopicPoliciesService` extension point gains a default +`CompletableFuture registerListenerAsync(TopicName, TopicPolicyListener)` method. Existing implementations +remain compatible because `registerListener(TopicName, TopicPolicyListener)` is retained and used by the default async +implementation. + No new namespace policy field is introduced. No new namespace admin REST endpoint or Java admin client method is introduced. @@ -221,6 +246,10 @@ This upgrade rule is intentionally conservative: This means some namespaces with an empty but already-created `__change_events` topic may continue using the system-topic backend. That is acceptable because it avoids missing legacy state. +Existing custom `TopicPoliciesService` implementations that only implement the synchronous `registerListener` method +continue to work through the default `registerListenerAsync` bridge. Implementations can override +`registerListenerAsync` when registration itself needs asynchronous backend resolution or initialization. + ## Downgrade / Rollback Rolling back to a broker version that does not understand legacy-aware routing returns topic-policies backend diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 0931aa64114a1..bf6ef7730e520 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1732,8 +1732,16 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece @FieldContext( category = CATEGORY_SERVER, - doc = "The class name of the topic policies service. The default config only takes affect when the " - + "systemTopicEnable config is true" + doc = """ + The class name of the topic policies service. There are 2 built-in implementations: + 1. "org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService" (default) + It stores a topic's policies in the `__change_events` topic. If `systemTopicEnabled` is false, + the topic policies will just be disabled + 2. "org.apache.pulsar.broker.service.MetadataStoreTopicPoliciesService" + It stores a topic's policies in the metadata store. If `systemTopicEnabled` is true and the + topic's namespace has a `__change_events` topic, the policies will still be stored in the + `__change_events` topic for backward compatibility. + """ ) private String topicPoliciesServiceClassName = "org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService"; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 749dfc13b17df..bf1f0e24848b1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -106,6 +106,7 @@ import org.apache.pulsar.broker.rest.Topics; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.HealthChecker; +import org.apache.pulsar.broker.service.LegacyAwareTopicPoliciesService; import org.apache.pulsar.broker.service.PulsarMetadataEventSynchronizer; import org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService; import org.apache.pulsar.broker.service.Topic; @@ -2256,8 +2257,15 @@ private TopicPoliciesService initTopicPoliciesService() throws Exception { return TopicPoliciesService.DISABLED; } } - return (TopicPoliciesService) Reflections.createInstance(className, + final var configuredService = (TopicPoliciesService) Reflections.createInstance(className, Thread.currentThread().getContextClassLoader()); + if (!config.isSystemTopicEnabled()) { + LOG.info("[{}] System topic is disabled, using configured topic policies service without legacy routing", + className); + return configuredService; + } + return new LegacyAwareTopicPoliciesService(this, new SystemTopicBasedTopicPoliciesService(this), + configuredService); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 8be1002b3a8db..9f6f2e2ae3826 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -552,7 +552,7 @@ protected boolean isProducersExceeded(boolean isRemote) { protected void registerTopicPolicyListener() { brokerService.getPulsar().getTopicPoliciesService() - .registerListener(TopicName.getPartitionedTopicName(topic), this); + .registerListenerAsync(TopicName.getPartitionedTopicName(topic), this); } protected void unregisterTopicPolicyListener() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java new file mode 100644 index 0000000000000..6e1bfd7bacc42 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.annotations.VisibleForTesting; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.function.Consumer; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; +import org.apache.pulsar.common.events.EventType; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.jspecify.annotations.NonNull; + +/** + * Routes topic policy operations to the legacy system-topic backend when a namespace already has + * a topic-policy {@code __change_events} system topic, and otherwise to the configured backend. + */ +public class LegacyAwareTopicPoliciesService implements TopicPoliciesService { + + private final AsyncLoadingCache isLegacyNamespace; + @VisibleForTesting + final SystemTopicBasedTopicPoliciesService systemTopicService; + private final TopicPoliciesService configuredService; + + public LegacyAwareTopicPoliciesService(PulsarService pulsar, + SystemTopicBasedTopicPoliciesService systemTopicService, + TopicPoliciesService configuredService) { + // Generally, we only need to check if the __change_events topic exists once because the __change_events topic + // should only be created by broker before the upgrade, where `SystemTopicBasedTopicPoliciesService` is + // configured as the topic policies service. + this.isLegacyNamespace = Caffeine.newBuilder().expireAfterWrite(Duration.ofHours(1)) + .buildAsync(new AsyncCacheLoader<>() { + @NonNull + @Override + public CompletableFuture asyncLoad(NamespaceName key, + @NonNull Executor executor) { + return NamespaceEventsSystemTopicFactory.checkSystemTopicExists(key, EventType.TOPIC_POLICY, + pulsar); + } + }); + this.systemTopicService = systemTopicService; + this.configuredService = configuredService; + if (configuredService instanceof SystemTopicBasedTopicPoliciesService) { + throw new IllegalArgumentException( + "configuredService should not be an instance of SystemTopicBasedTopicPoliciesService"); + } + } + + @Override + public void start(PulsarService pulsarService) { + // We should not call `systemTopicService.start()`, which just registers a namespace bundle listener to create + // a reader on `/__change_events` when the namespace's bundle is loaded firstly. It's just an + // optimization to create the reader before loading any topic. However, it could create a reader on a namespace + // that does not even have the __change_events topic. + configuredService.start(pulsarService); + } + + @Override + public void close() throws Exception { + try { + configuredService.close(); + } finally { + systemTopicService.close(); + } + } + + @Override + public CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type) { + return resolveService(topicName.getNamespaceObject()) + .thenCompose(service -> service.getTopicPoliciesAsync(topicName, type)); + } + + @Override + public CompletableFuture updateTopicPoliciesAsync(TopicName topicName, boolean isGlobalPolicy, + boolean skipUpdateWhenTopicPolicyDoesntExist, + Consumer policyUpdater) { + return resolveService(topicName.getNamespaceObject()) + .thenCompose(service -> service.updateTopicPoliciesAsync(topicName, isGlobalPolicy, + skipUpdateWhenTopicPolicyDoesntExist, policyUpdater)); + } + + @Override + public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName) { + return resolveService(topicName.getNamespaceObject()) + .thenCompose(service -> service.deleteTopicPoliciesAsync(topicName)); + } + + @Override + public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName, + boolean keepGlobalPoliciesAfterDeleting) { + return resolveService(topicName.getNamespaceObject()) + .thenCompose(service -> service.deleteTopicPoliciesAsync(topicName, + keepGlobalPoliciesAfterDeleting)); + } + + @Override + public CompletableFuture registerListenerAsync(TopicName topicName, TopicPolicyListener listener) { + return resolveService(topicName.getNamespaceObject()) + .thenCompose(service -> service.registerListenerAsync(topicName, listener)); + } + + @Override + public boolean registerListener(TopicName topicName, TopicPolicyListener listener) { + throw new RuntimeException("should not be called"); + } + + @Override + public void unregisterListener(TopicName topicName, TopicPolicyListener listener) { + configuredService.unregisterListener(topicName, listener); + systemTopicService.unregisterListener(topicName, listener); + } + + @VisibleForTesting + CompletableFuture resolveService(NamespaceName namespace) { + return isLegacyNamespace.get(namespace) + .thenApply(isLegacy -> isLegacy ? systemTopicService : configuredService); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MetadataStoreTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MetadataStoreTopicPoliciesService.java new file mode 100644 index 0000000000000..218c37472a3d5 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MetadataStoreTopicPoliciesService.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.annotations.VisibleForTesting; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.apache.pulsar.common.util.Codec; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataCache; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException; +import org.apache.pulsar.metadata.api.Notification; +import org.apache.pulsar.metadata.api.NotificationType; +import org.jspecify.annotations.Nullable; + +/** + * Topic policies service backed by Pulsar metadata stores. + */ +@Slf4j +public class MetadataStoreTopicPoliciesService implements TopicPoliciesService { + + public static final String GLOBAL_POLICIES_ROOT = "/admin/topic-policies/global"; + public static final String LOCAL_POLICIES_ROOT = "/admin/topic-policies/local"; + + private final AtomicBoolean closed = new AtomicBoolean(false); + private final Map> listeners = new ConcurrentHashMap<>(); + private MetadataCache localPoliciesCache; + private MetadataCache globalPoliciesCache; + + @Override + public void start(PulsarService pulsar) { + MetadataStore localStore = pulsar.getLocalMetadataStore(); + MetadataStore configurationStore = pulsar.getConfigurationMetadataStore(); + this.localPoliciesCache = localStore.getMetadataCache(TopicPolicies.class); + this.globalPoliciesCache = configurationStore.getMetadataCache(TopicPolicies.class); + localStore.registerListener(notification -> handleNotification(notification, false)); + configurationStore.registerListener(notification -> handleNotification(notification, true)); + } + + @Override + public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName) { + return deleteTopicPoliciesAsync(topicName, false); + } + + @Override + public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName, + boolean keepGlobalPoliciesAfterDeleting) { + TopicName partitionedTopicName = normalizeTopicName(topicName); + if (NamespaceService.isHeartbeatNamespace(partitionedTopicName.getNamespaceObject())) { + return CompletableFuture.completedFuture(null); + } + if (closed.get()) { + return CompletableFuture.failedFuture(new BrokerServiceException(getClass().getName() + " is closed.")); + } + CompletableFuture deleteLocal = + deleteIfExists(localPoliciesCache, pathFor(partitionedTopicName, false)); + if (keepGlobalPoliciesAfterDeleting) { + return deleteLocal; + } + CompletableFuture deleteGlobal = + deleteIfExists(globalPoliciesCache, pathFor(partitionedTopicName, true)); + return CompletableFuture.allOf(deleteLocal, deleteGlobal); + } + + @Override + public CompletableFuture updateTopicPoliciesAsync(TopicName topicName, boolean isGlobalPolicy, + boolean skipUpdateWhenTopicPolicyDoesntExist, + Consumer policyUpdater) { + TopicName partitionedTopicName = normalizeTopicName(topicName); + if (NamespaceService.isHeartbeatNamespace(partitionedTopicName.getNamespaceObject())) { + return CompletableFuture.failedFuture(new BrokerServiceException.NotAllowedException( + "Not allowed to update topic policy for the heartbeat topic")); + } + if (closed.get()) { + return CompletableFuture.failedFuture(new BrokerServiceException(getClass().getName() + " is closed.")); + } + MetadataCache cache = cache(isGlobalPolicy); + String path = pathFor(partitionedTopicName, isGlobalPolicy); + CompletableFuture updateFuture; + if (skipUpdateWhenTopicPolicyDoesntExist) { + updateFuture = cache.readModifyUpdate(path, + current -> updatePolicies(Optional.of(current), isGlobalPolicy, policyUpdater)); + } else { + updateFuture = cache.readModifyUpdateOrCreate(path, + current -> updatePolicies(current, isGlobalPolicy, policyUpdater)); + } + return updateFuture.thenAccept(__ -> { }).exceptionally(error -> { + if (skipUpdateWhenTopicPolicyDoesntExist + && FutureUtil.unwrapCompletionException(error) instanceof NotFoundException) { + return null; + } + throw FutureUtil.wrapToCompletionException(error); + }); + } + + @Override + public CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type) { + TopicName partitionedTopicName = normalizeTopicName(topicName); + if (NamespaceService.isHeartbeatNamespace(partitionedTopicName.getNamespaceObject())) { + return CompletableFuture.completedFuture(Optional.empty()); + } + if (closed.get()) { + return CompletableFuture.completedFuture(Optional.empty()); + } + boolean global = type == GetType.GLOBAL_ONLY; + return cache(global).get(pathFor(partitionedTopicName, global)) + .thenApply(policies -> policies.map(policy -> cloneWithScope(policy, global))); + } + + @Override + public boolean registerListener(TopicName topicName, TopicPolicyListener listener) { + listeners.compute(normalizeTopicName(topicName), (__, topicListeners) -> { + if (topicListeners == null) { + topicListeners = new CopyOnWriteArrayList<>(); + } + topicListeners.add(listener); + return topicListeners; + }); + return true; + } + + @Override + public void unregisterListener(TopicName topicName, TopicPolicyListener listener) { + listeners.computeIfPresent(normalizeTopicName(topicName), (__, topicListeners) -> { + topicListeners.remove(listener); + return topicListeners.isEmpty() ? null : topicListeners; + }); + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + listeners.clear(); + if (localPoliciesCache != null) { + localPoliciesCache.invalidateAll(); + } + if (globalPoliciesCache != null) { + globalPoliciesCache.invalidateAll(); + } + } + } + + private MetadataCache cache(boolean isGlobalPolicy) { + return isGlobalPolicy ? globalPoliciesCache : localPoliciesCache; + } + + private CompletableFuture deleteIfExists(MetadataCache cache, String path) { + return cache.delete(path).handle((__, error) -> { + cache.invalidate(path); + if (error == null || FutureUtil.unwrapCompletionException(error) instanceof NotFoundException) { + return null; + } + throw FutureUtil.wrapToCompletionException(error); + }); + } + + private static TopicPolicies updatePolicies(Optional currentPolicies, + boolean isGlobalPolicy, + Consumer policyUpdater) { + TopicPolicies policies = currentPolicies.map(TopicPolicies::clone).orElseGet(TopicPolicies::new); + policies.setIsGlobal(isGlobalPolicy); + policyUpdater.accept(policies); + return policies; + } + + private void handleNotification(Notification notification, boolean isGlobalPolicy) { + if (closed.get() + || (notification.getType() != NotificationType.Created + && notification.getType() != NotificationType.Modified + && notification.getType() != NotificationType.Deleted)) { + return; + } + String path = notification.getPath(); + String root = isGlobalPolicy ? GLOBAL_POLICIES_ROOT : LOCAL_POLICIES_ROOT; + Optional topicName = topicNameFromPath(root, path); + if (topicName.isEmpty()) { + return; + } + MetadataCache cache = cache(isGlobalPolicy); + cache.invalidate(path); + if (notification.getType() == NotificationType.Deleted) { + notifyListeners(topicName.get(), null); + return; + } + cache.get(path).whenComplete((policies, error) -> { + if (error != null) { + log.warn("[{}] Failed to refresh topic policies after metadata notification", path, error); + return; + } + notifyListeners(topicName.get(), + policies.map(policy -> cloneWithScope(policy, isGlobalPolicy)).orElse(null)); + }); + } + + private void notifyListeners(TopicName topicName, @Nullable TopicPolicies policies) { + List topicListeners = listeners.get(topicName); + if (topicListeners == null) { + return; + } + for (TopicPolicyListener listener : topicListeners) { + try { + listener.onUpdate(policies == null ? null : policies.clone()); + } catch (Throwable error) { + log.error("[{}] Call topic policy listener error", topicName, error); + } + } + } + + private static TopicName normalizeTopicName(TopicName topicName) { + return TopicName.get(topicName.getPartitionedTopicName()); + } + + private static TopicPolicies cloneWithScope(TopicPolicies policies, boolean isGlobalPolicy) { + TopicPolicies cloned = policies.clone(); + cloned.setIsGlobal(isGlobalPolicy); + return cloned; + } + + @VisibleForTesting + public CompletableFuture> getTopicPoliciesDirectFromStore(TopicName topicName, + boolean isGlobal) { + String path = pathFor(topicName, isGlobal); + MetadataCache c = cache(isGlobal); + c.invalidate(path); + return c.get(path).thenApply(opt -> opt.map(p -> cloneWithScope(p, isGlobal))); + } + + @VisibleForTesting + static String pathFor(TopicName topicName, boolean isGlobalPolicy) { + TopicName partitionedTopicName = normalizeTopicName(topicName); + return (isGlobalPolicy ? GLOBAL_POLICIES_ROOT : LOCAL_POLICIES_ROOT) + + "/" + partitionedTopicName.getTenant() + + "/" + partitionedTopicName.getNamespacePortion() + + "/" + partitionedTopicName.getDomain() + + "/" + partitionedTopicName.getEncodedLocalName(); + } + + @VisibleForTesting + private static Optional topicNameFromPath(String root, String path) { + if (!path.startsWith(root + "/")) { + return Optional.empty(); + } + String[] parts = path.substring(root.length() + 1).split("/", 4); + if (parts.length != 4) { + return Optional.empty(); + } + return Optional.of(TopicName.get(parts[2], parts[0], parts[1], Codec.decode(parts[3]))); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index ad18af308e2c4..f8a5d59366104 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -647,7 +647,7 @@ protected CompletableFuture> createSystemT return systemTopicClient.newReaderAsync(); } - private void removeOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { + void removeOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { NamespaceName namespace = namespaceBundle.getNamespaceObject(); if (NamespaceService.isHeartbeatNamespace(namespace)) { return; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index 239c1d3d9bad4..7f49686a17fcf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -100,6 +100,15 @@ default void start(PulsarService pulsar) { default void close() throws Exception { } + + /** + * @implNote This method is never called unless by the default implementation of + * {@link TopicPoliciesService#registerListenerAsync(TopicName, TopicPolicyListener)}, which is actually called + * internally. This method is only retained for backward compatibility on custom implementations. + */ + @Deprecated + boolean registerListener(TopicName topicName, TopicPolicyListener listener); + /** * Registers a listener for topic policies updates. * @@ -110,10 +119,10 @@ default void close() throws Exception { * guaranteed to be received by the listener. * In summary, the listener is guaranteed to receive only the latest value. *

- * - * @return true if the listener is registered successfully */ - boolean registerListener(TopicName topicName, TopicPolicyListener listener); + default CompletableFuture registerListenerAsync(TopicName topicName, TopicPolicyListener listener) { + return CompletableFuture.completedFuture(registerListener(topicName, listener)); + } /** * Unregister the topic policies listener. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index e7a006185af77..89e1a47de30c1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -4726,7 +4726,10 @@ private void updateSubscriptionsDispatcherRateLimiter() { protected CompletableFuture initTopicPolicy() { final var topicPoliciesService = brokerService.pulsar().getTopicPoliciesService(); final var partitionedTopicName = TopicName.getPartitionedTopicName(topic); - if (topicPoliciesService.registerListener(partitionedTopicName, this)) { + return topicPoliciesService.registerListenerAsync(partitionedTopicName, this).thenCompose(registered -> { + if (!registered) { + return CompletableFuture.completedFuture(null); + } if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { return CompletableFuture.completedFuture(null); } @@ -4738,8 +4741,7 @@ protected CompletableFuture initTopicPolicy() { TopicPoliciesService.GetType.LOCAL_ONLY)) .thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), brokerService.getTopicOrderedExecutor()); - } - return CompletableFuture.completedFuture(null); + }); } @VisibleForTesting diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java new file mode 100644 index 0000000000000..e7fefa164973a --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.admin; + +import org.apache.pulsar.broker.service.MetadataStoreTopicPoliciesService; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker-admin") +public class MetadataStoreTopicPoliciesTest extends TopicPoliciesTest { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + conf.setTopicPoliciesServiceClassName(MetadataStoreTopicPoliciesService.class.getName()); + super.setup(); + } + + @Override + protected void clearTopicPoliciesCache() { + } + + @Test(enabled = false) + @Override + public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Exception { + // This test is specific to SystemTopicBasedTopicPoliciesService (uses getPoliciesCacheInit). + // Not applicable to MetadataStoreTopicPoliciesService. + } + + @Test(enabled = false) + @Override + public void testSystemTopicShouldBeCompacted() throws Exception { + // Relies on __change_events system topic, which does not exist with MetadataStoreTopicPoliciesService. + } + + @Test(enabled = false) + @Override + public void testPoliciesCanBeDeletedWithTopic() throws Exception { + // Directly accesses __change_events PersistentTopic for compaction. + // Not applicable to MetadataStoreTopicPoliciesService. + } + + @Test(enabled = false) + @Override + public void testProduceChangesWithEncryptionRequired() throws Exception { + // Checks __change_events LAC, which does not exist with MetadataStoreTopicPoliciesService. + } + + @Test(enabled = false) + @Override + public void testTopicPoliciesAfterCompaction(String reloadPolicyType) throws Exception { + // The "Recreate_Service" variant creates a new SystemTopicBasedTopicPoliciesService, + // which is not applicable to MetadataStoreTopicPoliciesService. + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index 47b0d63aba635..099b6ed89211d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -119,7 +119,9 @@ import org.glassfish.jersey.client.JerseyClientBuilder; import org.mockito.Mockito; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -144,10 +146,11 @@ public class TopicPoliciesTest extends MockedPulsarServiceBaseTest { private final int testTopicPartitions = 2; - @BeforeMethod + @BeforeClass(alwaysRun = true) @Override protected void setup() throws Exception { this.conf.setDefaultNumberOfNamespaceBundles(1); + this.conf.setForceDeleteNamespaceAllowed(true); super.internalSetup(); admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); @@ -158,15 +161,48 @@ protected void setup() throws Exception { admin.topics().createPartitionedTopic(testTopic, testTopicPartitions); Producer producer = pulsarClient.newProducer().topic(testTopic).create(); producer.close(); - waitForZooKeeperWatchers(); } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) @Override public void cleanup() throws Exception { super.internalCleanup(); } + @BeforeMethod + void setupTestTopic() throws Exception { + // Recreate namespace to clear any policies set by previous tests + try { + admin.topics().deletePartitionedTopic(testTopic, true); + } catch (PulsarAdminException.NotFoundException e) { + // topic may already be deleted + } + try { + admin.namespaces().deleteNamespace(myNamespace, true); + } catch (PulsarAdminException.NotFoundException e) { + // namespace may already be deleted + } + try { + admin.namespaces().deleteNamespace(myNamespaceV1, true); + } catch (PulsarAdminException.NotFoundException e) { + // namespace may already be deleted + } + admin.namespaces().createNamespace(testTenant + "/" + testNamespace, Set.of("test")); + admin.namespaces().createNamespace(myNamespaceV1); + admin.topics().createPartitionedTopic(testTopic, testTopicPartitions); + // Acquire namespace bundle ownership so tests that call getOrCreateTopic() directly succeed. + // Without this, services that don't create a __change_events reader (e.g. MetadataStoreTopicPoliciesService) + // leave the bundle unowned after namespace recreation and the first broker-side topic load fails. + admin.lookups().lookupTopic(testTopic + "-partition-0"); + } + + @AfterMethod(alwaysRun = true) + void afterMethodCleanup() throws Exception{ + admin.brokers().updateDynamicConfiguration("maxPublishRatePerTopicInMessages", "0"); + admin.brokers().updateDynamicConfiguration("maxPublishRatePerTopicInBytes", "0"); + clearTopicPoliciesCache(); + } + @Test public void updatePropertiesForAutoCreatedTopicTest() throws Exception { TopicName topicName = TopicName.get( @@ -485,8 +521,8 @@ public Object[][] clientRequestType() { @Test(dataProvider = "clientRequestType") public void testPriorityOfGlobalPolicies(String clientRequestType) throws Exception { - final SystemTopicBasedTopicPoliciesService topicPoliciesService = - (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final TopicPoliciesService topicPoliciesService = + pulsar.getTopicPoliciesService(); final JerseyClient httpClient = JerseyClientBuilder.createClient(); // create topic and load it up. final String namespace = myNamespace; @@ -566,8 +602,8 @@ public void testPriorityOfGlobalPolicies(String clientRequestType) throws Except @Test(dataProvider = "clientRequestType") public void testPriorityOfGlobalPolicies2(String clientRequestType) throws Exception { - final SystemTopicBasedTopicPoliciesService topicPoliciesService = - (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final TopicPoliciesService topicPoliciesService = + pulsar.getTopicPoliciesService(); final JerseyClient httpClient = JerseyClientBuilder.createClient(); // create topic and load it up. final String namespace = myNamespace; @@ -653,8 +689,8 @@ public void testGlobalPolicyStillAffectsAfterUnloading() throws Exception { final TopicName topicName = TopicName.get(topic); admin.topics().createNonPartitionedTopic(topic); pulsarClient.newProducer().topic(topic).create().close(); - final SystemTopicBasedTopicPoliciesService topicPoliciesService = - (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final TopicPoliciesService topicPoliciesService = + pulsar.getTopicPoliciesService(); // Set non-global policy of the limitation of max consumers. // Set global policy of the limitation of max producers. @@ -695,8 +731,8 @@ public void testRetentionGlobalPolicyAffects() throws Exception { final TopicName topicName = TopicName.get(topic); admin.topics().createNonPartitionedTopic(topic); pulsarClient.newProducer().topic(topic).create().close(); - final SystemTopicBasedTopicPoliciesService topicPoliciesService = - (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final TopicPoliciesService topicPoliciesService = + pulsar.getTopicPoliciesService(); // Set non-global policy of the limitation of max consumers. // Set global policy of the persistence policies. @@ -2596,10 +2632,8 @@ public void testRemoveSubscribeRate() throws Exception { @Test public void testPublishRateInDifferentLevelPolicy() throws Exception { - cleanup(); - conf.setMaxPublishRatePerTopicInMessages(5); - conf.setMaxPublishRatePerTopicInBytes(50L); - setup(); + admin.brokers().updateDynamicConfiguration("maxPublishRatePerTopicInMessages", "5"); + admin.brokers().updateDynamicConfiguration("maxPublishRatePerTopicInBytes", "50"); final String topicName = "persistent://" + myNamespace + "/test-" + UUID.randomUUID(); pulsarClient.newProducer().topic(topicName).create().close(); @@ -2890,9 +2924,7 @@ public void testMaxSubscriptionsPerTopicWithExistingSubs() throws Exception { @Test public void testMaxUnackedMessagesOnSubscriptionPriority() throws Exception { - cleanup(); - conf.setMaxUnackedMessagesPerSubscription(30); - setup(); + restartBroker(conf -> conf.setMaxUnackedMessagesPerSubscription(30)); final String topic = "persistent://" + myNamespace + "/test-" + UUID.randomUUID(); // init cache @Cleanup @@ -2955,6 +2987,9 @@ public void testMaxUnackedMessagesOnSubscriptionPriority() throws Exception { && admin.topicPolicies().getMaxUnackedMessagesOnSubscription(topic) == null); messages = getMsgReceived(consumer1, Integer.MAX_VALUE); assertEquals(messages.size(), defaultMaxUnackedMsgOnBroker); + + // restore default config + restartBroker(conf -> conf.setMaxUnackedMessagesPerSubscription(4 * 50000)); } private void produceMsg(Producer producer, int msgNum) throws Exception{ @@ -3139,14 +3174,16 @@ public void testGetReplicatorRateApplied() throws Exception { @Test(timeOut = 30000) public void testAutoCreationDisabled() throws Exception { - cleanup(); - conf.setAllowAutoTopicCreation(false); - setup(); + admin.brokers().updateDynamicConfiguration("allowAutoTopicCreation", "false"); + final String topic = testTopic + UUID.randomUUID(); admin.topics().createPartitionedTopic(topic, 3); pulsarClient.newProducer().topic(topic).create().close(); //should not fail assertNull(admin.topicPolicies().getMessageTTL(topic)); + + // restore default + admin.brokers().updateDynamicConfiguration("allowAutoTopicCreation", "true"); } @Test @@ -3270,6 +3307,12 @@ public void testSubscriptionTypesEnabled() throws Exception { pulsarClient.newConsumer().topic(topic) .subscriptionType(SubscriptionType.Shared).subscriptionName("test") .subscribe().close(); + + // restore dynamic broker config and conf object + pulsar.getConfiguration().setSubscriptionTypesEnabled( + Set.of("Exclusive", "Shared", "Failover", "Key_Shared")); + admin.brokers().updateDynamicConfiguration("subscriptionTypesEnabled", + "Exclusive,Shared,Failover,Key_Shared"); } @Test(timeOut = 20000) @@ -3600,7 +3643,8 @@ public void testPolicyIsDeleteTogetherAutomatically() throws Exception { } @Test - public void testDoNotCreateSystemTopicForHeartbeatNamespace() { + public void testDoNotCreateSystemTopicForHeartbeatNamespace() throws Exception { + initEventsTopicAndPartitions(); assertTrue(pulsar.getBrokerService().getTopics().size() > 0); pulsar.getBrokerService().getTopics().forEach((k, v) -> { TopicName topicName = TopicName.get(k); @@ -3661,8 +3705,13 @@ public void testLoopCreateAndDeleteTopicPolicies() throws Exception { } private void triggerAndWaitNewTopicCompaction(String topicName) throws Exception { - PersistentTopic tp = - (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + Optional topicOpt = + pulsar.getBrokerService().getTopic(topicName, false).join(); + if (topicOpt.isEmpty()) { + // Topic doesn't exist (e.g., when not using system-topic-based policies service), nothing to compact. + return; + } + PersistentTopic tp = (PersistentTopic) topicOpt.get(); // Wait for the old task finish. Awaitility.await().untilAsserted(() -> { CompletableFuture compactionTask = WhiteboxImpl.getInternalState(tp, "currentCompaction"); @@ -3681,7 +3730,7 @@ private void triggerAndWaitNewTopicCompaction(String topicName) throws Exception * It is not a thread safety method, something will go to a wrong pointer if there is a task is trying to load a * topic policies. */ - private void clearTopicPoliciesCache() { + protected void clearTopicPoliciesCache() { TopicPoliciesService topicPoliciesService = pulsar.getTopicPoliciesService(); if (topicPoliciesService instanceof TopicPoliciesService.TopicPoliciesServiceDisabled) { return; @@ -3911,8 +3960,8 @@ public void testGlobalTopicPolicies() throws Exception { .isNull()); admin.topicPolicies(true).setRetention(topic, new RetentionPolicies(1, 2)); - SystemTopicBasedTopicPoliciesService topicPoliciesService = - (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + TopicPoliciesService topicPoliciesService = + pulsar.getTopicPoliciesService(); // check global topic policies can be added correctly. Awaitility.await().untilAsserted(() -> assertNotNull( @@ -3956,6 +4005,7 @@ public void testGlobalTopicPolicies() throws Exception { @Test public void testMaxMessageSizeWithChunking() throws Exception { + final var maxMessageSize = this.conf.getMaxMessageSize(); this.conf.setMaxMessageSize(1000); @Cleanup @@ -3984,6 +4034,7 @@ public void testMaxMessageSizeWithChunking() throws Exception { // chunk message send success producer.send(new byte[2000]); + this.conf.setMaxMessageSize(maxMessageSize); } @Test(timeOut = 30000) @@ -4037,6 +4088,7 @@ public void testGetTopicPoliciesWhenDeleteTopicPolicy() throws Exception { @Test public void testProduceChangesWithEncryptionRequired() throws Exception { + initEventsTopicAndPartitions(); final String beforeLac = admin.topics().getInternalStats(topicPolicyEventsTopic).lastConfirmedEntry; admin.namespaces().setEncryptionRequiredStatus(myNamespace, true); // just an update to trigger writes on __change_events @@ -4493,4 +4545,10 @@ public void testGetAppliedOffloadPoliciesWithLegacyNamespacePolicies() throws Ex assertEquals(offloadPolicies.getManagedLedgerOffloadThresholdInBytes(), (Long) (1024 * 1024 * 10L), "Should inherit offload threshold from legacy namespace policy"); } + + private void initEventsTopicAndPartitions() throws Exception { + try (Producer producer = pulsarClient.newProducer().topic(testTopic).create()) { + // No-op. Creating the producer initializes the events topic and partitions. + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesServiceTest.java new file mode 100644 index 0000000000000..47a7de0528de9 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesServiceTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.awaitility.Awaitility; +import org.awaitility.core.ThrowingRunnable; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Test order: testUpgrade() -> other tests (with MetadataStoreTopicPoliciesService configured) -> testDowngrade(). + */ +@Test(groups = "broker") +public class LegacyAwareTopicPoliciesServiceTest extends MockedPulsarServiceBaseTest { + + private static final String metaNamespace = "public/meta-ns"; + + @BeforeClass + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.setupDefaultTenantAndNamespace(); + } + + @AfterClass + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test(priority = -1) + public void testUpgrade() throws Exception { + final var topic = "test-upgrade"; + admin.topics().createNonPartitionedTopic(topic); + admin.topicPolicies().setCompactionThreshold(topic, 100); + waitUntilAssert(() -> assertEquals(admin.topicPolicies().getCompactionThreshold(topic), 100)); + + restartBroker(conf -> { + conf.setSystemTopicEnabled(false); + conf.setTopicPoliciesServiceClassName(MetadataStoreTopicPoliciesService.class.getName()); + }); + // The policies will be lost because when system topic is disabled, it will not try to read policies from the + // __change_events topic + assertNull(admin.topicPolicies().getCompactionThreshold(topic)); + + restartBroker(conf -> conf.setSystemTopicEnabled(true)); + // The default namespace still read policies from the __change_events topic if it exists + assertEquals(admin.topicPolicies().getCompactionThreshold(topic), 100); + assertFalse(pulsar.getLocalMetadataStore().exists(MetadataStoreTopicPoliciesService.LOCAL_POLICIES_ROOT).get()); + + // The global policies are still stored in the __change_events topic + admin.topicPolicies(true).setCompactionThreshold(topic, 200); + waitUntilAssert(() -> assertEquals(admin.topicPolicies(true).getCompactionThreshold(topic), 200)); + assertFalse(pulsar.getConfigurationMetadataStore() + .exists(MetadataStoreTopicPoliciesService.GLOBAL_POLICIES_ROOT).get()); + + admin.topicPolicies().deleteTopicPolicies(topic); + waitUntilAssert(() -> assertNull(admin.topicPolicies().getCompactionThreshold(topic))); + + admin.namespaces().createNamespace(metaNamespace); + } + + @Test(priority = 1) + public void testDowngrade() throws Exception { + final var topic1 = "downgrade"; // in default namespace + admin.topics().createNonPartitionedTopic(topic1); + admin.topicPolicies().setCompactionThreshold(topic1, 1); + waitUntilAssert(() -> assertEquals(admin.topicPolicies().getCompactionThreshold(topic1), 1)); + + final var topic2 = metaNamespace + "/downgrade"; + admin.topics().createNonPartitionedTopic(topic2); + admin.topicPolicies().setCompactionThreshold(topic2, 2); + waitUntilAssert(() -> assertEquals(admin.topicPolicies().getCompactionThreshold(topic2), 2)); + + restartBroker(conf -> + conf.setTopicPoliciesServiceClassName(SystemTopicBasedTopicPoliciesService.class.getName())); + assertEquals(admin.topicPolicies().getCompactionThreshold(topic1), 1); + // The policies will be lost because they are not stored in the __change_events topic + assertNull(admin.topicPolicies().getCompactionThreshold(topic2)); + } + + @DataProvider + public Object[][] namespaces() { + return new Object[][] { + { "public/default" }, + { metaNamespace } + }; + } + + @Test(dataProvider = "namespaces") + public void testPoliciesOperations(String namespace) throws Exception { + final var topicName = TopicName.get(namespace + "/test-policies-operations"); + final var topic = topicName.toString(); + admin.topics().createNonPartitionedTopic(topic); + + final var compactionThreshold = new AtomicLong(0); + // Verify the exception thrown from one listener does not affect other listeners + pulsar.getTopicPoliciesService().registerListenerAsync(topicName, __ -> { + throw new RuntimeException("injected failure"); + }).get(); + pulsar.getTopicPoliciesService().registerListenerAsync(topicName, policies -> + Optional.ofNullable(policies).map(TopicPolicies::getCompactionThreshold).ifPresentOrElse( + compactionThreshold::set, () -> compactionThreshold.set(-1))).get(); + + // Verify Created events are handled + admin.topicPolicies(false).setCompactionThreshold(topic, 100); + waitUntilAssert(() -> assertEquals(compactionThreshold.get(), 100)); + final var localStore = pulsar.getLocalMetadataStore(); + final var configurationStore = pulsar.getConfigurationMetadataStore(); + + if (namespace.equals(metaNamespace)) { + assertTrue(localStore.exists(MetadataStoreTopicPoliciesService.pathFor(topicName, false)).get()); + assertFalse(configurationStore.exists(MetadataStoreTopicPoliciesService.pathFor(topicName, true)).get()); + } + + admin.topicPolicies(true).setCompactionThreshold(topic, 200); + waitUntilAssert(() -> assertEquals(compactionThreshold.get(), 200)); + if (namespace.equals(metaNamespace)) { + assertTrue(configurationStore.exists(MetadataStoreTopicPoliciesService.pathFor(topicName, true)).get()); + } + + // Verify Modified events are handled + admin.topicPolicies(false).setCompactionThreshold(topic, 300); + waitUntilAssert(() -> assertEquals(compactionThreshold.get(), 300)); + + admin.topicPolicies(true).setCompactionThreshold(topic, 400); + waitUntilAssert(() -> assertEquals(compactionThreshold.get(), 400)); + + final var readerNamespaces = ((LegacyAwareTopicPoliciesService) pulsar.getTopicPoliciesService()) + .systemTopicService.getReaderCaches().keySet(); + assertFalse(readerNamespaces.contains(NamespaceName.get(metaNamespace))); + + // Verify Deleted events are handled + admin.topicPolicies(false).deleteTopicPolicies(topic); + waitUntilAssert(() -> assertEquals(compactionThreshold.get(), -1)); + if (namespace.equals(metaNamespace)) { + assertFalse(localStore.exists(MetadataStoreTopicPoliciesService.pathFor(topicName, false)).get()); + assertFalse(configurationStore.exists(MetadataStoreTopicPoliciesService.pathFor(topicName, true)).get()); + } + } + + @Test + public void testUserCreatedEventsTopicAreIgnored() throws Exception { + final var topic = TopicName.get(metaNamespace + "/" + System.currentTimeMillis()).toString(); + admin.topics().createNonPartitionedTopic(topic); + admin.topicPolicies().setCompactionThreshold(topic, 1); + waitUntilAssert(() -> assertEquals(admin.topicPolicies().getCompactionThreshold(topic), 1)); + + final var eventsTopic = metaNamespace + "/" + SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME; + admin.topics().createNonPartitionedTopic(eventsTopic); + // Even if the __change_events topic is created, since it has detected the namespace didn't have the events + // topic before, it will be ignored and the policies are still read from metadata store. + waitUntilAssert(() -> assertEquals(admin.topicPolicies().getCompactionThreshold(topic), 1)); + admin.topics().delete(eventsTopic); + } + + private static void waitUntilAssert(ThrowingRunnable assertion) { + Awaitility.await().atMost(Duration.ofSeconds(1)).untilAsserted(assertion); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 6b050e8b421c6..a0fdf039294bd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -109,7 +109,7 @@ public void onUpdate(TopicPolicies data) { CompletableFuture f = CompletableFuture.completedFuture(null).thenRunAsync(() -> { for (int i = 0; i < 100; i++) { TopicPolicyListener listener = new TopicPolicyListenerImpl(); - systemTopicBasedTopicPoliciesService.registerListener(topicName, listener); + systemTopicBasedTopicPoliciesService.registerListenerAsync(topicName, listener); Assert.assertNotNull(systemTopicBasedTopicPoliciesService.listeners.get(topicName)); Assert.assertTrue(systemTopicBasedTopicPoliciesService.listeners.get(topicName).size() >= 1); systemTopicBasedTopicPoliciesService.unregisterListener(topicName, listener); @@ -118,7 +118,7 @@ public void onUpdate(TopicPolicies data) { for (int i = 0; i < 100; i++) { TopicPolicyListener listener = new TopicPolicyListenerImpl(); - systemTopicBasedTopicPoliciesService.registerListener(topicName, listener); + systemTopicBasedTopicPoliciesService.registerListenerAsync(topicName, listener); Assert.assertNotNull(systemTopicBasedTopicPoliciesService.listeners.get(topicName)); Assert.assertTrue(systemTopicBasedTopicPoliciesService.listeners.get(topicName).size() >= 1); systemTopicBasedTopicPoliciesService.unregisterListener(topicName, listener); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java index 6b9735d59b21a..7e9c697fb5d03 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java @@ -72,6 +72,13 @@ public static TopicPolicies getGlobalTopicPolicies(TopicPoliciesService topicPol public static Optional getTopicPoliciesBypassCache(TopicPoliciesService topicPoliciesService, TopicName topicName, boolean isGlobal) throws Exception { + if (topicPoliciesService instanceof LegacyAwareTopicPoliciesService legacyService) { + TopicPoliciesService resolved = legacyService.resolveService(topicName.getNamespaceObject()).get(); + return getTopicPoliciesBypassCache(resolved, topicName, isGlobal); + } + if (topicPoliciesService instanceof MetadataStoreTopicPoliciesService metadataStoreService) { + return metadataStoreService.getTopicPoliciesDirectFromStore(topicName, isGlobal).get(); + } @Cleanup final var reader = ((SystemTopicBasedTopicPoliciesService) topicPoliciesService) .getNamespaceEventsSystemTopicFactory() .createTopicPoliciesSystemTopicClient(topicName.getNamespaceObject()) From 0b8c00e13c02186f3e82dc3dc358508379795d1e Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 8 Jun 2026 18:23:33 +0800 Subject: [PATCH 134/201] [improve][meta] Upgrade Oxia client to 0.8.0 (#25964) --- distribution/server/src/assemble/LICENSE.bin.txt | 5 +++-- pom.xml | 5 +++-- .../java/org/apache/pulsar/metadata/MetadataStoreTest.java | 4 +--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 8bc5f5c7f4b8d..c9318c5f05d6a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -278,6 +278,7 @@ The Apache Software License, Version 2.0 - io.swagger-swagger-annotations-1.6.2.jar - io.swagger-swagger-core-1.6.2.jar - io.swagger-swagger-models-1.6.2.jar + * slog -- io.github.merlimat.slog-slog-0.9.5.jar * DataSketches - com.yahoo.datasketches-memory-0.8.3.jar - com.yahoo.datasketches-sketches-core-0.8.3.jar @@ -497,8 +498,8 @@ The Apache Software License, Version 2.0 * Prometheus - io.prometheus-simpleclient_httpserver-0.16.0.jar * Oxia - - io.github.oxia-db-oxia-client-api-0.7.4.jar - - io.github.oxia-db-oxia-client-0.7.4.jar + - io.github.oxia-db-oxia-client-api-0.8.0.jar + - io.github.oxia-db-oxia-client-0.8.0.jar * OpenHFT - net.openhft-zero-allocation-hashing-0.16.jar * Java JSON WebTokens diff --git a/pom.xml b/pom.xml index 1c634d08c7383..fad7d483e56ff 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,8 @@ flexible messaging model and an intuitive client API. 4.5.13 4.4.15 0.7.7 - 0.7.4 + 0.8.0 + 0.7.4 2.0 1.10.12 5.5.0 @@ -1306,7 +1307,7 @@ flexible messaging model and an intuitive client API. io.github.oxia-db oxia-testcontainers - ${oxia.version} + ${oxia-testcontainers.version} diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java index bae1eae71efcf..d88ac51d92b79 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java @@ -32,7 +32,6 @@ import static org.testng.Assert.fail; import io.oxia.client.ClientConfig; import io.oxia.client.api.AsyncOxiaClient; -import io.oxia.client.session.SessionFactory; import io.oxia.client.session.SessionManager; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -621,8 +620,7 @@ public void testOxiaLoadConfigFromFile() throws Exception { OxiaMetadataStore store = (OxiaMetadataStore) MetadataStoreFactory.create(oxia, config); var client = (AsyncOxiaClient) WhiteboxImpl.getInternalState(store, "client"); var sessionManager = (SessionManager) WhiteboxImpl.getInternalState(client, "sessionManager"); - var sessionFactory = (SessionFactory) WhiteboxImpl.getInternalState(sessionManager, "factory"); - var clientConfig = (ClientConfig) WhiteboxImpl.getInternalState(sessionFactory, "config"); + var clientConfig = (ClientConfig) WhiteboxImpl.getInternalState(sessionManager, "clientConfig"); var sessionTimeout = clientConfig.sessionTimeout(); assertEquals(sessionTimeout, Duration.ofSeconds(60)); } From e64fb18372c9bd0afeeea175d314fd75139a02ac Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 8 Jun 2026 21:53:07 +0300 Subject: [PATCH 135/201] Bump version to next snapshot version --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- jetcd-core-shaded/pom.xml | 2 +- jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml | 2 +- jetty-upgrade/pom.xml | 2 +- jetty-upgrade/zookeeper-prometheus-metrics/pom.xml | 2 +- jetty-upgrade/zookeeper-with-patched-admin/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- microbench/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-bom/pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-cli-utils/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-dependencies-minimized/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-docs-tools/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/azure-data-explorer/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis-kpl-shaded/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-opentelemetry/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 135 files changed, 138 insertions(+), 138 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 45eabb7e3e004..3cbedba9f5752 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.2 + 4.2.3-SNAPSHOT bouncy-castle-bc diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 1114969543a93..52f809a039e18 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.2 + 4.2.3-SNAPSHOT bcfips-include-test diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index fc9d055b5f2ff..52566cceae225 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.2 + 4.2.3-SNAPSHOT bouncy-castle-bcfips diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index b135c12cd8bff..ecd43412030ac 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.2 + 4.2.3-SNAPSHOT diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 7652d26aff9ff..9a591de7693bc 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 4.2.2 + 4.2.3-SNAPSHOT jar Pulsar Build Tools @@ -43,7 +43,7 @@ - 2026-06-05T15:47:49Z + 2026-06-08T18:53:03Z 1.8 1.8 8 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 0a6604701c29d..df2ed00fab9ac 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.2 + 4.2.3-SNAPSHOT pulsar-io-distribution diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index bc1c7dcff886c..200e54031b63d 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.2 + 4.2.3-SNAPSHOT pulsar-offloader-distribution diff --git a/distribution/pom.xml b/distribution/pom.xml index 3c1948acea233..ff4ff01eff5bc 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.2 + 4.2.3-SNAPSHOT distribution diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 128744a8ed97e..2703055d64c4c 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.2 + 4.2.3-SNAPSHOT pulsar-server-distribution diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 94a43e65297a1..c59a0a3ce88f8 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.2 + 4.2.3-SNAPSHOT pulsar-shell-distribution diff --git a/docker/pom.xml b/docker/pom.xml index 1b26a7b42cbb0..844626bdcd048 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.2 + 4.2.3-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index f3b25777f66f1..88717deda118e 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.2 + 4.2.3-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 1d15f1b3de1a0..6f8ad31612b20 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.2 + 4.2.3-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index cd52704753694..3a1ae930e5077 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.2 + 4.2.3-SNAPSHOT jclouds-shaded diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml index 407228ca19ef2..57e7474f14256 100644 --- a/jetcd-core-shaded/pom.xml +++ b/jetcd-core-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.2 + 4.2.3-SNAPSHOT jetcd-core-shaded diff --git a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml index f2221382d425e..b37620f76dcec 100644 --- a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml +++ b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.2 + 4.2.3-SNAPSHOT pulsar-bookkeeper-prometheus-metrics-provider Apache Pulsar :: BookKeeper Stats Providers :: Prometheus diff --git a/jetty-upgrade/pom.xml b/jetty-upgrade/pom.xml index 0d7618069d346..99e552adb514a 100644 --- a/jetty-upgrade/pom.xml +++ b/jetty-upgrade/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.2 + 4.2.3-SNAPSHOT jetty-upgrade diff --git a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml index 29a3009d9ab80..510a81240e202 100755 --- a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml +++ b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.2 + 4.2.3-SNAPSHOT pulsar-zookeeper-prometheus-metrics diff --git a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml index 83d6b819c9746..86a6dea63821b 100644 --- a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml +++ b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar jetty-upgrade - 4.2.2 + 4.2.3-SNAPSHOT zookeeper-with-patched-admin diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 15d1542c53e13..b538c38b7df10 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.2 + 4.2.3-SNAPSHOT managed-ledger diff --git a/microbench/pom.xml b/microbench/pom.xml index 637c038eb365d..76c1781a8ab31 100644 --- a/microbench/pom.xml +++ b/microbench/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.2 + 4.2.3-SNAPSHOT microbench diff --git a/pom.xml b/pom.xml index fad7d483e56ff..9dff11d137e5e 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 4.2.2 + 4.2.3-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -94,7 +94,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - 2026-06-05T15:47:43Z + 2026-06-08T18:52:58Z true 4.1.12.1 5.7.1 @@ -266,9 +266,9 @@ flexible messaging model and an intuitive client API. 2.0.3 2.15.0 3.19.0 - 2.21.0 - 1.20.0 - 1.3.5 + 2.22.0 + 1.22.0 + 1.3.6 2.1.6 2.1.9 3.1.0 @@ -294,8 +294,8 @@ flexible messaging model and an intuitive client API. 1.0 9.1.6 6.2.12 - 4.5.13 - 4.4.15 + 4.5.14 + 4.4.16 0.7.7 0.8.0 0.7.4 @@ -373,7 +373,7 @@ flexible messaging model and an intuitive client API. 3.33.0 9.37.4 1.11.0 - 2.15.0 + 2.15.1 2.1.10 1.10.3 3.0.2 From 75fbe6b02ab1578b72458f4547f84ff5a42d130d Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 10 Jun 2026 00:19:11 -0700 Subject: [PATCH 141/201] [fix][ml] Fix eviction trigger race that cleared the in-progress marker (#25988) (cherry picked from commit fbcfacf324ef02c7c6fe639056fc7aa779262767) --- .../mledger/impl/cache/RangeEntryCacheManagerImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java index 6d048e11389cd..bcadb78b71039 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java @@ -142,9 +142,9 @@ Optional> triggerEvictionWhenNeeded() { while (evictionCompletionFuture == null) { evictionCompletionFuture = evictionInProgress.get(); if (evictionCompletionFuture == null) { - evictionCompletionFuture = evictionInProgress.updateAndGet( - currentValue -> currentValue == null ? new CompletableFuture<>() : null); - if (evictionCompletionFuture != null) { + CompletableFuture newEvictionFuture = new CompletableFuture<>(); + if (evictionInProgress.compareAndSet(null, newEvictionFuture)) { + evictionCompletionFuture = newEvictionFuture; triggerEvictionToMakeSpace(evictionCompletionFuture); } } From f62a4598aeee56059bf0bfe7bffc86b6e15a9603 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 10 Jun 2026 05:30:52 -0700 Subject: [PATCH 142/201] [fix][ml] Reset messageMetadataInitialized when recycling RangeCacheEntryWrapper (#25987) (cherry picked from commit 98bac651d0535d5e92e2e49910785ff90c8a68da) --- .../bookkeeper/mledger/impl/cache/RangeCacheEntryWrapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheEntryWrapper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheEntryWrapper.java index d3dd9de038b76..02d148504baa6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheEntryWrapper.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeCacheEntryWrapper.java @@ -191,6 +191,7 @@ void recycle() { size = 0; timestampNanos = 0; requeueCount = 0; + messageMetadataInitialized = false; accessed = false; recyclerHandle.recycle(this); } From 08bb8269c891bc6112c0a2a8f5dce4cd8a380d6b Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 12 Jun 2026 11:34:33 +0800 Subject: [PATCH 143/201] [fix][broker] Fix compacted read could be stuck forever or message loss due to cursor mark delete (#25998) (cherry picked from commit 7a9fefb4c4a25b1aa2c0fb335d99aa7b00ee3c91) --- .../persistent/PersistentSubscription.java | 18 ++++ .../pulsar/compaction/CompactionTest.java | 85 +++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index ce7795a917adc..843342e8b749a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -57,6 +57,7 @@ import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.service.AbstractDispatcherSingleActiveConsumer; import org.apache.pulsar.broker.service.AbstractSubscription; import org.apache.pulsar.broker.service.AnalyzeBacklogResult; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -447,6 +448,23 @@ public void acknowledgeMessage(List positions, AckType ackType, Map ml.getFirstPosition().getLedgerId()) { + log.warn("Received an ACK whose position is " + position + ", valid ledgers: " + + ml.getLedgersInfo().keySet()); + } + return; + } + } cursor.asyncMarkDelete(position, mergeCursorProperties(properties), markDeleteCallback, previousMarkDeletePosition); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index e5ccec35e9ffa..e7cb6300c516c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -56,6 +56,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiConsumer; @@ -70,12 +71,15 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.intercept.MockBrokerInterceptor; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -87,6 +91,7 @@ import org.apache.pulsar.client.api.EncryptionKeyInfo; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; @@ -124,6 +129,7 @@ public class CompactionTest extends MockedPulsarServiceBaseTest { protected ScheduledExecutorService compactionScheduler; protected BookKeeper bk; private PublishingOrderCompactor compactor; + private volatile java.util.function.Consumer consumerCreated = __ -> {}; @Override protected void doInitConf() throws Exception { @@ -135,6 +141,14 @@ protected void doInitConf() throws Exception { @Override public void setup() throws Exception { super.internalSetup(); + pulsar.getBrokerService().setInterceptor(new MockBrokerInterceptor() { + + @Override + public void consumerCreated(ServerCnx cnx, org.apache.pulsar.broker.service.Consumer consumer, + Map metadata) { + consumerCreated.accept(consumer); + } + }); admin.clusters().createCluster(configClusterName, ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); @@ -164,6 +178,8 @@ public void beforeMethod() throws Exception { admin.namespaces().removeRetention("my-tenant/my-ns"); AbstractTwoPhaseCompactor.injectionAfterSeekInPhaseTwo = () -> {}; AbstractTwoPhaseCompactor.injectionPhaseTwoSeek = RawReader::seekAsync; + consumerCreated = __ -> {}; + pulsarTestContext.getMockBookKeeper().setDefaultReadEntriesDelayMillis(1); } protected long compact(String topic) throws ExecutionException, InterruptedException { @@ -2643,4 +2659,73 @@ private void triggerAndWaitCompaction(String topic) throws Exception { Awaitility.await().untilAsserted(() -> assertEquals( admin.topics().compactionStatus(topic).status, LongRunningProcessStatus.Status.SUCCESS)); } + + @Test + public void testReaderReadOnDeletedLedger() throws Exception { + final var topic = "persistent://my-tenant/my-ns/reader-read-on-deleted-ledger"; + try (final var producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create()) { + for (int i = 0; i < 3; i++) { + producer.newMessage().key("key-" + i).value("value-" + i).send(); + } + } + // Trigger the ledger rollover + var ml = (ManagedLedgerImpl) ((PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topic).get() + .orElseThrow()).getManagedLedger(); + ml.getConfig().setMaxEntriesPerLedger(1); + ml.getConfig().setMaxSizePerLedgerMb(0); + ml.getConfig().setMinimumRolloverTime(0, TimeUnit.MILLISECONDS); + ml.rollCurrentLedgerIfFull(); + Awaitility.await().untilAsserted(() -> assertEquals(ml.getLedgersInfo().size(), 2)); + + final var subName = "sub-" + System.currentTimeMillis(); + @Cleanup final var reader = pulsarClient.newReader(Schema.STRING).readCompacted(true).topic(topic) + .subscriptionName(subName) + .startMessageId(MessageId.earliest).create(); + + // Slow down the pre-fetching + pulsarTestContext.getMockBookKeeper().setDefaultReadEntriesDelayMillis(500); + + // Receive 1 message so that the startMessageId will be reset to ledger_id:0 after reconnection + assertTrue(reader.hasMessageAvailable()); + final var firstMsg = reader.readNext(3, TimeUnit.SECONDS); + assertNotNull(firstMsg); + + triggerAndWaitCompaction(topic); + + // Simulate the pending cumulative acknowledgment is flushed after the consumer is created + // We don't need such interception if we can support controlling the acknowledgment flush for reader. + final var firstTime = new AtomicBoolean(true); + consumerCreated = serverConsumer -> { + final var subscription = serverConsumer.getSubscription(); + if (subscription.getName().contains(subName) && firstTime.compareAndSet(true, false)) { + final var msgId = (MessageIdAdv) firstMsg.getMessageId(); + subscription.acknowledgeMessage(List.of(PositionFactory.create(msgId.getLedgerId(), + msgId.getEntryId())), CommandAck.AckType.Cumulative, Map.of()); + } + }; + + // Trigger the reconnection and trim the first ledger. + admin.namespaces().unload("my-tenant/my-ns"); + admin.lookups().lookupTopic(topic); + final var persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopic(topic, true).get() + .orElseThrow(); + final var trimFuture = new CompletableFuture(); + persistentTopic.getManagedLedger().trimConsumedLedgersInBackground(trimFuture); + trimFuture.get(); + assertEquals(persistentTopic.getManagedLedger().getLedgersInfo().size(), 1); + + pulsarTestContext.getMockBookKeeper().setDefaultReadEntriesDelayMillis(1); + + while (reader.hasMessageAvailable()) { + final var msg = reader.readNextAsync().get(3, TimeUnit.SECONDS); + log.info("read id={} key={} value={}", msg.getMessageId(), msg.getKey(), msg.getValue()); + } + + final var serverConsumer = persistentTopic.getSubscription(subName).getDispatcher().getConsumers().get(0); + assertEquals(((MessageIdAdv) serverConsumer.getStartMessageId()).getEntryId(), 0L); + + final var emptyLedgerId = persistentTopic.getManagedLedger().getLedgersInfo().lastEntry().getKey(); + assertEquals(persistentTopic.getTopicCompactionService().getLastCompactedPosition().get(), + PositionFactory.create(emptyLedgerId, -1L)); + } } From ab4aaaa3f6f1e5f7a506d86b6abbb7d76cc311bb Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 12 Jun 2026 08:40:43 +0300 Subject: [PATCH 144/201] [fix][meta] Keep the leader value in the election cycle and make leader reads authoritative (#26000) (cherry picked from commit c1f2a2b260728f00f0428db38ea5c57339163828) --- .../pulsar/broker/admin/impl/BrokersBase.java | 7 +- .../broker/admin/impl/NamespacesBase.java | 71 ++++---- .../loadbalance/LeaderElectionService.java | 10 + .../broker/namespace/NamespaceService.java | 131 +++++++------ .../LeaderElectionServiceTest.java | 4 + .../api/coordination/LeaderElection.java | 23 ++- .../coordination/impl/LeaderElectionImpl.java | 172 +++++++++++++----- .../replication/AutoRecoveryMainTest.java | 10 +- .../pulsar/metadata/LeaderElectionTest.java | 155 ++++++++++++++++ .../impl/LeaderElectionImplTest.java | 28 +++ 10 files changed, 464 insertions(+), 147 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index 47b78bda80695..d165335719c00 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -128,8 +128,11 @@ public void getActiveBrokers(@Suspended final AsyncResponse asyncResponse) throw public void getLeaderBroker(@Suspended final AsyncResponse asyncResponse) { validateBothSuperuserAndBrokerOperation(pulsar().getConfig().getClusterName(), pulsar().getBrokerId(), BrokerOperation.GET_LEADER_BROKER) - .thenAccept(__ -> { - LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader() + // The authoritative read: waits for an in-progress leader election to settle + // instead of returning 404 while a re-election is still in flight. + .thenCompose(__ -> pulsar().getLeaderElectionService().readCurrentLeader()) + .thenAccept(leader -> { + LeaderBroker leaderBroker = leader .orElseThrow(() -> new RestException(Status.NOT_FOUND, "Couldn't find leader broker")); BrokerInfo brokerInfo = BrokerInfo.builder() .serviceUrl(leaderBroker.getServiceUrl()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 7b5a4fbb00554..044dbdaeb68e1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -58,7 +58,6 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.admin.AdminResource; -import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; @@ -1352,42 +1351,44 @@ private CompletableFuture validateLeaderBrokerAsync() { if (this.isLeaderBroker()) { return CompletableFuture.completedFuture(null); } - Optional currentLeaderOpt = pulsar().getLeaderElectionService().getCurrentLeader(); - if (currentLeaderOpt.isEmpty()) { - String errorStr = "The current leader is empty."; - log.error(errorStr); - return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, errorStr)); - } - LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader().get(); - String leaderBrokerId = leaderBroker.getBrokerId(); - return pulsar().getNamespaceService() - .createLookupResult(leaderBrokerId, false, null) - .thenCompose(lookupResult -> { - String redirectUrl = isRequestHttps() ? lookupResult.getLookupData().getHttpUrlTls() - : lookupResult.getLookupData().getHttpUrl(); - if (redirectUrl == null) { - log.error("Redirected broker's service url is not configured"); - return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, - "Redirected broker's service url is not configured.")); - } + // The authoritative read: waits for an in-progress leader election to settle instead of + // failing the request while a re-election is still in flight. + return pulsar().getLeaderElectionService().readCurrentLeader().thenCompose(currentLeaderOpt -> { + if (currentLeaderOpt.isEmpty()) { + String errorStr = "The current leader is empty."; + log.error(errorStr); + return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, errorStr)); + } + String leaderBrokerId = currentLeaderOpt.get().getBrokerId(); + return pulsar().getNamespaceService() + .createLookupResult(leaderBrokerId, false, null) + .thenCompose(lookupResult -> { + String redirectUrl = isRequestHttps() ? lookupResult.getLookupData().getHttpUrlTls() + : lookupResult.getLookupData().getHttpUrl(); + if (redirectUrl == null) { + log.error("Redirected broker's service url is not configured"); + return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, + "Redirected broker's service url is not configured.")); + } - try { - URL url = new URL(redirectUrl); - URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(url.getHost()) - .port(url.getPort()) - .replaceQueryParam("authoritative", - false).build(); - // Redirect - if (log.isDebugEnabled()) { - log.debug("Redirecting the request call to leader - {}", redirect); + try { + URL url = new URL(redirectUrl); + URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(url.getHost()) + .port(url.getPort()) + .replaceQueryParam("authoritative", + false).build(); + // Redirect + if (log.isDebugEnabled()) { + log.debug("Redirecting the request call to leader - {}", redirect); + } + return FutureUtil.failedFuture(( + new WebApplicationException(Response.temporaryRedirect(redirect).build()))); + } catch (MalformedURLException exception) { + log.error("The redirect url is malformed - {}", redirectUrl); + return FutureUtil.failedFuture(new RestException(exception)); } - return FutureUtil.failedFuture(( - new WebApplicationException(Response.temporaryRedirect(redirect).build()))); - } catch (MalformedURLException exception) { - log.error("The redirect url is malformed - {}", redirectUrl); - return FutureUtil.failedFuture(new RestException(exception)); - } - }); + }); + }); } public CompletableFuture setNamespaceBundleAffinityAsync(String bundleRange, String destinationBroker) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java index 2e53b54e98f61..21f67bd6b3613 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java @@ -56,10 +56,20 @@ public void close() throws Exception { leaderElection.close(); } + /** + * Authoritative read of the current leader: if a leader election is in progress, the returned + * future completes once it settles (bounded by the default metadata operation timeout). Use + * this whenever a decision is made based on who the leader is. + */ public CompletableFuture> readCurrentLeader() { return leaderElection.getLeaderValue(); } + /** + * Non-blocking snapshot of the current leader; empty while a re-election is settling even + * though a leader may technically exist. Only suitable for best-effort uses such as logging — + * decision-making callers must use {@link #readCurrentLeader()}. + */ public Optional getCurrentLeader() { return leaderElection.getLeaderValueIfPresent(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 44988d8fd5c3b..8a14b16f37339 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -61,7 +61,6 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.ResourceUnit; @@ -561,7 +560,6 @@ public CompletableFuture getHeartbeatOrSLAMonitorBrokerId( private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture> lookupFuture, LookupOptions options) { - String candidateBroker; LeaderElectionService les = pulsar.getLeaderElectionService(); if (les == null) { LOG.warn("The leader election has not yet been completed! NamespaceBundle[{}]", bundle); @@ -570,67 +568,98 @@ private void searchForCandidateBroker(NamespaceBundle bundle, return; } - boolean authoritativeRedirect = les.isLeader(); + selectCandidateBroker(bundle, options, les) + .thenAcceptAsync(selection -> { + if (selection.isEmpty()) { + LOG.warn("Load manager didn't return any available broker. " + + "Returning empty result to lookup. NamespaceBundle[{}]", + bundle); + lookupFuture.complete(Optional.empty()); + return; + } + acquireOwnershipOrRedirect(bundle, options, selection.get(), lookupFuture); + }, pulsar.getExecutor()) + .exceptionally(e -> { + LOG.warn("Error when searching for candidate broker to acquire {}: {}", bundle, e.getMessage(), e); + lookupFuture.completeExceptionally(FutureUtil.unwrapCompletionException(e)); + return null; + }); + } - try { - // check if this is Heartbeat or SLAMonitor namespace - candidateBroker = getHeartbeatOrSLAMonitorBrokerId(bundle, cb -> - CompletableFuture.completedFuture(isBrokerActive(cb))) - .get(config.getMetadataStoreOperationTimeoutSeconds(), SECONDS); + /** The broker selected for a bundle assignment, and whether the redirect to it is authoritative. */ + private record CandidateBrokerSelection(String candidateBroker, boolean authoritativeRedirect) { } - if (candidateBroker == null) { - Optional currentLeader = pulsar.getLeaderElectionService().getCurrentLeader(); + private CompletableFuture> selectCandidateBroker( + NamespaceBundle bundle, LookupOptions options, LeaderElectionService les) { + boolean authoritativeRedirect = les.isLeader(); - if (options.isAuthoritative()) { - // leader broker already assigned the current broker as owner - candidateBroker = pulsar.getBrokerId(); - } else { + // check if this is Heartbeat or SLAMonitor namespace + return getHeartbeatOrSLAMonitorBrokerId(bundle, cb -> + CompletableFuture.completedFuture(isBrokerActive(cb))) + .thenComposeAsync(heartbeatOrSlaBroker -> { + if (heartbeatOrSlaBroker != null) { + return completedSelection(heartbeatOrSlaBroker, authoritativeRedirect); + } + if (options.isAuthoritative()) { + // leader broker already assigned the current broker as owner + return completedSelection(pulsar.getBrokerId(), authoritativeRedirect); + } LoadManager loadManager = this.loadManager.get(); - boolean makeLoadManagerDecisionOnThisBroker = !loadManager.isCentralized() || les.isLeader(); - if (!makeLoadManagerDecisionOnThisBroker) { - // If leader is not active, fallback to pick the least loaded from current broker loadmanager + if (!loadManager.isCentralized() || les.isLeader()) { + return selectLeastLoadedBroker(bundle); + } + // The load manager decision belongs to the leader: read the leader + // authoritatively (waits for an in-progress election to settle) instead of + // acting on a possibly-empty snapshot during a leadership handoff. + return les.readCurrentLeader().thenComposeAsync(currentLeader -> { boolean leaderBrokerActive = currentLeader.isPresent() && isBrokerActive(currentLeader.get().getBrokerId()); - if (!leaderBrokerActive) { - makeLoadManagerDecisionOnThisBroker = true; - if (currentLeader.isEmpty()) { - LOG.warn( - "The information about the current leader broker wasn't available. " - + "Handling load manager decisions in a decentralized way. " - + "NamespaceBundle[{}]", - bundle); - } else { - LOG.warn( - "The current leader broker {} isn't active. " - + "Handling load manager decisions in a decentralized way. " - + "NamespaceBundle[{}]", - currentLeader.get(), bundle); - } + if (leaderBrokerActive) { + // forward to leader broker to make assignment + return completedSelection(currentLeader.get().getBrokerId(), authoritativeRedirect); } - } - if (makeLoadManagerDecisionOnThisBroker) { - Optional availableBroker = getLeastLoadedFromLoadManager(bundle); - if (availableBroker.isEmpty()) { - LOG.warn("Load manager didn't return any available broker. " - + "Returning empty result to lookup. NamespaceBundle[{}]", + // If leader is not active, fallback to pick the least loaded from current broker loadmanager + if (currentLeader.isEmpty()) { + LOG.warn( + "The information about the current leader broker wasn't available. " + + "Handling load manager decisions in a decentralized way. " + + "NamespaceBundle[{}]", bundle); - lookupFuture.complete(Optional.empty()); - return; + } else { + LOG.warn( + "The current leader broker {} isn't active. " + + "Handling load manager decisions in a decentralized way. " + + "NamespaceBundle[{}]", + currentLeader.get(), bundle); } - candidateBroker = availableBroker.get(); - authoritativeRedirect = true; - } else { - // forward to leader broker to make assignment - candidateBroker = currentLeader.get().getBrokerId(); - } - } - } + return selectLeastLoadedBroker(bundle); + }, pulsar.getExecutor()); + }, pulsar.getExecutor()); + } + + private static CompletableFuture> completedSelection( + String candidateBroker, boolean authoritativeRedirect) { + return CompletableFuture.completedFuture( + Optional.of(new CandidateBrokerSelection(candidateBroker, authoritativeRedirect))); + } + + // The decentralized decision is authoritative: this broker picked the owner itself. + private CompletableFuture> selectLeastLoadedBroker(NamespaceBundle bundle) { + Optional availableBroker; + try { + availableBroker = getLeastLoadedFromLoadManager(bundle); } catch (Exception e) { - LOG.warn("Error when searching for candidate broker to acquire {}: {}", bundle, e.getMessage(), e); - lookupFuture.completeExceptionally(e); - return; + return CompletableFuture.failedFuture(e); } + return CompletableFuture.completedFuture( + availableBroker.map(broker -> new CandidateBrokerSelection(broker, true))); + } + private void acquireOwnershipOrRedirect(NamespaceBundle bundle, LookupOptions options, + CandidateBrokerSelection selection, + CompletableFuture> lookupFuture) { + final String candidateBroker = selection.candidateBroker(); + final boolean authoritativeRedirect = selection.authoritativeRedirect(); try { Objects.requireNonNull(candidateBroker); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java index 3014c185cc802..a54a492144b99 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java @@ -21,6 +21,7 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import com.google.common.collect.Sets; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -113,6 +114,9 @@ public void anErrorShouldBeThrowBeforeLeaderElected() throws PulsarServerExcepti leaderBrokerReference.get() != null); Mockito.when(leaderElectionService.getCurrentLeader()) .thenAnswer(invocation -> Optional.ofNullable(leaderBrokerReference.get())); + Mockito.when(leaderElectionService.readCurrentLeader()) + .thenAnswer(invocation -> + CompletableFuture.completedFuture(Optional.ofNullable(leaderBrokerReference.get()))); leaderElectionServiceReference.set(leaderElectionService); // broker, webService and leaderElectionService is started, but elect not ready; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/LeaderElection.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/LeaderElection.java index 016b7d061d073..4b34815c558e6 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/LeaderElection.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/LeaderElection.java @@ -45,19 +45,30 @@ public interface LeaderElection extends AutoCloseable { LeaderElectionState getState(); /** - * Get the value set by the elected leader, or empty if there's currently no leader. + * Get the value set by the elected leader. + *

+ * This is the authoritative read: if a leader election is currently in progress (e.g. the + * previous leader's node was just deleted and the participants are re-electing), the returned + * future completes once the election has settled, with the newly determined leader value. The + * future completes exceptionally with a {@link java.util.concurrent.TimeoutException} if the + * election does not complete within the default metadata operation timeout. + *

+ * An instance that never participated in the election (no {@link #elect(Object)} call) reads + * the leader value directly from the metadata store. A closed instance does not wait: it + * reports an empty leader if it held the leadership when closed, or its last known view + * otherwise. * * @return a future that will track the completion of the operation */ CompletableFuture> getLeaderValue(); /** - * Get the value set by the elected leader, or empty if there's currently no leader. + * Get a non-blocking snapshot of the value set by the elected leader, or empty if no leader is + * known right now. *

- * The call is non blocking and in certain cases can return Optional.empty() even though a leader is - * technically elected. - * - * @return a future that will track the completion of the operation + * The snapshot can return Optional.empty() even though a leader is technically + * elected (for example while a re-election is still settling). Callers that need the + * authoritative leader must use {@link #getLeaderValue()} instead. */ Optional getLeaderValueIfPresent(); diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java index ab35eb7040c10..954bcfaf45a34 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java @@ -20,20 +20,18 @@ import com.fasterxml.jackson.databind.type.TypeFactory; import com.google.common.annotations.VisibleForTesting; +import java.time.Duration; import java.util.EnumSet; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.GetResult; -import org.apache.pulsar.metadata.api.MetadataCache; -import org.apache.pulsar.metadata.api.MetadataCacheConfig; import org.apache.pulsar.metadata.api.MetadataSerde; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.MetadataStoreException.AlreadyClosedException; @@ -52,14 +50,19 @@ class LeaderElectionImpl implements LeaderElection { private final String path; private final MetadataSerde serde; private final MetadataStoreExtended store; - private final MetadataCache cache; private final Consumer stateChangesListener; - private final ScheduledFuture updateCachedValueFuture; private LeaderElectionState leaderElectionState; private Optional version = Optional.empty(); private Optional proposedValue; + // The leader value as known by the election cycle (the leader can only change through an + // election cycle). Pending while no leader is known — election in progress or the leader node + // deleted — and completed with the leader value once the election settles. Readers of + // getLeaderValue() wait on it (bounded by leaderElectionCompletionTimeoutSeconds); + // getLeaderValueIfPresent() takes a non-blocking snapshot of it. + private CompletableFuture> currentLeaderFuture = new CompletableFuture<>(); + private final ScheduledExecutorService executor; private final FutureUtil.Sequencer sequencer; @@ -71,16 +74,21 @@ private enum InternalState { private static final int LEADER_ELECTION_RETRY_DELAY_SECONDS = 5; + // Upper bound for getLeaderValue() waiting on an election that never settles, aligned with the + // default metadata-store operation timeout (the broker's metadataStoreOperationTimeoutSeconds). + private volatile int leaderElectionCompletionTimeoutSeconds = 30; + + @VisibleForTesting + void setLeaderElectionCompletionTimeoutSeconds(int leaderElectionCompletionTimeoutSeconds) { + this.leaderElectionCompletionTimeoutSeconds = leaderElectionCompletionTimeoutSeconds; + } + LeaderElectionImpl(MetadataStoreExtended store, Class clazz, String path, Consumer stateChangesListener, ScheduledExecutorService executor) { this.path = path; this.serde = new JSONMetadataSerdeSimpleType<>(TypeFactory.defaultInstance().constructSimpleType(clazz, null)); this.store = store; - MetadataCacheConfig metadataCacheConfig = MetadataCacheConfig.builder() - .expireAfterWriteMillis(-1L) - .build(); - this.cache = store.getMetadataCache(clazz, metadataCacheConfig); this.leaderElectionState = LeaderElectionState.NoLeader; this.internalState = InternalState.Init; this.stateChangesListener = stateChangesListener; @@ -88,13 +96,38 @@ private enum InternalState { this.sequencer = FutureUtil.Sequencer.create(); store.registerListener(this::handlePathNotification); store.registerSessionListener(this::handleSessionNotification); - updateCachedValueFuture = executor.scheduleWithFixedDelay(this::getLeaderValue, - metadataCacheConfig.getRefreshAfterWriteMillis() / 2, - metadataCacheConfig.getRefreshAfterWriteMillis(), TimeUnit.MILLISECONDS); + } + + /** + * Record the leader value determined by the election cycle, waking up any getLeaderValue() + * callers waiting for the election to settle. + */ + private synchronized void leaderKnown(Optional leaderValue) { + if (currentLeaderFuture.isDone()) { + currentLeaderFuture = CompletableFuture.completedFuture(leaderValue); + } else { + currentLeaderFuture.complete(leaderValue); + } + } + + /** + * Mark the leader as unknown (the leader node was deleted) so getLeaderValue() callers wait for + * the next election cycle to settle instead of observing a stale value. + */ + private synchronized void leaderUnknown() { + if (currentLeaderFuture.isDone()) { + currentLeaderFuture = new CompletableFuture<>(); + } } @Override public synchronized CompletableFuture elect(T proposedValue) { + if (internalState == InternalState.Closed) { + // Reopened after close() (e.g. the broker's LeaderElectionService is close()d and then + // start()ed again): reset so a fresh election cycle runs and readers wait for it. + leaderElectionState = LeaderElectionState.NoLeader; + currentLeaderFuture = new CompletableFuture<>(); + } if (leaderElectionState != LeaderElectionState.NoLeader) { return CompletableFuture.completedFuture(leaderElectionState); } @@ -112,12 +145,6 @@ private synchronized CompletableFuture elect() { } else { return tryToBecomeLeader(); } - }).thenCompose(leaderElectionState -> { - // make sure that the cache contains the current leader - // so that getLeaderValueIfPresent works on all brokers - cache.refresh(path); - return cache.get(path) - .thenApply(__ -> leaderElectionState); }); } @@ -137,6 +164,7 @@ private synchronized CompletableFuture handleExistingLeader log.info("Keeping the existing value {} for {} as it's from the same session stat={}", existingValue, path, res.getStat()); // The value is still valid because it was created in the same session + leaderKnown(Optional.of(existingValue)); changeState(LeaderElectionState.Leading); return CompletableFuture.completedFuture(LeaderElectionState.Leading); } else { @@ -158,6 +186,7 @@ private synchronized CompletableFuture handleExistingLeader } // If the existing value is different, it means there's already another leader + leaderKnown(Optional.of(existingValue)); changeState(LeaderElectionState.Following); return CompletableFuture.completedFuture(LeaderElectionState.Following); } @@ -188,35 +217,18 @@ private synchronized CompletableFuture tryToBecomeLeader() .thenAccept(stat -> { synchronized (LeaderElectionImpl.this) { if (internalState == InternalState.ElectionInProgress) { - // Do a get() in order to force a notification later, if the z-node disappears - cache.get(path) - .thenRun(() -> { - synchronized (LeaderElectionImpl.this) { - log.info("Acquired leadership on {} with {}", path, value); - internalState = InternalState.LeaderIsPresent; - if (leaderElectionState != LeaderElectionState.Leading) { - leaderElectionState = LeaderElectionState.Leading; - try { - stateChangesListener.accept(leaderElectionState); - } catch (Throwable t) { - log.warn("Exception in state change listener", t); - } - } - result.complete(leaderElectionState); - } - }).exceptionally(ex -> { - // We fail to do the get(), so clean up the leader election fail the whole - // operation - log.warn("Failed to get the current state after acquiring leadership on {}. " - + " Conditionally deleting current entry.", path, ex); - store.delete(path, Optional.of(stat.getVersion())) - .thenRun(() -> result.completeExceptionally(ex)) - .exceptionally(ex2 -> { - result.completeExceptionally(ex2); - return null; - }); - return null; - }); + log.info("Acquired leadership on {} with {}", path, value); + internalState = InternalState.LeaderIsPresent; + leaderKnown(Optional.of(value)); + if (leaderElectionState != LeaderElectionState.Leading) { + leaderElectionState = LeaderElectionState.Leading; + try { + stateChangesListener.accept(leaderElectionState); + } catch (Throwable t) { + log.warn("Exception in state change listener", t); + } + } + result.complete(leaderElectionState); } else { log.info("Leadership on {} with value {} was lost. " + "Conditionally deleting entry with stat={}.", path, value, stat); @@ -254,7 +266,6 @@ private synchronized CompletableFuture tryToBecomeLeader() @Override public void close() throws Exception { - updateCachedValueFuture.cancel(true); try { asyncClose().join(); } catch (CompletionException e) { @@ -269,6 +280,12 @@ public synchronized CompletableFuture asyncClose() { } internalState = InternalState.Closed; + // A closed election reports "no leader" rather than waiting or failing: callers like the + // extensible load manager's handleNoChannelOwnerError() key off the resulting + // "no channel owner" condition to restart the election. + if (!currentLeaderFuture.isDone()) { + currentLeaderFuture.complete(Optional.empty()); + } if (leaderElectionState != LeaderElectionState.Leading) { return CompletableFuture.completedFuture(null); @@ -278,6 +295,9 @@ public synchronized CompletableFuture asyncClose() { .thenAccept(__ -> { synchronized (LeaderElectionImpl.this) { leaderElectionState = LeaderElectionState.NoLeader; + // The deleted leader node was ours and a closed instance no longer + // observes elections; don't keep reporting ourselves as leader. + currentLeaderFuture = CompletableFuture.completedFuture(Optional.empty()); } } ); @@ -290,12 +310,61 @@ public synchronized LeaderElectionState getState() { @Override public CompletableFuture> getLeaderValue() { - return cache.get(path); + CompletableFuture> future; + synchronized (this) { + if (internalState == InternalState.Init) { + // This instance never participated in the election (a pure observer, e.g. + // BookKeeper's MetadataDrivers helpers querying the current auditor): there is no + // local election cycle to wait for, so the store content is the authoritative + // answer. + return readLeaderValueFromStore(); + } + future = currentLeaderFuture; + } + if (future.isDone()) { + // Hand out a derived future so callers cannot complete the internal one. + return future.thenApply(value -> value); + } + int timeoutSeconds = leaderElectionCompletionTimeoutSeconds; + return FutureUtil.addTimeoutHandling(whenLeaderKnown(future), + Duration.ofSeconds(timeoutSeconds), executor, + () -> FutureUtil.createTimeoutException( + "Leader election on path " + path + " did not complete within " + + timeoutSeconds + " seconds", + LeaderElectionImpl.class, "getLeaderValue()")); + } + + private CompletableFuture> readLeaderValueFromStore() { + return store.get(path).thenApply(optRes -> optRes.map(res -> { + try { + return serde.deserialize(path, res.getValue(), res.getStat()); + } catch (Throwable t) { + throw new CompletionException(t); + } + })); + } + + // Track the internal future without exposing it: completing/cancelling the returned future + // (e.g. by the timeout handling) must not complete the election's own future. + private CompletableFuture> whenLeaderKnown(CompletableFuture> future) { + CompletableFuture> result = new CompletableFuture<>(); + future.whenComplete((value, ex) -> { + if (ex != null) { + result.completeExceptionally(ex); + } else { + result.complete(value); + } + }); + return result; } @Override public Optional getLeaderValueIfPresent() { - return cache.getIfCached(path); + CompletableFuture> future; + synchronized (this) { + future = currentLeaderFuture; + } + return future.isDone() && !future.isCompletedExceptionally() ? future.join() : Optional.empty(); } private void handleSessionNotification(SessionEvent event) { @@ -333,6 +402,9 @@ private void handlePathNotification(Notification notification) { } leaderElectionState = LeaderElectionState.NoLeader; + // The leader is unknown until the re-election below settles; getLeaderValue() + // callers wait for it instead of observing the stale value. + leaderUnknown(); if (proposedValue.isPresent()) { elect() diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java index 146e4f0dd58ca..47d5e9f1f7964 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java @@ -143,12 +143,16 @@ public void testAutoRecoverySessionLoss() throws Exception { } BookieId currentAuditor = main1.auditorElector.getCurrentAuditor(); assertNotNull(currentAuditor); - Auditor auditor1 = main1.auditorElector.getAuditor(); assertEquals("Current Auditor should be AR1", currentAuditor, BookieImpl.getBookieId(confByIndex(0))); + // getCurrentAuditor() can resolve as soon as the election settles, before the elector + // thread has constructed the Auditor instance — re-read getAuditor() on every poll instead + // of capturing a possibly-null reference once. Awaitility.waitAtMost(30, TimeUnit.SECONDS).untilAsserted(() -> { - assertNotNull(auditor1); - assertTrue("Auditor of AR1 should be running", auditor1.isRunning()); + Auditor a1 = main1.auditorElector.getAuditor(); + assertNotNull(a1); + assertTrue("Auditor of AR1 should be running", a1.isRunning()); }); + Auditor auditor1 = main1.auditorElector.getAuditor(); /* diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java index 4b48f3c20b02b..cb26953738374 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java @@ -18,10 +18,13 @@ */ package org.apache.pulsar.metadata; +import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import java.util.EnumSet; +import java.util.List; import java.util.Optional; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -35,6 +38,7 @@ import org.apache.pulsar.metadata.api.extended.CreateOption; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl; +import org.awaitility.Awaitility; import org.testng.annotations.Test; public class LeaderElectionTest extends BaseMetadataStoreTest { @@ -284,4 +288,155 @@ public void revalidateLeaderWithDifferentSessionsDifferentValue(String provider, assertEquals(le.getLeaderValue().join(), Optional.of("test-1")); assertEqualsAndRetry(() -> le.getLeaderValueIfPresent(), Optional.of("test-1"), Optional.empty()); } + + @Test(dataProvider = "impl", timeOut = 30000) + public void readsDoNotObserveEmptyLeaderDuringReElection(String provider, Supplier urlSupplier) + throws Exception { + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs = new CoordinationServiceImpl(store); + + @Cleanup + LeaderElection le = cs.getLeaderElection(String.class, path, __ -> { + }); + + assertEquals(le.elect("test-1").join(), LeaderElectionState.Leading); + + // Externally delete the leader node: the instance re-elects itself. An authoritative read + // issued during the churn either returns the last settled value or waits for the + // re-election to settle — it never observes an empty leader. + store.delete(path, Optional.empty()).join(); + assertEquals(le.getLeaderValue().join(), Optional.of("test-1")); + + Awaitility.await().untilAsserted(() -> { + assertEquals(le.getState(), LeaderElectionState.Leading); + assertEquals(le.getLeaderValueIfPresent(), Optional.of("test-1")); + }); + } + + @Test(dataProvider = "zkImpls", timeOut = 30000) + public void followerReadsResolveToTheNewLeaderAfterHandoff(String provider, Supplier urlSupplier) + throws Exception { + @Cleanup + MetadataStoreExtended store1 = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().build()); + @Cleanup + MetadataStoreExtended store2 = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs1 = new CoordinationServiceImpl(store1); + @Cleanup + CoordinationService cs2 = new CoordinationServiceImpl(store2); + + @Cleanup + LeaderElection le1 = cs1.getLeaderElection(String.class, path, __ -> { + }); + @Cleanup + LeaderElection le2 = cs2.getLeaderElection(String.class, path, __ -> { + }); + + assertEquals(le1.elect("test-1").join(), LeaderElectionState.Leading); + assertEquals(le2.elect("test-2").join(), LeaderElectionState.Following); + assertEquals(le2.getLeaderValue().join(), Optional.of("test-1")); + + // The leader hands off: le2 re-elects itself. Authoritative reads during the handoff + // return one of the settled leader values and converge to the new leader, but never + // observe an empty leader. + le1.close(); + List> observed = new CopyOnWriteArrayList<>(); + Awaitility.await().untilAsserted(() -> { + Optional leader = le2.getLeaderValue().join(); + observed.add(leader); + assertEquals(leader, Optional.of("test-2")); + }); + assertThat(observed) + .as("authoritative reads during the leadership handoff") + .doesNotContain(Optional.empty()); + } + + @Test(dataProvider = "impl", timeOut = 30000) + public void closedLeaderReportsEmptyLeader(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs = new CoordinationServiceImpl(store); + + LeaderElection le = cs.getLeaderElection(String.class, path, __ -> { + }); + + assertEquals(le.elect("test-1").join(), LeaderElectionState.Leading); + assertEquals(le.getLeaderValue().join(), Optional.of("test-1")); + + // Closing the leader releases the leadership; reads on the closed instance must report an + // empty leader without waiting (recovery paths key off the "no leader" condition). + le.close(); + assertEquals(le.getLeaderValue().join(), Optional.empty()); + assertEquals(le.getLeaderValueIfPresent(), Optional.empty()); + } + + @Test(dataProvider = "impl", timeOut = 30000) + public void electAfterCloseRunsANewElection(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs = new CoordinationServiceImpl(store); + + @Cleanup + LeaderElection le = cs.getLeaderElection(String.class, path, __ -> { + }); + + assertEquals(le.elect("test-1").join(), LeaderElectionState.Leading); + le.close(); + + // Re-electing on a closed instance reopens it (the broker's LeaderElectionService is + // close()d and start()ed again to force a leadership change). + assertEquals(le.elect("test-1").join(), LeaderElectionState.Leading); + assertEquals(le.getLeaderValue().join(), Optional.of("test-1")); + assertEquals(le.getLeaderValueIfPresent(), Optional.of("test-1")); + } + + @Test(dataProvider = "impl", timeOut = 30000) + public void observerReadsLeaderValueFromStore(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs = new CoordinationServiceImpl(store); + @Cleanup + CoordinationService observerCs = new CoordinationServiceImpl(store); + + @Cleanup + LeaderElection le = cs.getLeaderElection(String.class, path, __ -> { + }); + // The observer never calls elect(): there is no local election cycle to wait for, so the + // authoritative read goes directly to the metadata store, while the snapshot stays empty. + @Cleanup + LeaderElection observer = observerCs.getLeaderElection(String.class, path, __ -> { + }); + + assertEquals(observer.getLeaderValue().join(), Optional.empty()); + + assertEquals(le.elect("test-1").join(), LeaderElectionState.Leading); + assertEquals(observer.getLeaderValue().join(), Optional.of("test-1")); + assertEquals(observer.getLeaderValueIfPresent(), Optional.empty()); + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java index 09c9d71c41a30..4827c4a197ad5 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java @@ -18,7 +18,14 @@ */ package org.apache.pulsar.metadata.coordination.impl; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import lombok.Cleanup; import org.apache.pulsar.metadata.BaseMetadataStoreTest; @@ -62,4 +69,25 @@ public void validateDeadLock(String provider, Supplier urlSupplier) }); blockFuture.join(); } + + @Test(timeOut = 20000) + public void getLeaderValueTimesOutWhenElectionNeverCompletes() { + MetadataStoreExtended store = mock(MetadataStoreExtended.class); + // The store never answers, so the election never settles. + when(store.get(anyString())).thenReturn(new CompletableFuture<>()); + + @Cleanup("shutdownNow") + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + LeaderElectionImpl le = new LeaderElectionImpl<>(store, String.class, + "/getLeaderValueTimesOutWhenElectionNeverCompletes", __ -> { + }, executor); + le.setLeaderElectionCompletionTimeoutSeconds(1); + + le.elect("test-1"); + + assertThatThrownBy(() -> le.getLeaderValue().join()) + .hasCauseInstanceOf(TimeoutException.class) + .cause() + .hasMessageContaining("did not complete within"); + } } From 2baa1f7e67a4091b01c558048bb75b2667f2dfa5 Mon Sep 17 00:00:00 2001 From: David Kjerrumgaard <35466513+david-streamlio@users.noreply.github.com> Date: Wed, 17 Jun 2026 12:00:56 -0700 Subject: [PATCH 145/201] [fix][client] Prevent client shutdown from leaking event loop threads when DNS resolver close fails (#26045) (cherry picked from commit d58727f3bf2777170ccc19afccf0c4364b94e4da) --- .../pulsar/client/impl/PulsarClientImpl.java | 14 ++++++-- .../client/impl/PulsarClientImplTest.java | 34 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index 2082f8b17502e..d83cbbb74827d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -946,11 +946,21 @@ public void shutdown() throws PulsarClientException { } if (addressResolver != null) { - addressResolver.close(); + try { + addressResolver.close(); + } catch (Throwable t) { + log.warn("Failed to close addressResolver", t); + throwable = t; + } } if (dnsResolverGroupLocalInstance != null) { - dnsResolverGroupLocalInstance.close(); + try { + dnsResolverGroupLocalInstance.close(); + } catch (Throwable t) { + log.warn("Failed to close dnsResolverGroup", t); + throwable = t; + } } try { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java index 72c942eaf0c5b..5d44649dac13d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java @@ -23,6 +23,7 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -38,6 +39,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPromise; import io.netty.channel.EventLoopGroup; +import io.netty.resolver.AddressResolver; import io.netty.resolver.dns.DefaultDnsServerAddressStreamProvider; import io.netty.util.HashedWheelTimer; import io.netty.util.concurrent.DefaultThreadFactory; @@ -179,6 +181,38 @@ public void testInitializeWithoutTimer() throws Exception { verify(timer).stop(); } + @Test + public void testShutdownContinuesWhenAddressResolverCloseFails() throws Exception { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setServiceUrl("pulsar://localhost:6650"); + PulsarClientImpl client = new PulsarClientImpl(conf); + + // Simulate the Netty DNS resolver shutdown race where AddressResolver.close() + // throws "channel not registered to an event loop". + @SuppressWarnings("unchecked") + AddressResolver failingResolver = mock(AddressResolver.class); + doThrow(new IllegalStateException("channel not registered to an event loop")) + .when(failingResolver).close(); + Field addressResolverField = PulsarClientImpl.class.getDeclaredField("addressResolver"); + addressResolverField.setAccessible(true); + addressResolverField.set(client, failingResolver); + + // Replace the timer with a mock so we can assert that shutdown still reaches the + // cleanup steps that run after the address resolver is closed. + HashedWheelTimer timer = mock(HashedWheelTimer.class); + Field timerField = PulsarClientImpl.class.getDeclaredField("timer"); + timerField.setAccessible(true); + timerField.set(client, timer); + + // shutdown() still surfaces the original failure, but it must not abort early: + // the remaining resources have to be released regardless. The key assertion is + // that timer.stop() (a step that runs after the resolver is closed) is reached. + assertThrows(IllegalStateException.class, client::shutdown); + + verify(failingResolver).close(); + verify(timer).stop(); + } + @Test public void testInitializeWithTimer() throws PulsarClientException { ClientConfigurationData conf = new ClientConfigurationData(); From d5bdaaa3256df5ea2bf57c4500e99db5a275ea0c Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sat, 13 Jun 2026 07:01:22 +0800 Subject: [PATCH 146/201] [fix][broker] Fix delayed messages stalling with isDelayedDeliveryDeliverAtTimeStrict=true (#26012) Co-authored-by: Lari Hotari (cherry picked from commit 027f4e94b4a2e975c4aae2d6111bf31d201dc8b1) --- .../AbstractDelayedDeliveryTracker.java | 91 +++++++-- .../InMemoryDelayedDeliveryTracker.java | 46 +++-- .../bucket/BucketDelayedDeliveryTracker.java | 7 +- .../delayed/InMemoryDeliveryTrackerTest.java | 186 +++++++++++++++++- 4 files changed, 293 insertions(+), 37 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java index bec5134c4f79a..de843d035068b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java @@ -34,20 +34,25 @@ public abstract class AbstractDelayedDeliveryTracker implements DelayedDeliveryT // Reference to the shared (per-broker) timer for delayed delivery protected final Timer timer; - // Current timeout or null if not set - protected Timeout timeout; + // Current timeout or null if not set. Guarded by timeoutLock. + private Timeout timeout; - // Timestamp at which the timeout is currently set + // Timestamp at which the timeout is currently set. Guarded by timeoutLock. private long currentTimeoutTarget; - // Last time the TimerTask was triggered for this class + // Last time the TimerTask was triggered for this class. Guarded by timeoutLock. private long lastTickRun; - protected long tickTimeMillis; + // Updated through resetTickTime() from dispatcher threads and read on the timer thread. + protected volatile long tickTimeMillis; protected final Clock clock; private final boolean isDelayedDeliveryDeliverAtTimeStrict; + // Guards the timer state (timeout, currentTimeoutTarget, lastTickRun) against concurrent access from + // dispatcher threads (updateTimer/rescheduleTimer/close) and the timer thread (run). It is a leaf lock: + // no subclass method is invoked while holding it. + private final Object timeoutLock = new Object(); public AbstractDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, @@ -81,14 +86,34 @@ protected long getCutoffTime() { return isDelayedDeliveryDeliverAtTimeStrict ? clock.millis() : clock.millis() + tickTimeMillis; } + protected boolean isDeliverAtTimeStrict() { + return isDelayedDeliveryDeliverAtTimeStrict; + } + public void resetTickTime(long tickTime) { if (this.tickTimeMillis != tickTime) { this.tickTimeMillis = tickTime; } } - protected void updateTimer() { - if (getNumberOfDelayedMessages() == 0) { + /** + * Update the delivery timer to fire when the next message in the tracker becomes due. + * + * Callers are expected to serialize all tracker state mutations (at the dispatcher or tracker level), so the + * snapshot of {@link #getNumberOfDelayedMessages()} and {@link #nextDeliveryTime()} is taken before acquiring + * timeoutLock. This keeps timeoutLock a leaf lock that never calls into subclass methods, ruling out lock + * ordering deadlocks with subclasses that synchronize those methods on the tracker instance. + */ + protected final void updateTimer() { + long numberOfDelayedMessages = getNumberOfDelayedMessages(); + long nextDeliveryTimestamp = numberOfDelayedMessages > 0 ? nextDeliveryTime() : -1; + synchronized (timeoutLock) { + doUpdateTimer(numberOfDelayedMessages, nextDeliveryTimestamp); + } + } + + private void doUpdateTimer(long numberOfDelayedMessages, long timestamp) { + if (numberOfDelayedMessages == 0) { if (timeout != null) { currentTimeoutTarget = -1; timeout.cancel(); @@ -96,7 +121,6 @@ protected void updateTimer() { } return; } - long timestamp = nextDeliveryTime(); if (timestamp == currentTimeoutTarget) { // The timer is already set to the correct target time return; @@ -104,18 +128,22 @@ protected void updateTimer() { if (timeout != null) { timeout.cancel(); + timeout = null; } + // Reset the tracked state so a subsequent updateTimer() call cannot short-circuit on a stale + // currentTimeoutTarget while no live timer remains. See #25996. + currentTimeoutTarget = -1; long now = clock.millis(); long delayMillis = timestamp - now; - if (delayMillis < 0) { + if (delayMillis <= 0) { // There are messages that are already ready to be delivered. If // the dispatcher is not getting them is because the consumer is // either not connected or slow. // We don't need to keep retriggering the timer. When the consumer // catches up, the dispatcher will do the readMoreEntries() and - // get these messages + // get these messages. return; } @@ -126,7 +154,6 @@ protected void updateTimer() { if (log.isDebugEnabled()) { log.debug("[{}] Start timer in {} millis", dispatcher.getName(), calculatedDelayMillis); } - // Even though we may delay longer than this timestamp because of the tick delay, we still track the // current timeout with reference to the next message's timestamp. currentTimeoutTarget = timestamp; @@ -134,27 +161,53 @@ protected void updateTimer() { } @Override - public void run(Timeout timeout) throws Exception { + public void run(Timeout triggeredTimeout) throws Exception { if (log.isDebugEnabled()) { log.debug("[{}] Timer triggered", dispatcher.getName()); } - if (timeout == null || timeout.isCancelled()) { + if (triggeredTimeout == null || triggeredTimeout.isCancelled()) { return; } - synchronized (dispatcher) { + synchronized (timeoutLock) { lastTickRun = clock.millis(); - currentTimeoutTarget = -1; - this.timeout = null; + // Only reset the timer state if the triggered timeout is the currently armed one. A timeout that + // was already superseded by updateTimer()/rescheduleTimer() may still fire if it passed its + // isCancelled() check before being cancelled; it must not clear the state of the newer timer. + if (triggeredTimeout == this.timeout) { + currentTimeoutTarget = -1; + this.timeout = null; + } + } + + synchronized (dispatcher) { dispatcher.readMoreEntriesAsync(); } } + /** + * Cancel the current timer (if any) and schedule the timer task to run after the given delay. Used by + * subclasses to trigger a dispatch round from asynchronous completions instead of mutating the timer + * state directly. + */ + protected final void rescheduleTimer(long delayMillis) { + synchronized (timeoutLock) { + if (timeout != null) { + timeout.cancel(); + } + currentTimeoutTarget = -1; + timeout = timer.newTimeout(this, delayMillis, TimeUnit.MILLISECONDS); + } + } + @Override public void close() { - if (timeout != null) { - timeout.cancel(); - timeout = null; + synchronized (timeoutLock) { + if (timeout != null) { + timeout.cancel(); + timeout = null; + } + currentTimeoutTarget = -1; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 1ded2cc428d32..4f05661a28b25 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -64,6 +64,7 @@ public class InMemoryDelayedDeliveryTracker extends AbstractDelayedDeliveryTrack // The bit count to trim to reduce memory occupation. private final int timestampPrecisionBitCnt; + private final long precisionMillis; // Count of delayed messages in the tracker. private final AtomicLong delayedMessagesCount = new AtomicLong(0); @@ -83,6 +84,7 @@ public InMemoryDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsum super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); this.fixedDelayDetectionLookahead = fixedDelayDetectionLookahead; this.timestampPrecisionBitCnt = calculateTimestampPrecisionBitCnt(tickTimeMillis); + this.precisionMillis = 1L << timestampPrecisionBitCnt; } /** @@ -125,7 +127,7 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { deliverAt - clock.millis()); } - long timestamp = trimLowerBit(deliverAt, timestampPrecisionBitCnt); + long timestamp = roundTimestamp(deliverAt); Roaring64Bitmap bitmap = delayedMessageMap.computeIfAbsent(timestamp, k -> new Long2ObjectRBTreeMap<>()) .computeIfAbsent(ledgerId, k -> new Roaring64Bitmap()); // Roaring64Bitmap does not store duplicates, so track if it a new element @@ -133,7 +135,7 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { boolean isNew = !bitmap.contains(entryId); if (isNew) { - bitmap.add(entryId); + bitmap.addLong(entryId); delayedMessagesCount.incrementAndGet(); } @@ -144,6 +146,27 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { return true; } + /** + * Round the deliverAt timestamp to the bucket boundary used as the key in {@link #delayedMessageMap}, so that + * all messages within the same bucket share a single map entry to reduce memory usage. + * + * In strict delivery mode the timestamp is rounded up: a bucket then becomes due only after every deliverAt + * time inside it has passed, so messages are delivered up to one bucket (less than tickTimeMillis) late, but + * never before their deliverAt time. Rounding down instead would let {@link #getScheduledMessages(int)} hand a + * message to the dispatcher before its deliverAt time; the dispatcher would put it back and re-trigger reads + * in a loop until the deliverAt time is reached (see issue #25996). + * + * In non-strict mode the timestamp is rounded down, since delivering up to tickTimeMillis early is allowed. + */ + private long roundTimestamp(long deliverAt) { + if (isDeliverAtTimeStrict()) { + // round up, saturating at Long.MAX_VALUE instead of overflowing for deliverAt close to Long.MAX_VALUE + long roundedUp = deliverAt + precisionMillis - 1; + return trimLowerBit(roundedUp < deliverAt ? Long.MAX_VALUE : roundedUp, timestampPrecisionBitCnt); + } + return trimLowerBit(deliverAt, timestampPrecisionBitCnt); + } + /** * Check that new delivery time comes after the current highest, or at * least within a single tick time interval of 1 second. @@ -189,20 +212,22 @@ public NavigableSet getScheduledMessages(int maxMessages) { for (Long2ObjectMap.Entry ledgerEntry : ledgerMap.long2ObjectEntrySet()) { long ledgerId = ledgerEntry.getLongKey(); Roaring64Bitmap entryIds = ledgerEntry.getValue(); - int cardinality = (int) entryIds.getLongCardinality(); + long cardinality = entryIds.getLongCardinality(); if (cardinality <= n) { + int cardinalityInt = (int) cardinality; entryIds.forEach(entryId -> { positions.add(PositionFactory.create(ledgerId, entryId)); }); - n -= cardinality; - delayedMessagesCount.addAndGet(-cardinality); + n -= cardinalityInt; + delayedMessagesCount.addAndGet(-cardinalityInt); ledgerIdToDelete.add(ledgerId); } else { - long[] entryIdsArray = entryIds.toArray(); - for (int i = 0; i < n; i++) { - positions.add(PositionFactory.create(ledgerId, entryIdsArray[i])); - entryIds.removeLong(entryIdsArray[i]); - } + Roaring64Bitmap entryIdsToRemove = new Roaring64Bitmap(); + entryIds.stream().limit(n).forEach(entryId -> { + positions.add(PositionFactory.create(ledgerId, entryId)); + entryIdsToRemove.addLong(entryId); + }); + entryIds.andNot(entryIdsToRemove); delayedMessagesCount.addAndGet(-n); n = 0; } @@ -217,7 +242,6 @@ public NavigableSet getScheduledMessages(int maxMessages) { delayedMessageMap.remove(timestamp); } } - if (log.isDebugEnabled()) { log.debug("[{}] Get scheduled messages - found {}", dispatcher.getName(), positions.size()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 9ed304c73312d..67bc9d29360fc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -682,12 +682,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load, System.currentTimeMillis() - loadStartTime); } - synchronized (this) { - if (timeout != null) { - timeout.cancel(); - } - timeout = timer.newTimeout(this, 0, TimeUnit.MILLISECONDS); - } + rescheduleTimer(0); }); if (!checkPendingLoadDone() || loadFuture.isCompletedExceptionally()) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index 2affb7fef22e2..1c16ed4228f33 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -21,6 +21,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -33,13 +35,16 @@ import io.netty.util.concurrent.DefaultThreadFactory; import java.lang.reflect.Method; import java.time.Clock; +import java.util.Map; import java.util.NavigableMap; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import lombok.Cleanup; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.service.persistent.AbstractPersistentDispatcherMultipleConsumers; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -114,6 +119,39 @@ public Object[][] provider(Method method) throws Exception { new InMemoryDelayedDeliveryTracker(dispatcher, timer, 8, clock, true, 100) }}; + case "testStrictModeNeverDeliversEarlyAndKeepsTimerArmed", + "testStaleTimerTriggerDoesNotClearNewerTimer" -> { + // Mock timer that records the currently-armed timeouts so the test can observe whether a + // delivery timer is live and fire it like the wheel would (passing the armed Timeout instance, + // which the tracker's run() compares against its current timeout). Cancelling a timeout removes + // it from the map; firing (polling) it does not mark it cancelled, mirroring the wheel. + Timer mockTimer = mock(Timer.class); + NavigableMap> tasks = new TreeMap<>(); + when(mockTimer.newTimeout(any(), anyLong(), any())).then(invocation -> { + TimerTask task = invocation.getArgument(0, TimerTask.class); + long delay = invocation.getArgument(1, Long.class); + TimeUnit unit = invocation.getArgument(2, TimeUnit.class); + long scheduleAt = clockTime.get() + unit.toMillis(delay); + Timeout t = mock(Timeout.class); + Map.Entry entry = Map.entry(task, t); + AtomicBoolean cancelled = new AtomicBoolean(); + when(t.cancel()).then(i -> { + cancelled.set(true); + return tasks.remove(scheduleAt, entry); + }); + when(t.isCancelled()).then(i -> cancelled.get()); + tasks.put(scheduleAt, entry); + return t; + }); + // tickTimeMillis=1000 -> delivery timestamps are bucketed at 512ms granularity (lower 9 bits), + // rounded up in strict mode so that messages are never visible before their deliverAt time. + yield new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, mockTimer, 1000, clock, + true, 0), + tasks, + mockTimer + }}; + } default -> new Object[][]{{ new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, true, 0) @@ -229,7 +267,7 @@ public void testClose() throws Exception { true, 0) { @Override public void run(Timeout timeout) throws Exception { - super.timeout = timer.newTimeout(this, 1, TimeUnit.MILLISECONDS); + rescheduleTimer(1); if (timeout == null || timeout.isCancelled()) { return; } @@ -274,6 +312,106 @@ public void testDelaySequence(InMemoryDelayedDeliveryTracker tracker) throws Exc tracker.close(); } + /** + * Regression test for https://github.com/apache/pulsar/issues/25996 and for the strict-mode guarantee that + * messages are never delivered before their deliverAt time. + * + * With isDelayedDeliveryDeliverAtTimeStrict=true and tickTimeMillis=1000, delivery timestamps are bucketed + * at 512ms granularity and rounded UP, so a bucket only becomes due once every deliverAt time inside it has + * passed. Previously timestamps were rounded down, so a message could be popped up to ~511ms early; the + * dispatcher would re-add the not-yet-due message and the re-add left a stale {@code currentTimeoutTarget} + * behind that suppressed re-arming the delivery timer, stalling all remaining delayed messages until an + * unrelated dispatch event occurred. + * + * The rounded-up buckets (multiples of 512ms) used below: + * M1 deliverAt=60400 -> bucket 60416 + * M2 deliverAt=61000 -> bucket 61440 + */ + @Test(dataProvider = "delayedTracker") + public void testStrictModeNeverDeliversEarlyAndKeepsTimerArmed(InMemoryDelayedDeliveryTracker tracker, + NavigableMap> tasks, Timer mockTimer) throws Exception { + clockTime.set(0); + + // Two delayed messages in different buckets. A delivery timer is armed for the earliest. + assertTrue(tracker.addMessage(1, 1, 60400)); + assertTrue(tracker.addMessage(2, 2, 61000)); + assertEquals(tasks.size(), 1, "a delivery timer should be armed for the earliest message"); + assertEquals(tasks.firstKey().longValue(), 60416, "the timer should target M1's rounded-up bucket"); + + // Before M1's bucket time, nothing may be visible to the dispatcher (no early delivery), and a + // dispatch round that finds nothing must leave the delivery timer armed (issue #25996). + clockTime.set(60000); + assertFalse(tracker.hasMessageAvailable()); + assertTrue(tracker.getScheduledMessages(100).isEmpty(), + "strict mode must not deliver a message before its deliverAt time"); + assertEquals(tasks.size(), 1, "the delivery timer must remain armed"); + + // The timer fires at M1's bucket time; M1 is delivered at 60416 >= deliverAt 60400, so the + // dispatcher never needs to re-add it. + clockTime.set(60416); + Map.Entry firedTimeout = tasks.pollFirstEntry().getValue(); + firedTimeout.getKey().run(firedTimeout.getValue()); + Set scheduled = tracker.getScheduledMessages(100); + assertEquals(scheduled, Set.of(PositionFactory.create(1, 1))); + + // M2 is still pending and not yet due, so a delivery timer must have been re-armed for it. With the + // issue #25996 bug, the timer state went stale at this point and M2 stalled indefinitely. + assertEquals(tracker.getNumberOfDelayedMessages(), 1); + assertFalse(tracker.hasMessageAvailable()); + assertEquals(tasks.size(), 1, "a delivery timer must remain armed for the pending message M2"); + assertEquals(tasks.firstKey().longValue(), 61440, "the timer should target M2's rounded-up bucket"); + + // The timer fires again and M2 is delivered, also never early. + clockTime.set(61440); + firedTimeout = tasks.pollFirstEntry().getValue(); + firedTimeout.getKey().run(firedTimeout.getValue()); + scheduled = tracker.getScheduledMessages(100); + assertEquals(scheduled, Set.of(PositionFactory.create(2, 2))); + assertEquals(tracker.getNumberOfDelayedMessages(), 0); + + tracker.close(); + } + + /** + * A timeout that was superseded by a newer one may still fire: HashedWheelTimer can run a task that passed + * its isCancelled() check just before updateTimer() cancelled it. Such a stale trigger must not clear the + * state of the newer armed timer, otherwise the next updateTimer() call would arm a duplicate timer. + */ + @Test(dataProvider = "delayedTracker") + public void testStaleTimerTriggerDoesNotClearNewerTimer(InMemoryDelayedDeliveryTracker tracker, + NavigableMap> tasks, Timer mockTimer) throws Exception { + clockTime.set(0); + + // Arm a timer for M2, then supersede it with an earlier message M1. + assertTrue(tracker.addMessage(2, 2, 61000)); + assertEquals(tasks.firstKey().longValue(), 61440); + assertTrue(tracker.addMessage(1, 1, 60400)); + assertEquals(tasks.size(), 1); + assertEquals(tasks.firstKey().longValue(), 60416, "M1's timer should have replaced M2's"); + + // The superseded (cancelled) timeout for M2 fires anyway, racing with the cancellation. The tracker + // must keep the state of the currently armed timer for M1. + Timeout staleTimeout = mock(Timeout.class); + tracker.run(staleTimeout); + + // A subsequent updateTimer() (here through hasMessageAvailable()) must recognize the armed timer + // instead of arming a duplicate one: still exactly the two newTimeout() calls from the adds above. + assertFalse(tracker.hasMessageAvailable()); + assertEquals(tasks.size(), 1); + assertEquals(tasks.firstKey().longValue(), 60416); + verify(mockTimer, times(2)).newTimeout(any(), anyLong(), any()); + + // The armed timer fires and delivery proceeds normally. + clockTime.set(60416); + Map.Entry firedTimeout = tasks.pollFirstEntry().getValue(); + firedTimeout.getKey().run(firedTimeout.getValue()); + Set scheduled = tracker.getScheduledMessages(100); + assertEquals(scheduled, Set.of(PositionFactory.create(1, 1))); + assertEquals(tasks.size(), 1, "a delivery timer must be re-armed for the still pending M2"); + + tracker.close(); + } + @Test(dataProvider = "delayedTracker") public void testAddMultipleMessagesSameWindow(InMemoryDelayedDeliveryTracker tracker) throws Exception { tracker.addMessage(1, 1, 50); @@ -284,4 +422,50 @@ public void testAddMultipleMessagesSameWindow(InMemoryDelayedDeliveryTracker tra tracker.getScheduledMessages(10); } + + /** + * Covers the partial drain of a per-ledger entry id bitmap in getScheduledMessages, where the bitmap holds + * more entries than the remaining maxMessages budget (the cardinality > n branch): only the lowest n entry + * ids may be returned and the rest must stay tracked, without duplicates across calls. + */ + @Test(dataProvider = "delayedTracker") + public void testGetScheduledMessagesWithMaxMessagesSmallerThanBucket(InMemoryDelayedDeliveryTracker tracker) + throws Exception { + clockTime.set(0); + + // Two ledgers within the same delivery bucket: ledger 1 with 2 entries, ledger 2 with 5 entries. + assertTrue(tracker.addMessage(1, 0, 10)); + assertTrue(tracker.addMessage(1, 1, 10)); + for (int entryId = 0; entryId < 5; entryId++) { + assertTrue(tracker.addMessage(2, entryId, 10)); + } + assertEquals(tracker.getNumberOfDelayedMessages(), 7); + + clockTime.set(10); + + // maxMessages drains ledger 1 fully and ledger 2 partially (cardinality > n on ledger 2's bitmap). + Set scheduled = tracker.getScheduledMessages(4); + assertEquals(scheduled, Set.of( + PositionFactory.create(1, 0), + PositionFactory.create(1, 1), + PositionFactory.create(2, 0), + PositionFactory.create(2, 1))); + assertEquals(tracker.getNumberOfDelayedMessages(), 3); + + // Another partial drain of ledger 2's remaining entries (cardinality > n again): continues with the + // next lowest entry ids, no duplicates from the previous call. + scheduled = tracker.getScheduledMessages(2); + assertEquals(scheduled, Set.of( + PositionFactory.create(2, 2), + PositionFactory.create(2, 3))); + assertEquals(tracker.getNumberOfDelayedMessages(), 1); + + // The last remaining entry is returned and the tracker is emptied. + scheduled = tracker.getScheduledMessages(10); + assertEquals(scheduled, Set.of(PositionFactory.create(2, 4))); + assertEquals(tracker.getNumberOfDelayedMessages(), 0); + assertFalse(tracker.hasMessageAvailable()); + + tracker.close(); + } } From f83504b989ee7ba011483107287ffb4683294436 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 13 Jun 2026 04:55:20 +0300 Subject: [PATCH 147/201] [fix][broker] Prevent subscribe rate limit from stalling compaction and blocking forced deletion (#26015) (cherry picked from commit a1b5a0dbd8d67f3fcb5db4db53f73e7acd8c825c) --- .../service/persistent/PersistentTopic.java | 11 +++++-- .../pulsar/compaction/CompactionTest.java | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 89e1a47de30c1..32fb03ee40d3f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -992,8 +992,15 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St new NamingException("Subscription with reserved subscription name attempted")); } - if (cnx.clientAddress() != null && cnx.clientAddress().toString().contains(":") - && subscribeRateLimiter.isPresent()) { + // The subscribe rate limit must not apply to the broker-internal compaction subscription: the + // compactor's reader re-subscribes after the phase-two seek, and throttling that re-subscribe stalls + // the compaction, which in turn blocks forced topic/namespace deletion waiting on the in-flight + // compaction. System topics are exempt as well, consistent with the publish/dispatch rate limiters, + // since throttling broker-internal readers (e.g. on __change_events) can stall topic policy updates. + if (subscribeRateLimiter.isPresent() + && !isSystemTopic() + && !isCompactionSubscription(subscriptionName) + && cnx.clientAddress() != null && cnx.clientAddress().toString().contains(":")) { SubscribeRateLimiter.ConsumerIdentifier consumer = new SubscribeRateLimiter.ConsumerIdentifier( cnx.clientAddress().toString().split(":")[0], consumerName, consumerId); if (!subscribeRateLimiter.get().subscribeAvailable(consumer) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index e7cb6300c516c..0c2d5fcc746a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -110,6 +110,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.protocol.Markers; import org.apache.pulsar.common.util.FutureUtil; @@ -196,6 +197,34 @@ protected PublishingOrderCompactor getCompactor() { return compactor; } + @Test + public void testCompactionNotBlockedBySubscribeRateLimit() throws Exception { + String namespace = "my-tenant/my-ns"; + String topic = "persistent://" + namespace + "/compaction-with-subscribe-rate-limit"; + + try (Producer producer = pulsarClient.newProducer().topic(topic).enableBatching(false).create()) { + for (int i = 0; i < 10; i++) { + producer.newMessage().key("key" + (i % 2)).value(("my-message-" + i).getBytes()).send(); + } + } + + // Allow a single subscribe per consumer within a long period. The compactor's reader consumes the only + // token when it first subscribes, so the re-subscribe triggered by the phase-two seek would be throttled + // if the limit applied to the compaction subscription, stalling the compaction. + admin.namespaces().setSubscribeRate(namespace, new SubscribeRate(1, 3600)); + Awaitility.await().untilAsserted(() -> { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + assertTrue(persistentTopic.getSubscribeRateLimiter().isPresent()); + }); + + try { + compactor.compact(topic).get(30, TimeUnit.SECONDS); + } finally { + admin.namespaces().removeSubscribeRate(namespace); + } + } + @Test public void testCompaction() throws Exception { String topic = "persistent://my-tenant/my-ns/compaction"; From dfc1d2daeb20a2239d7b176afcbbb632a781065e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 13 Jun 2026 08:53:57 +0300 Subject: [PATCH 148/201] [fix][broker] Fix forced topic/namespace deletion hanging or failing when compaction is in progress (#26016) (cherry picked from commit ec56aea8370e2dfdfda7aeeb1f5ee730d2d86866) --- .../service/persistent/PersistentTopic.java | 18 +++--- .../pulsar/client/impl/RawReaderImpl.java | 11 ++++ .../pulsar/client/impl/RawReaderTest.java | 19 ++++++ .../pulsar/compaction/CompactionTest.java | 58 +++++++++++++++++-- 4 files changed, 95 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 32fb03ee40d3f..5bc882b011189 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1374,8 +1374,16 @@ private void asyncDeleteCursorWithCleanCompactionLedger(PersistentSubscription s return; } } - // Unsubscribe compaction cursor and delete compacted ledger. - currentCompaction.thenCompose(__ -> { + // Unsubscribe compaction cursor and delete compacted ledger. Wait for any in-flight compaction to finish + // first, but don't let a compaction that completed exceptionally block the cursor deletion: the deletion + // would otherwise fail on every retry until the topic instance is reloaded (issue #24148). Note that a + // fenced topic makes the compactor's reader fail with an unrecoverable error, so a forced deletion + // terminates an in-flight compaction exceptionally rather than waiting for it to complete normally. + currentCompaction.exceptionally(compactionEx -> { + log.info("[{}][{}] Last compaction task failed, proceeding to delete the compaction cursor", + topic, subscriptionName, compactionEx); + return null; + }).thenCompose(__ -> { asyncDeleteCursor(subscriptionName, unsubscribeFuture); return unsubscribeFuture; }).thenAccept(__ -> { @@ -1393,11 +1401,7 @@ private void asyncDeleteCursorWithCleanCompactionLedger(PersistentSubscription s disablingCompaction.compareAndSet(true, false); } }).exceptionally(ex -> { - if (currentCompaction.isCompletedExceptionally()) { - log.warn("[{}][{}] Last compaction task failed", topic, subscriptionName); - } else { - log.warn("[{}][{}] Failed to delete cursor task failed", topic, subscriptionName); - } + log.warn("[{}][{}] Failed to delete the compaction cursor", topic, subscriptionName, ex); // Reset the variable: disablingCompaction, disablingCompaction.compareAndSet(true, false); unsubscribeFuture.completeExceptionally(ex); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index c3f8254fb5e30..1bc88ed0532da 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -230,6 +230,17 @@ CompletableFuture receiveRawAsync() { CompletableFuture result = new CompletableFuture<>(); pendingRawReceives.add(result); tryCompletePending(); + // Once the consumer has reached a terminal state (for example it was closed after an + // unrecoverable error such as the topic or namespace being deleted), no further message + // will arrive and no close callback will run for receives enqueued from now on, so the + // future would never complete. Re-checking the state after enqueueing closes the race + // with a concurrent close() draining the queue, since close() drains only after moving to + // a terminal state. This matters for compaction: a never-completing read leaves the + // compaction future pending, which in turn blocks forced topic/namespace deletion. + State state = getState(); + if (state == State.Closing || state == State.Closed || state == State.Failed) { + failPendingRawReceives(); + } return result; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java index bb98ca1e4e77d..9320e22e42e8b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java @@ -624,6 +624,25 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { admin.topics().delete(topic, false); } + @Test(timeOut = 30000) + public void testReadNextAsyncCompletesAfterConsumerClosed() throws Exception { + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); + admin.topics().createNonPartitionedTopic(topic); + RawReader reader = RawReader.create(pulsarClient, topic, subscription).get(); + + // Put the reader's underlying consumer into a terminal state. In production this happens when a + // compaction's RawReader hits an unrecoverable error (e.g. the topic/namespace is being deleted). + reader.closeAsync().get(5, TimeUnit.SECONDS); + + // A read issued once the consumer has reached a terminal state must complete instead of hanging + // forever: a never-completing read keeps the compaction future pending, which blocks forced + // topic/namespace deletion (issue #24148). + CompletableFuture readFuture = reader.readNextAsync(); + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertTrue(readFuture.isDone())); + assertTrue(readFuture.isCompletedExceptionally()); + } + @Test(timeOut = 100000) public void testPauseAndResume() throws Exception { log.info("-- Starting testPauseAndResume test --"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 0c2d5fcc746a7..34ead65dff65b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -136,6 +136,7 @@ public class CompactionTest extends MockedPulsarServiceBaseTest { protected void doInitConf() throws Exception { super.doInitConf(); conf.setDispatcherMaxReadBatchSize(1); + conf.setForceDeleteNamespaceAllowed(true); } @BeforeClass @@ -2533,10 +2534,59 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { Thread.sleep(3000); delayReadSignal.countDown(); - // Verify: topic deletion is successfully executed. - Awaitility.await().atMost(15, TimeUnit.SECONDS).untilAsserted(() -> { - assertTrue(deleteTopicFuture.isDone()); - }); + // Verify: topic deletion is successfully executed. Asserting success (not just completion) covers the + // case where fencing the topic terminates the in-flight compaction exceptionally: the failed compaction + // must not fail the deletion (issue #24148). + deleteTopicFuture.get(15, TimeUnit.SECONDS); + } + + @Test + public void testForcedDeleteSucceedsAfterFailedCompaction() throws Exception { + String topicName = newUniqueName("persistent://my-tenant/my-ns/delete-after-failed-compaction"); + admin.topics().createNonPartitionedTopic(topicName); + try (Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create()) { + for (int i = 0; i < 10; i++) { + producer.newMessage().key("key" + (i % 2)).value("value-" + i).send(); + } + } + + // Fail the compaction after the phase-two seek + AbstractTwoPhaseCompactor.injectionPhaseTwoSeek = + (reader, id) -> CompletableFuture.failedFuture(new RuntimeException("injected compaction failure")); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + persistentTopic.triggerCompaction(); + Awaitility.await().untilAsserted(() -> + assertEquals(persistentTopic.compactionStatus().status, LongRunningProcessStatus.Status.ERROR)); + AbstractTwoPhaseCompactor.injectionPhaseTwoSeek = RawReader::seekAsync; + + // The failed compaction must not block the deletion of the compaction cursor + admin.topics().delete(topicName, true); + } + + @Test + public void testForcedNamespaceDeleteWithInflightCompaction() throws Exception { + String namespace = "my-tenant/my-ns-inflight-compaction"; + admin.namespaces().createNamespace(namespace, Set.of(configClusterName)); + final String topicName = newUniqueName("persistent://" + namespace + "/inflight-compaction"); + admin.topics().createNonPartitionedTopic(topicName); + try (Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create()) { + // dispatcherMaxReadBatchSize=1 makes the compactor read these one at a time, keeping the compaction + // in-flight for several seconds while the namespace is deleted + for (int i = 0; i < 2000; i++) { + producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i)).send(); + } + } + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + persistentTopic.triggerCompactionWithCheckHasMoreMessages().join(); + Awaitility.await().untilAsserted(() -> + assertEquals(persistentTopic.getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), + 1)); + + // Forced namespace deletion must succeed while the compaction is in-flight: fencing the topic terminates + // the compaction exceptionally, which must not fail the deletion of the compaction cursor (issue #24148) + deleteNamespaceWithRetry(namespace, true, admin); } @Test From 86ad58db7b9dbe7d5e97a395559d6c721ff158e9 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sun, 14 Jun 2026 14:25:57 +0300 Subject: [PATCH 149/201] [fix][broker] Fix forced topic/namespace deletion still hanging when the compaction reader reconnect stalls (#26026) (cherry picked from commit 2325d1eb17d090afc0bd5eaeaa5c2a2e6a4445cf) --- .../service/persistent/PersistentTopic.java | 20 ++++-- .../pulsar/client/impl/RawReaderImpl.java | 20 ++++++ .../pulsar/client/impl/RawReaderTest.java | 63 +++++++++++++++++++ .../pulsar/compaction/CompactionTest.java | 37 +++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 4 ++ 5 files changed, 138 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 5bc882b011189..2b078029bd24d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1374,12 +1374,20 @@ private void asyncDeleteCursorWithCleanCompactionLedger(PersistentSubscription s return; } } - // Unsubscribe compaction cursor and delete compacted ledger. Wait for any in-flight compaction to finish - // first, but don't let a compaction that completed exceptionally block the cursor deletion: the deletion - // would otherwise fail on every retry until the topic instance is reloaded (issue #24148). Note that a - // fenced topic makes the compactor's reader fail with an unrecoverable error, so a forced deletion - // terminates an in-flight compaction exceptionally rather than waiting for it to complete normally. - currentCompaction.exceptionally(compactionEx -> { + // Unsubscribe compaction cursor and delete compacted ledger. Normally we wait for any in-flight compaction + // to finish first, but a compaction that completed exceptionally must not block the cursor deletion: it + // would otherwise fail on every retry until the topic instance is reloaded (issue #24148). + // + // Moreover, when the topic is being closed or deleted it is already fenced and any in-flight compaction is + // being aborted. Waiting for that compaction to complete is both unnecessary and unsafe here: the + // compactor's reader is expected to fail once the topic is fenced, but that depends on how the client + // reconnect surfaces the failure (a retriable lookup-stage error keeps the reader reconnecting instead of + // failing the in-flight read), so the compaction future may stay pending for far longer than the deletion + // can wait. Proceed with the cursor deletion right away in that case so a forced topic/namespace deletion + // cannot hang (issue #24148). + CompletableFuture compactionToWait = + isClosingOrDeleting ? CompletableFuture.completedFuture(null) : currentCompaction; + compactionToWait.exceptionally(compactionEx -> { log.info("[{}][{}] Last compaction task failed, proceeding to delete the compaction cursor", topic, subscriptionName, compactionEx); return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index 1bc88ed0532da..940ec5dfe6011 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -41,6 +41,7 @@ import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -160,6 +161,25 @@ protected boolean isUnrecoverableError(Throwable t) { return super.isUnrecoverableError(t); } + @Override + public boolean connectionFailed(PulsarClientException exception) { + // A compaction reader is created with retryOnRecoverableErrors=false. When the topic is fenced or + // deleted, a reconnect can fail at the lookup/connection stage with a retriable error such as + // ServiceNotReadyException. The base ConsumerImpl.connectionFailed only consults isUnrecoverableError + // for non-retriable errors (or after the lookup deadline has passed), so for such a retriable error it + // would keep reconnecting and never fail the in-flight read, leaving the compaction future pending. + // Honor isUnrecoverableError here too so the reader is closed promptly and pending reads are failed, + // mirroring the subscribe-stage handling in ConsumerImpl.connectionOpened(). This matters for + // compaction: a never-failing read keeps the compaction future pending, which blocks forced + // topic/namespace deletion (issue #24148). + Throwable actError = FutureUtil.unwrapCompletionException(exception); + if (isUnrecoverableError(actError)) { + closeWhenReceivedUnrecoverableError(actError, null); + return false; + } + return super.connectionFailed(exception); + } + void tryCompletePending() { CompletableFuture future = null; RawMessageAndCnx messageAndCnx = null; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java index 9320e22e42e8b..a043f71923a6a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java @@ -51,6 +51,7 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; @@ -643,6 +644,68 @@ public void testReadNextAsyncCompletesAfterConsumerClosed() throws Exception { assertTrue(readFuture.isCompletedExceptionally()); } + @Test(timeOut = 30000) + public void testConnectionFailureTerminatesReadWhenNotRetryingRecoverableErrors() throws Exception { + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); + admin.topics().createNonPartitionedTopic(topic); + + // A compaction reader is created with retryOnRecoverableErrors=false. When the topic is fenced or deleted, + // a reconnect can fail at the lookup/connection stage (handled by ConsumerImpl.connectionFailed) with a + // retriable error such as ServiceNotReadyException. The base class would keep reconnecting, leaving the + // in-flight read pending forever; for a compaction reader that keeps the compaction future pending and + // blocks forced topic/namespace deletion (issue #24148). Such an unrecoverable error must instead + // terminate the reader and fail the in-flight read promptly. + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.getTopicNames().add(topic); + conf.setSubscriptionName(subscription); + conf.setSubscriptionType(SubscriptionType.Exclusive); + conf.setReceiverQueueSize(DEFAULT_RECEIVER_QUEUE_SIZE); + conf.setSubscriptionInitialPosition(SubscriptionInitialPosition.Earliest); + conf.setReadCompacted(true); + CompletableFuture> consumerFuture = new CompletableFuture<>(); + RawReaderImpl.RawConsumerImpl consumer = new RawReaderImpl.RawConsumerImpl( + (PulsarClientImpl) pulsarClient, conf, consumerFuture, false, false); + consumerFuture.get(10, TimeUnit.SECONDS); + + CompletableFuture readFuture = consumer.receiveRawAsync(); + boolean keepReconnecting = + consumer.connectionFailed(new PulsarClientException.ServiceNotReadyException("injected")); + + Assert.assertFalse(keepReconnecting); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> assertTrue(readFuture.isDone())); + assertTrue(readFuture.isCompletedExceptionally()); + } + + @Test(timeOut = 30000) + public void testConnectionFailureBeforeSubscribeFailsReaderCreation() throws Exception { + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); + admin.topics().createNonPartitionedTopic(topic); + + // Same compaction-reader scenario as the test above, but the unrecoverable error (e.g. a lookup + // failure with ServiceNotReadyException) arrives BEFORE the initial subscribe completes. The + // subscribe (consumer) future must then be completed exceptionally; otherwise RawReader.create(...) + // would stay pending forever, which keeps the compaction future pending and blocks forced + // topic/namespace deletion (issue #24148). + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.getTopicNames().add(topic); + conf.setSubscriptionName(subscription); + conf.setSubscriptionType(SubscriptionType.Exclusive); + conf.setReceiverQueueSize(DEFAULT_RECEIVER_QUEUE_SIZE); + conf.setSubscriptionInitialPosition(SubscriptionInitialPosition.Earliest); + conf.setReadCompacted(true); + CompletableFuture> consumerFuture = new CompletableFuture<>(); + RawReaderImpl.RawConsumerImpl consumer = new RawReaderImpl.RawConsumerImpl( + (PulsarClientImpl) pulsarClient, conf, consumerFuture, false, false); + // Inject the failure without waiting for the subscribe to complete, i.e. while the consumer + // future is still pending (construction returns before the async subscribe round-trip finishes). + boolean keepReconnecting = + consumer.connectionFailed(new PulsarClientException.ServiceNotReadyException("injected")); + + Assert.assertFalse(keepReconnecting); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> assertTrue(consumerFuture.isDone())); + assertTrue(consumerFuture.isCompletedExceptionally()); + } + @Test(timeOut = 100000) public void testPauseAndResume() throws Exception { log.info("-- Starting testPauseAndResume test --"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 34ead65dff65b..ae901426f243a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -2540,6 +2540,43 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { deleteTopicFuture.get(15, TimeUnit.SECONDS); } + @Test(timeOut = 60 * 1000) + public void testForcedDeleteCompletesWhileCompactionStuck() throws Exception { + final String topicName = newUniqueName("persistent://my-tenant/my-ns/forced-delete-stuck-compaction"); + admin.topics().createNonPartitionedTopic(topicName); + try (Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create()) { + for (int i = 0; i < 10; i++) { + producer.newMessage().key("key" + (i % 2)).value("value-" + i).send(); + } + } + + // Block the compaction at the phase-two seek so its future never completes on its own. This reproduces an + // in-flight compaction whose reader does not fail promptly when the topic is fenced (e.g. because a + // reconnect keeps retrying a retriable lookup-stage error): the compaction future stays pending (issue + // #24148). + CompletableFuture blockedSeek = new CompletableFuture<>(); + CountDownLatch reachedSeek = new CountDownLatch(1); + AbstractTwoPhaseCompactor.injectionPhaseTwoSeek = (reader, id) -> { + reachedSeek.countDown(); + return blockedSeek; + }; + try { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + persistentTopic.triggerCompaction(); + assertTrue(reachedSeek.await(30, TimeUnit.SECONDS)); + assertEquals(persistentTopic.compactionStatus().status, LongRunningProcessStatus.Status.RUNNING); + + // The compaction future is stuck, but a forced deletion must complete promptly instead of waiting for + // the in-flight compaction to finish (issue #24148). + persistentTopic.deleteForcefully().get(15, TimeUnit.SECONDS); + } finally { + AbstractTwoPhaseCompactor.injectionPhaseTwoSeek = RawReader::seekAsync; + // Unblock the stuck compaction so it can unwind and release its reader. + blockedSeek.complete(null); + } + } + @Test public void testForcedDeleteSucceedsAfterFailedCompaction() throws Exception { String topicName = newUniqueName("persistent://my-tenant/my-ns/delete-after-failed-compaction"); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 5c1d94992262e..54ef0db71aa0c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1029,6 +1029,10 @@ protected void closeWhenReceivedUnrecoverableError(Throwable t, ClientCnx cnx) { final String cnxStr = cnx == null ? "null" : String.valueOf(cnx.channel().remoteAddress()); log.warn("[{}][{}] {} Closed consumer because get an error that does not support to retry: {} {}", topic, subscription, cnxStr, t.getClass().getName(), t.getMessage()); + // If the unrecoverable error occurs before the initial subscribe completes, fail the subscribe + // future as well; otherwise callers waiting on it (e.g. RawReader.create() / subscribeAsync()) + // would hang forever. This is a no-op when the subscribe future has already completed. + subscribeFuture.completeExceptionally(t); closeAsync().whenComplete((__, ex) -> { if (ex == null) { fail(t); From e2f14fda9e0cc8360ce92a48b47331f983c2f6b1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sun, 14 Jun 2026 16:22:58 +0300 Subject: [PATCH 150/201] [fix][broker] Don't let a stuck or aborted topic policies cache init make a namespace's topics unloadable (#26025) (cherry picked from commit 24f2270ec21906608429d05f86ec7347e9bf0edf) --- conf/broker.conf | 7 + conf/standalone.conf | 7 + .../pulsar/broker/ServiceConfiguration.java | 13 ++ .../SystemTopicBasedTopicPoliciesService.java | 135 +++++++++++++++++- ...temTopicBasedTopicPoliciesServiceTest.java | 121 ++++++++++++++-- 5 files changed, 271 insertions(+), 12 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 0447b3e05acd4..066138ef98db9 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -768,6 +768,13 @@ systemTopicSchemaCompatibilityStrategy=ALWAYS_COMPATIBLE # Please enable the system topic first. topicLevelPoliciesEnabled=true +# Amount of seconds to timeout initializing the topic policies cache of a namespace (reading the namespace's +# __change_events system topic to the end). Topic loading waits for this initialization, so if the system-topic +# reader gets stuck, this bounds the wait: the broker fails the initialization, closes the stuck reader and clears +# the cached state so loading the namespace's topics can be retried with a fresh reader instead of hanging until the +# broker is restarted. Set to 0 or a negative value to disable the timeout (not recommended). +topicPoliciesCacheInitTimeoutSeconds=60 + # If a topic remains fenced for this number of seconds, it will be closed forcefully. # If it is set to 0 or a negative number, the fenced topic will not be closed. topicFencingTimeoutSeconds=0 diff --git a/conf/standalone.conf b/conf/standalone.conf index 15038c6aa4f22..c1c3289fa7d1e 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -579,6 +579,13 @@ systemTopicSchemaCompatibilityStrategy=ALWAYS_COMPATIBLE # Please enable the system topic first. topicLevelPoliciesEnabled=true +# Amount of seconds to timeout initializing the topic policies cache of a namespace (reading the namespace's +# __change_events system topic to the end). Topic loading waits for this initialization, so if the system-topic +# reader gets stuck, this bounds the wait: the broker fails the initialization, closes the stuck reader and clears +# the cached state so loading the namespace's topics can be retried with a fresh reader instead of hanging until the +# broker is restarted. Set to 0 or a negative value to disable the timeout (not recommended). +topicPoliciesCacheInitTimeoutSeconds=60 + # If a topic remains fenced for this number of seconds, it will be closed forcefully. # If it is set to 0 or a negative number, the fenced topic will not be closed. topicFencingTimeoutSeconds=0 diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index bf6ef7730e520..5e977b5b09f28 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -570,6 +570,19 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece ) private long topicLoadTimeoutSeconds = 60; + @FieldContext( + category = CATEGORY_SERVER, + dynamic = true, + doc = "Amount of seconds to timeout initializing the topic policies cache of a namespace (reading the " + + "namespace's __change_events system topic to the end). Topic loading waits for this " + + "initialization, so if the system-topic reader gets stuck (for example after __change_events is " + + "unloaded and the reconnected reader stops making progress), this bounds the wait: the broker " + + "fails the initialization, closes the stuck reader and clears the cached state so that loading " + + "the namespace's topics can be retried with a fresh reader instead of hanging until the broker " + + "is restarted. Set to 0 or a negative value to disable the timeout (not recommended)." + ) + private long topicPoliciesCacheInitTimeoutSeconds = 60; + @FieldContext( category = CATEGORY_SERVER, doc = "Whether we should enable metadata operations batching" diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index f8a5d59366104..7e1b453b35b88 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -24,6 +24,7 @@ import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; +import io.opentelemetry.api.metrics.LongCounter; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -34,7 +35,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -78,10 +81,14 @@ */ public class SystemTopicBasedTopicPoliciesService implements TopicPoliciesService { + public static final String TOPIC_POLICIES_CACHE_INIT_TIMEOUT_METRIC_NAME = + "pulsar.broker.topic.policies.cache.init.timeout.count"; + private final PulsarService pulsarService; private final HashSet localCluster; private final String clusterName; private final AtomicBoolean closed = new AtomicBoolean(false); + private final LongCounter policyCacheInitTimeoutCounter; private final ConcurrentInitializer namespaceEventsSystemTopicFactoryLazyInitializer = new LazyInitializer<>() { @@ -125,6 +132,13 @@ public SystemTopicBasedTopicPoliciesService(PulsarService pulsarService) { this.pulsarService = pulsarService; this.clusterName = pulsarService.getConfiguration().getClusterName(); this.localCluster = Sets.newHashSet(clusterName); + this.policyCacheInitTimeoutCounter = pulsarService.getOpenTelemetry().getMeter() + .counterBuilder(TOPIC_POLICIES_CACHE_INIT_TIMEOUT_METRIC_NAME) + .setDescription("The number of times initializing a namespace's topic policies cache timed out " + + "because the __change_events system-topic reader was stuck. Each occurrence closes the " + + "stuck reader and clears the cached state so topic loading can be retried.") + .setUnit("{timeout}") + .build(); this.writerCaches = Caffeine.newBuilder() .expireAfterAccess(5, TimeUnit.MINUTES) .removalListener((namespaceName, writer, cause) -> { @@ -583,6 +597,12 @@ public void addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { CompletableFuture existingFuture = policyCacheInitMap.putIfAbsent(namespace, initNamespacePolicyFuture); if (existingFuture == null) { + // Topic loading waits on this future, so a system-topic reader that gets stuck (e.g. after + // __change_events is unloaded and the reconnected reader stops making progress) would pin the + // policy cache for the whole namespace until the broker restarts (issue #25294). Bound the + // initialization so it fails fast and cleans up, letting topic loading retry with a fresh + // reader. + scheduleInitPolicesCacheTimeout(namespace, initNamespacePolicyFuture); final CompletableFuture> readerCompletableFuture = newReader(namespace); readerCompletableFuture @@ -597,14 +617,19 @@ public void addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { return null; }).exceptionally(ex -> { try { + // Identity-guarded cleanup: a concurrent timeout cleanup or a namespace unload + // may already have dropped this future and let a retry install a fresh + // future/reader, so clean up by namespace key here would clobber that newer + // attempt. Tear down state only while this future still owns the namespace. if (readerCompletableFuture.isCompletedExceptionally()) { log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); initNamespacePolicyFuture.completeExceptionally(ex); - cleanPoliciesCacheInitMap(namespace, true); + cleanupFailedPolicyCacheInit(namespace, initNamespacePolicyFuture, true); } else { initNamespacePolicyFuture.completeExceptionally(ex); - cleanPoliciesCacheInitMap(namespace, isAlreadyClosedException(ex)); + cleanupFailedPolicyCacheInit(namespace, initNamespacePolicyFuture, + isAlreadyClosedException(ex)); } } catch (Throwable cleanupEx) { // Adding this catch to avoid break callback chain @@ -685,6 +710,86 @@ public boolean test(NamespaceBundle namespaceBundle) { }); } + /** + * Bound the topic-policies cache initialization for {@code namespace} so a stuck {@code __change_events} reader + * cannot pin the cache (and therefore topic loading) for the whole namespace indefinitely (issue #25294). If the + * timeout wins the race to complete {@code initNamespacePolicyFuture}, the cached state is cleared and the stuck + * reader is closed so a subsequent load retries from scratch rather than waiting until the broker restarts. + */ + private void scheduleInitPolicesCacheTimeout(@NonNull NamespaceName namespace, + @NonNull CompletableFuture initNamespacePolicyFuture) { + long timeoutSeconds = pulsarService.getConfiguration().getTopicPoliciesCacheInitTimeoutSeconds(); + if (timeoutSeconds <= 0) { + return; + } + final ScheduledFuture timeoutTask = pulsarService.getExecutor().schedule(() -> { + TimeoutException timeoutException = new TimeoutException(String.format( + "Timed out after %d seconds initializing the topic policies cache for namespace %s; the " + + "__change_events reader did not reach the end of the topic", timeoutSeconds, namespace)); + if (initNamespacePolicyFuture.completeExceptionally(timeoutException)) { + policyCacheInitTimeoutCounter.add(1); + log.error("[{}] Timed out initializing the topic policies cache after {} seconds; closing the stuck " + + "__change_events reader so the namespace can be loaded again", namespace, timeoutSeconds); + try { + cleanupFailedPolicyCacheInit(namespace, initNamespacePolicyFuture, true); + } catch (Throwable cleanupEx) { + log.error("[{}] Failed to clean up the topic policies cache after init timeout", namespace, + cleanupEx); + } + } + }, timeoutSeconds, TimeUnit.SECONDS); + // Cancel the timeout once initialization finishes (successfully or not) so we don't leak scheduled tasks. + initNamespacePolicyFuture.whenComplete((__, ex) -> timeoutTask.cancel(false)); + } + + /** + * Identity-guarded cleanup for an initialization that failed (it timed out, the {@code __change_events} reader + * could not be created, or reading the topic threw). Unlike {@link #cleanPoliciesCacheInitMap}, which + * removes/closes by namespace key unconditionally, this only tears down state that still belongs to + * {@code initFuture}. By the time the failure is observed, a concurrent retry — or a namespace-bundle unload that + * left the init future orphaned — may already own the namespace with a fresh future and reader; removing by key + * would drop that newer future and close its reader, pinning the namespace again. Guarding on identity ensures a + * late failure never clobbers a newer initialization. + * + * @param closeReader when {@code true}, also clears the cached policies and closes the reader that belongs to this + * initialization; when {@code false}, only the init future is dropped, leaving the reader cached + * for the retry to reuse (mirrors the transient read-error path of + * {@link #cleanPoliciesCacheInitMap}). + */ + @VisibleForTesting + void cleanupFailedPolicyCacheInit(@NonNull NamespaceName namespace, + @NonNull CompletableFuture initFuture, boolean closeReader) { + // Capture the reader before dropping the init future so we only close the reader that belongs to this + // initialization, never one a concurrent retry creates immediately afterwards. + CompletableFuture> readerFuture = + closeReader ? readerCaches.get(namespace) : null; + if (!policyCacheInitMap.remove(namespace, initFuture)) { + // Superseded by a retry or an unload; that owner is responsible for its own reader/state. + return; + } + // Complete the dropped future (a no-op if the caller already completed it) outside any map remapping function, + // so awaiting topic loads fail fast and retry instead of hanging until the broker restarts (issue #25294). + failPendingPolicyCacheInit(namespace, initFuture); + if (!closeReader) { + return; + } + policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); + globalPoliciesCache.entrySet() + .removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); + TopicPolicyMessageHandlerTracker tracker = topicPolicyMessageHandlerTrackers.remove(namespace); + if (tracker != null) { + tracker.close(); + } + if (readerFuture != null && readerCaches.remove(namespace, readerFuture) + && !readerFuture.isCompletedExceptionally()) { + readerFuture.thenCompose(SystemTopicClient.Reader::closeAsync) + .exceptionally(ex -> { + log.warn("[{}] Close change_event reader fail.", namespace, ex); + return null; + }); + } + } + private void initPolicesCache(SystemTopicClient.Reader reader, CompletableFuture future) { if (closed.get()) { future.completeExceptionally(new BrokerServiceException(getClass().getName() + " is closed.")); @@ -742,7 +847,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp @VisibleForTesting void cleanPoliciesCacheInitMap(@NonNull NamespaceName namespace, boolean closeReader) { if (!closeReader) { - policyCacheInitMap.remove(namespace); + failPendingPolicyCacheInit(namespace, policyCacheInitMap.remove(namespace)); return; } @@ -753,12 +858,18 @@ void cleanPoliciesCacheInitMap(@NonNull NamespaceName namespace, boolean closeRe } CompletableFuture> readerFuture = readerCaches.remove(namespace); + MutableObject> removedInitFuture = new MutableObject<>(); policyCacheInitMap.compute(namespace, (k, v) -> { + removedInitFuture.setValue(v); policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); globalPoliciesCache.entrySet() .removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); return null; }); + // Complete the removed init future outside the compute() remapping function: completing it can run the + // awaiting topic-load callbacks synchronously, and doing that while holding the ConcurrentHashMap bin lock + // risks a recursive map update / deadlock (see #24977). + failPendingPolicyCacheInit(namespace, removedInitFuture.getValue()); if (readerFuture != null && !readerFuture.isCompletedExceptionally()) { readerFuture .thenCompose(SystemTopicClient.Reader::closeAsync) @@ -769,6 +880,20 @@ void cleanPoliciesCacheInitMap(@NonNull NamespaceName namespace, boolean closeRe } } + /** + * Complete an init future that is being dropped from {@link #policyCacheInitMap} but never completed, so the + * topic loads awaiting it fail fast and retry instead of hanging until the broker restarts (issue #25294). + * No-op if the future was already completed by its own initialization chain. + */ + private void failPendingPolicyCacheInit(@NonNull NamespaceName namespace, + @Nullable CompletableFuture initFuture) { + if (initFuture != null && !initFuture.isDone()) { + initFuture.completeExceptionally(new BrokerServiceException( + "Topic policies cache initialization for namespace " + namespace + + " was aborted because the cached state was cleared")); + } + } + private void cleanWriterCache(@NonNull NamespaceName namespace) { writerCaches.synchronous().invalidate(namespace); } @@ -1033,6 +1158,10 @@ public void close() throws Exception { } }); readerCaches.clear(); + // Release any topic loads still waiting on an in-progress policy cache initialization so they fail + // fast instead of hanging until the awaited future is completed indirectly (issue #25294). + policyCacheInitMap.forEach(this::failPendingPolicyCacheInit); + policyCacheInitMap.clear(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index a0fdf039294bd..467a5e42d20f4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -27,6 +27,7 @@ import static org.testng.AssertJUnit.assertFalse; import static org.testng.AssertJUnit.assertNotNull; import static org.testng.AssertJUnit.assertNull; +import static org.testng.AssertJUnit.assertSame; import static org.testng.AssertJUnit.assertTrue; import java.util.HashSet; import java.util.List; @@ -39,12 +40,15 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.naming.NamespaceName; @@ -550,17 +554,18 @@ public void testPrepareInitPoliciesCacheAsyncThrowExceptionAfterCreateReader() t Assert.assertNull(readerCompletableFuture1); }); - - // make sure not do cleanPoliciesCacheInitMap() twice - // totally trigger prepareInitPoliciesCacheAsync() twice, so the time of cleanPoliciesCacheInitMap() is 2. - // in previous code, the time would be 3 + // Cleanup must run exactly once per trigger and not repeat recursively (in older code it ran 3 times). + // Two failures are triggered here: the reader.close() above drives readMorePoliciesAsync into + // cleanPoliciesCacheInitMap (1x), and the second prepareInitPoliciesCacheAsync fails in initPolicesCache and + // is torn down by the identity-guarded cleanupFailedPolicyCacheInit (1x). boolean logFound = testLogAppender.getEvents().stream().anyMatch(logEvent -> logEvent.getMessage().toString().contains("Failed to create reader on __change_events topic")); assertFalse(logFound); boolean logFound2 = testLogAppender.getEvents().stream().anyMatch(logEvent -> logEvent.getMessage().toString().contains("Failed to check the move events for the system topic")); assertTrue(logFound2); - verify(spyService, times(2)).cleanPoliciesCacheInitMap(any(), anyBoolean()); + verify(spyService, times(1)).cleanPoliciesCacheInitMap(any(), anyBoolean()); + verify(spyService, times(1)).cleanupFailedPolicyCacheInit(any(), any(), anyBoolean()); // make sure not occur Recursive update boolean logFound3 = testLogAppender.getEvents().stream().anyMatch(logEvent -> @@ -619,9 +624,9 @@ public void testPrepareInitPoliciesCacheAsyncThrowExceptionInCreateReader() thro Assert.assertNull(readerCompletableFuture1); }); - - // make sure not do cleanPoliciesCacheInitMap() twice - // totally trigger prepareInitPoliciesCacheAsync() once, so the time of cleanPoliciesCacheInitMap() is 1. + // Reader creation fails, so the single cleanup runs once via the identity-guarded + // cleanupFailedPolicyCacheInit (the reader-creation-failure branch no longer goes through + // cleanPoliciesCacheInitMap), and must not run more than once. boolean logFound = testLogAppender.getEvents().stream().anyMatch(logEvent -> logEvent.getMessage().toString().contains("Failed to create reader on __change_events topic")); assertTrue(logFound); @@ -629,6 +634,104 @@ public void testPrepareInitPoliciesCacheAsyncThrowExceptionInCreateReader() thro logEvent.getMessage().toString().contains("Failed to check the move events for the system topic") || logEvent.getMessage().toString().contains("Failed to read event from the system topic")); assertFalse(logFound2); - verify(spyService, times(1)).cleanPoliciesCacheInitMap(any(), anyBoolean()); + verify(spyService, times(1)).cleanupFailedPolicyCacheInit(any(), any(), anyBoolean()); + verify(spyService, times(0)).cleanPoliciesCacheInitMap(any(), anyBoolean()); + } + + @Test(timeOut = 60_000) + public void testPrepareInitPoliciesCacheAsyncTimesOutWhenReaderStuck() throws Exception { + // Bound the policy-cache initialization to a short timeout for the test. + pulsar.getConfiguration().setTopicPoliciesCacheInitTimeoutSeconds(3); + + pulsar.getTopicPoliciesService().close(); + SystemTopicBasedTopicPoliciesService spyService = + Mockito.spy(new SystemTopicBasedTopicPoliciesService(pulsar)); + FieldUtils.writeField(pulsar, "topicPoliciesService", spyService, true); + + admin.namespaces().createNamespace(NAMESPACE5); + final NamespaceName namespace = NamespaceName.get(NAMESPACE5); + + // Create a real __change_events reader, then spy it so that it reports more events but never delivers one — + // i.e. a reader that reconnected but is stuck (issue #25294). initPolicesCache would otherwise never complete. + SystemTopicClient.Reader stuckReader = Mockito.spy(spyService.createSystemTopicClient(namespace) + .get(30, TimeUnit.SECONDS)); + Mockito.doReturn(CompletableFuture.completedFuture(true)).when(stuckReader).hasMoreEventsAsync(); + Mockito.doReturn(new CompletableFuture>()).when(stuckReader).readNextAsync(); + Mockito.doReturn(CompletableFuture.completedFuture(stuckReader)) + .when(spyService).createSystemTopicClient(namespace); + + // Without the timeout the returned future never completes and topic loading for the namespace hangs forever. + CompletableFuture prepareFuture = spyService.prepareInitPoliciesCacheAsync(namespace); + try { + prepareFuture.get(15, TimeUnit.SECONDS); + Assert.fail("Expected the topic policies cache initialization to time out"); + } catch (ExecutionException e) { + assertTrue("Expected a TimeoutException cause but got " + e.getCause(), + e.getCause() instanceof TimeoutException); + } + + // The poisoned cache entry must be cleared and the stuck reader closed so a subsequent load can retry with a + // fresh reader instead of being pinned until the broker restarts. + Awaitility.await().untilAsserted(() -> assertNull(spyService.getPoliciesCacheInit(namespace))); + Mockito.verify(stuckReader, Mockito.atLeastOnce()).closeAsync(); + } + + @Test + public void testCleanPoliciesCacheInitMapCompletesPendingInitFuture() { + SystemTopicBasedTopicPoliciesService service = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final NamespaceName namespace = NamespaceName.get(NAMESPACE1); + + // Dropping the cached init future (e.g. on a namespace-bundle unload) must complete it so the topic loads + // awaiting it fail fast and retry, instead of hanging until the broker restarts (issue #25294). + CompletableFuture pendingWithReaderClose = new CompletableFuture<>(); + service.policyCacheInitMap.put(namespace, pendingWithReaderClose); + service.cleanPoliciesCacheInitMap(namespace, true); + assertTrue(pendingWithReaderClose.isCompletedExceptionally()); + assertNull(service.getPoliciesCacheInit(namespace)); + + CompletableFuture pendingWithoutReaderClose = new CompletableFuture<>(); + service.policyCacheInitMap.put(namespace, pendingWithoutReaderClose); + service.cleanPoliciesCacheInitMap(namespace, false); + assertTrue(pendingWithoutReaderClose.isCompletedExceptionally()); + assertNull(service.getPoliciesCacheInit(namespace)); + + // An already-completed init future must not be overwritten/disturbed. + CompletableFuture alreadyDone = CompletableFuture.completedFuture(null); + service.policyCacheInitMap.put(namespace, alreadyDone); + service.cleanPoliciesCacheInitMap(namespace, true); + assertFalse(alreadyDone.isCompletedExceptionally()); + } + + @Test + @SuppressWarnings("unchecked") + public void testCleanupFailedPolicyCacheInitIsIdentityGuarded() { + SystemTopicBasedTopicPoliciesService service = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final NamespaceName namespace = NamespaceName.get(NAMESPACE1); + + // A newer init attempt (B) already owns the namespace with a fresh future and reader. + CompletableFuture newerInitFuture = new CompletableFuture<>(); + SystemTopicClient.Reader newerReader = Mockito.mock(SystemTopicClient.Reader.class); + Mockito.doReturn(CompletableFuture.completedFuture(null)).when(newerReader).closeAsync(); + service.policyCacheInitMap.put(namespace, newerInitFuture); + service.getReaderCaches().put(namespace, CompletableFuture.completedFuture(newerReader)); + + // A stale init attempt (A) whose reader was already torn down (e.g. by the timeout cleanup) fires its failure + // callback late. Cleaning it up by identity must be a no-op: it must not clobber B's future or close B's reader + // (issue #25294 follow-up). + CompletableFuture staleInitFuture = new CompletableFuture<>(); + service.cleanupFailedPolicyCacheInit(namespace, staleInitFuture, true); + assertSame(newerInitFuture, service.getPoliciesCacheInit(namespace)); + assertFalse(newerInitFuture.isDone()); + assertNotNull(service.getReaderCaches().get(namespace)); + Mockito.verify(newerReader, Mockito.never()).closeAsync(); + + // Cleaning up the owning attempt does drop its future and close its reader. + service.cleanupFailedPolicyCacheInit(namespace, newerInitFuture, true); + assertNull(service.getPoliciesCacheInit(namespace)); + assertTrue(newerInitFuture.isCompletedExceptionally()); + assertNull(service.getReaderCaches().get(namespace)); + Mockito.verify(newerReader, Mockito.times(1)).closeAsync(); } } From 49ead2f7fbfee0f987b1013e71c39c328372ce79 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 15 Jun 2026 18:59:02 +0800 Subject: [PATCH 151/201] [fix][broker] Fail fast for load balancer misconfigurations instead of falling back to SimpleLoadManagerImpl (#26031) (cherry picked from commit c3f8c05e4f27b194d889cbe3eefc4d83548628aa) --- .../broker/loadbalance/LoadManager.java | 52 +++++++++---------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java index 9399fcb3aad7a..c9aa673d6b53d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java @@ -151,35 +151,31 @@ default void writeLoadReportOnZookeeper(boolean force) throws Exception { void initialize(PulsarService pulsar); static LoadManager create(final PulsarService pulsar) { - try { - final ServiceConfiguration conf = pulsar.getConfiguration(); - - String loadManagerClassName = conf.getLoadManagerClassName(); - if (StringUtils.isBlank(loadManagerClassName)) { - loadManagerClassName = SimpleLoadManagerImpl.class.getName(); - } - - // Assume there is a constructor with one argument of PulsarService. - final Object loadManagerInstance = Reflections.createInstance(loadManagerClassName, - Thread.currentThread().getContextClassLoader()); - if (loadManagerInstance instanceof LoadManager casted) { - casted.initialize(pulsar); - return casted; - } else if (loadManagerInstance instanceof ModularLoadManager modularLoadManager) { - final LoadManager casted = new ModularLoadManagerWrapper(modularLoadManager); - casted.initialize(pulsar); - return casted; - } else if (loadManagerInstance instanceof ExtensibleLoadManager) { - final LoadManager casted = - new ExtensibleLoadManagerWrapper((ExtensibleLoadManagerImpl) loadManagerInstance); - casted.initialize(pulsar); - return casted; - } - } catch (Exception e) { - LOG.warn("Error when trying to create load manager: ", e); + final ServiceConfiguration conf = pulsar.getConfiguration(); + + String loadManagerClassName = conf.getLoadManagerClassName(); + if (StringUtils.isBlank(loadManagerClassName)) { + loadManagerClassName = SimpleLoadManagerImpl.class.getName(); + } + + // Assume there is a constructor with one argument of PulsarService. + final Object loadManagerInstance = Reflections.createInstance(loadManagerClassName, + Thread.currentThread().getContextClassLoader()); + if (loadManagerInstance instanceof LoadManager casted) { + casted.initialize(pulsar); + return casted; + } else if (loadManagerInstance instanceof ModularLoadManager modularLoadManager) { + final LoadManager casted = new ModularLoadManagerWrapper(modularLoadManager); + casted.initialize(pulsar); + return casted; + } else if (loadManagerInstance instanceof ExtensibleLoadManager) { + final LoadManager casted = + new ExtensibleLoadManagerWrapper((ExtensibleLoadManagerImpl) loadManagerInstance); + casted.initialize(pulsar); + return casted; + } else { + throw new IllegalArgumentException(loadManagerClassName + " is not a supported LoadManager"); } - // If we failed to create a load manager, default to SimpleLoadManagerImpl. - return new SimpleLoadManagerImpl(pulsar); } } From 3d060848930c93f4324d43b52b3ac4b2d9911449 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 16 Jun 2026 11:23:00 +0300 Subject: [PATCH 152/201] [improve][fn] Upgrade pulsar-client-python to 3.12.0 (#26033) (cherry picked from commit cd7738b4b62c84fd60b2228bdda4956db29f244f) --- docker/pulsar/Dockerfile | 2 +- docker/pulsar/Dockerfile.wolfi | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 9a9738d221283..8c0730e00e628 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -112,7 +112,7 @@ ARG PULSAR_CLIENT_PYTHON_VERSION RUN pip3 install --break-system-packages --no-cache-dir \ --only-binary \ grpcio==1.78.0 \ - protobuf==6.33.5 \ + protobuf==6.33.6 \ pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION} \ kazoo diff --git a/docker/pulsar/Dockerfile.wolfi b/docker/pulsar/Dockerfile.wolfi index 300c468b55eee..87e2488981be7 100644 --- a/docker/pulsar/Dockerfile.wolfi +++ b/docker/pulsar/Dockerfile.wolfi @@ -98,7 +98,7 @@ ARG PULSAR_CLIENT_PYTHON_VERSION RUN pip3 install --break-system-packages --no-cache-dir \ --only-binary \ grpcio==1.78.0 \ - protobuf==6.33.5 \ + protobuf==6.33.6 \ pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION} \ kazoo \ pyyaml diff --git a/pom.xml b/pom.xml index ef369c215cddf..84967564b449d 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ flexible messaging model and an intuitive client API. ${maven.compiler.target} 17 - 3.10.0 + 3.12.0 21 From d92018ed570eef6a1604a482f1b27c5b802782e9 Mon Sep 17 00:00:00 2001 From: SongOf <46475785+SongOf@users.noreply.github.com> Date: Thu, 18 Jun 2026 06:41:18 +0800 Subject: [PATCH 153/201] [fix][fn] Make exclusiveLeaderProducer volatile in FunctionMetaDataManager (#26046) Co-authored-by: maxlisongsong (cherry picked from commit b8b1e2c57bd7145fc77ac3c5232957490aee9b77) --- .../apache/pulsar/functions/worker/FunctionMetaDataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java index 944b4e8e34491..92680c60c3e79 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java @@ -72,7 +72,7 @@ public class FunctionMetaDataManager implements AutoCloseable { // The producer of the metadata topic when we are the leader. // Note that this variable serves a double duty. A non-null value // implies we are the leader, while a null value means we are not the leader - private Producer exclusiveLeaderProducer; + private volatile Producer exclusiveLeaderProducer; @Getter private volatile MessageId lastMessageSeen = MessageId.earliest; From 3df6f52a16c46594d5de9e1acec5d23c1b6df79a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 18 Jun 2026 01:57:06 +0300 Subject: [PATCH 154/201] [fix][broker] Run topic policy notifications on the topic-ordered executor (#26042) (cherry picked from commit a9278c2c4c26deb0d2a1f6bb02bd0001e65c13a4) --- .../pulsar/broker/service/AbstractTopic.java | 11 +++ .../pulsar/broker/service/BrokerService.java | 45 ++++++++- .../SystemTopicBasedTopicPoliciesService.java | 93 ++++++++++++------- .../nonpersistent/NonPersistentTopic.java | 4 +- .../service/persistent/PersistentTopic.java | 7 +- ...temTopicBasedTopicPoliciesServiceTest.java | 30 ++++++ 6 files changed, 148 insertions(+), 42 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 9f6f2e2ae3826..1ab26994f81ee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -43,6 +43,7 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLongFieldUpdater; @@ -185,10 +186,16 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener { protected final Clock clock; protected Set additionalSystemCursorNames = new TreeSet<>(); + private final ExecutorService topicPoliciesNotifyThread; public AbstractTopic(String topic, BrokerService brokerService) { this.topic = topic; this.namespace = TopicName.get(topic).getNamespaceObject(); + // Pin the per-topic policies-notify thread once. BrokerService#getTopicPoliciesNotifyThread centralizes + // the topic-to-thread mapping so it stays consistent with SystemTopicBasedTopicPoliciesService. In unit + // tests that construct topics with a mock BrokerService this returns null (the thread is unused there). + this.topicPoliciesNotifyThread = + brokerService.getTopicPoliciesNotifyThread(TopicName.getPartitionedTopicName(topic)); this.clock = brokerService.getClock(); this.brokerService = brokerService; this.producers = new ConcurrentHashMap<>(); @@ -1451,4 +1458,8 @@ private static Map getNonPartitionedPropertiesAsync(PulsarServic return Collections.emptyMap(); } } + + protected ExecutorService getPoliciesNotifyThread() { + return topicPoliciesNotifyThread; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index d21477f000d50..0664d9ff290b8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -67,6 +67,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.Semaphore; @@ -2666,9 +2667,10 @@ private void handleLocalPoliciesUpdates(NamespaceName namespace) { log.debug("Notifying topic that local policies have changed: {}", name); } topic.ifPresent(t -> { - if (t instanceof PersistentTopic) { - PersistentTopic topic1 = (PersistentTopic) t; - topic1.onLocalPoliciesUpdate(); + if (t instanceof PersistentTopic persistentTopic) { + runOnTopicPoliciesNotifyThread(t, () -> { + persistentTopic.onLocalPoliciesUpdate(); + }); } }); }); @@ -2688,7 +2690,8 @@ private void handlePoliciesUpdates(NamespaceName namespace) { log.info("[{}] updating with {}", namespace, policies); topics.forEach((name, topicFuture) -> { - if (namespace.includes(TopicName.get(name))) { + TopicName topicName = TopicName.get(name); + if (namespace.includes(topicName)) { // If the topic is already created, immediately apply the updated policies, otherwise // once the topic is created it'll apply the policies update topicFuture.thenAccept(topic -> { @@ -2696,7 +2699,11 @@ private void handlePoliciesUpdates(NamespaceName namespace) { log.debug("Notifying topic that policies have changed: {}", name); } - topic.ifPresent(t -> t.onPoliciesUpdate(policies)); + topic.ifPresent(t -> { + runOnTopicPoliciesNotifyThread(t, () -> { + t.onPoliciesUpdate(policies); + }); + }); }); } }); @@ -2707,6 +2714,16 @@ private void handlePoliciesUpdates(NamespaceName namespace) { }, pulsar.getExecutor()); } + private void runOnTopicPoliciesNotifyThread(Topic t, Runnable runnable) { + ExecutorService policiesNotifyThread; + if (t instanceof AbstractTopic abstractTopic) { + policiesNotifyThread = abstractTopic.getPoliciesNotifyThread(); + } else { + policiesNotifyThread = getTopicPoliciesNotifyThread(TopicName.getPartitionedTopicName(t.getName())); + } + policiesNotifyThread.execute(runnable); + } + private void handleDynamicConfigurationUpdates() { DynamicConfigurationResources dynamicConfigResources = null; try { @@ -3569,6 +3586,24 @@ public OrderedExecutor getTopicOrderedExecutor() { return topicOrderedExecutor; } + /** + * Returns the single executor thread used to apply topic-policy updates for the given topic. All + * topic-policy notifications and policy application for a topic must run on this deterministically-chosen + * thread so that they are serialized and never run concurrently. Centralizing the topic-to-thread mapping + * here keeps it consistent between {@link AbstractTopic} and + * {@link SystemTopicBasedTopicPoliciesService} so the two cannot accidentally diverge. + */ + public ExecutorService getTopicPoliciesNotifyThread(TopicName topicName) { + TopicName baseTopicName; + if (topicName.isPartitioned()) { + // for partitioned topics, we need to use the base topic name + baseTopicName = TopicName.get(topicName.getPartitionedTopicName()); + } else { + baseTopicName = topicName; + } + return topicOrderedExecutor.chooseThread(baseTopicName); + } + /** * If per-broker unacked message reached to limit then it blocks dispatcher if its unacked message limit has been * reached to {@link #maxUnackedMsgsPerDispatcher}. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 7e1b453b35b88..623933eaafcbc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -35,6 +35,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -485,16 +486,7 @@ private void notifyListener(Message msg) { if (msg.getValue() == null) { TopicName topicName = TopicName.get(TopicPoliciesService.unwrapEventKey(msg.getKey()) .getPartitionedTopicName()); - List listeners = this.listeners.get(topicName); - if (listeners != null) { - for (TopicPolicyListener listener : listeners) { - try { - listener.onUpdate(null); - } catch (Throwable error) { - log.error("[{}] call listener error.", topicName, error); - } - } - } + notifyListenersForTopic(topicName, null); return; } @@ -504,19 +496,62 @@ private void notifyListener(Message msg) { TopicPoliciesEvent event = msg.getValue().getTopicPoliciesEvent(); TopicName topicName = TopicName.get(event.getDomain(), event.getTenant(), event.getNamespace(), event.getTopic()); - List listeners = this.listeners.get(topicName); - if (listeners != null) { - TopicPolicies policies = event.getPolicies(); - for (TopicPolicyListener listener : listeners) { - try { - listener.onUpdate(policies); - } catch (Throwable error) { - log.error("[{}] call listener error.", topicName, error); - } + notifyListenersForTopic(topicName, event.getPolicies()); + } + + /** + * Notifies the topic-policy listeners registered for {@code topicName} of a policy update. + * + *

The {@link TopicPolicyListener#onUpdate} calls are dispatched to the per-topic ordered executor + * rather than run inline. The reader callbacks that drive notifications (the {@link #initPolicesCache} + * replay loop and {@link #readMorePoliciesAsync}) run on the single, process-wide shared + * {@code broker-client-shared-internal-executor} thread. A listener (e.g. + * {@code PersistentTopic.onUpdate} -> {@code applyUpdatedTopicPolicies}) can perform non-trivial and + * even blocking work, so running it inline serializes and can stall topic-policy loading for every + * namespace (issue #26037). Keying {@code executeOrdered} by {@code topicName} preserves per-topic + * notification ordering. + */ + private void notifyListenersForTopic(TopicName topicName, @Nullable TopicPolicies policies) { + // The per-topic value is a CopyOnWriteArrayList, so iterating it later on the executor thread stays + // safe even if a listener is registered/unregistered between dispatch and execution. + List topicListeners = listeners.get(topicName); + if (topicListeners == null || topicListeners.isEmpty()) { + return; + } + pulsarService.getBrokerService().getTopicPoliciesNotifyThread(topicName).execute(() -> { + internalNotifyTopicListeners(topicName, policies, topicListeners); + }); + } + + // this method should only be called from the topic ordered executor thread + // use notifyListenersForTopic/notifyListenersForTopicAsync instead + private static void internalNotifyTopicListeners(TopicName topicName, @Nullable TopicPolicies policies, + List topicListeners) { + for (TopicPolicyListener listener : topicListeners) { + try { + listener.onUpdate(policies); + } catch (Throwable error) { + log.error("[{}] Error in notifying listener {} on topic policy update.", + topicName, listener, error); } } } + private CompletableFuture notifyListenersForTopicAsync(TopicName topicName, + @Nullable TopicPolicies policies) { + // The per-topic value is a CopyOnWriteArrayList, so iterating it later on the executor thread stays + // safe even if a listener is registered/unregistered between dispatch and execution. + List topicListeners = listeners.get(topicName); + if (topicListeners == null || topicListeners.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + ExecutorService pinnedTopicOrderedExecutor = + pulsarService.getBrokerService().getTopicPoliciesNotifyThread(topicName); + return CompletableFuture.runAsync(() -> { + internalNotifyTopicListeners(topicName, policies, topicListeners); + }, pinnedTopicOrderedExecutor); + } + @Override public CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type) { requireNonNull(topicName); @@ -827,19 +862,13 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp } // replay policy message - policiesCache.forEach(((topicName, topicPolicies) -> { - if (listeners.get(topicName) != null) { - for (TopicPolicyListener listener : listeners.get(topicName)) { - try { - listener.onUpdate(topicPolicies); - } catch (Throwable error) { - log.error("[{}] call listener error.", topicName, error); - } - } - } - })); - - future.complete(null); + List> notifyFutures = new ArrayList<>(); + for (Map.Entry entry : policiesCache.entrySet()) { + TopicName topicName = entry.getKey(); + TopicPolicies policies = entry.getValue(); + notifyFutures.add(notifyListenersForTopicAsync(topicName, policies)); + } + FutureUtil.completeAfter(future, FutureUtil.waitForAll(notifyFutures)); } }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 45a92c92f7ee9..d512d1379b3fe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -166,7 +166,7 @@ private CompletableFuture updateClusterMigrated() { public CompletableFuture initialize() { return brokerService.pulsar().getPulsarResources().getNamespaceResources() .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) - .thenCompose(optPolicies -> { + .thenComposeAsync(optPolicies -> { final Policies policies; if (optPolicies.isEmpty()) { log.warn("[{}] Policies not present and isEncryptionRequired will be set to false", topic); @@ -181,7 +181,7 @@ public CompletableFuture initialize() { updatePublishRateLimiter(); updateResourceGroupLimiter(policies); return updateClusterMigrated(); - }); + }, getPoliciesNotifyThread()); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 2b078029bd24d..4dd2abf9ef8b2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -497,7 +497,7 @@ public CompletableFuture initialize() { this.isEncryptionRequired = policies.encryption_required; isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema; - }, getOrderedExecutor()) + }, getPoliciesNotifyThread()) .thenCompose(ignore -> initTopicPolicy()) .thenCompose(ignore -> removeOrphanReplicationCursors()) .exceptionally(ex -> { @@ -4745,6 +4745,7 @@ private void updateSubscriptionsDispatcherRateLimiter() { protected CompletableFuture initTopicPolicy() { final var topicPoliciesService = brokerService.pulsar().getTopicPoliciesService(); final var partitionedTopicName = TopicName.getPartitionedTopicName(topic); + return topicPoliciesService.registerListenerAsync(partitionedTopicName, this).thenCompose(registered -> { if (!registered) { return CompletableFuture.completedFuture(null); @@ -4755,11 +4756,11 @@ protected CompletableFuture initTopicPolicy() { return topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, TopicPoliciesService.GetType.GLOBAL_ONLY) .thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), - brokerService.getTopicOrderedExecutor()) + getPoliciesNotifyThread()) .thenCompose(__ -> topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, TopicPoliciesService.GetType.LOCAL_ONLY)) .thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), - brokerService.getTopicOrderedExecutor()); + getPoliciesNotifyThread()); }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 467a5e42d20f4..0cf2a8a03ff2d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -133,6 +133,36 @@ public void onUpdate(TopicPolicies data) { Assert.assertFalse(systemTopicBasedTopicPoliciesService.listeners.containsKey(topicName)); } + @Test + public void testListenerNotificationRunsOffSharedReaderThread() throws Exception { + // Regression test for #26037: topic-policy listener callbacks must not run on the single, + // process-wide shared "broker-client-shared-internal-executor" reader thread. A slow or blocking + // onUpdate there serializes — and can stall — topic-policy loading for every namespace. The + // notification must instead be dispatched to the per-topic ordered executor ("broker-topic-workers"). + + // Initialize the policy cache and start the change-events reader for the namespace. + systemTopicBasedTopicPoliciesService.updateTopicPoliciesAsync(TOPIC1, false, false, topicPolicies -> + topicPolicies.setMaxConsumerPerTopic(10)).get(); + Awaitility.await().untilAsserted(() -> Assert.assertTrue(systemTopicBasedTopicPoliciesService + .getPoliciesCacheInit(TOPIC1.getNamespaceObject()).isDone())); + + // Register a listener that records the thread its onUpdate runs on. + CompletableFuture onUpdateThreadName = new CompletableFuture<>(); + TopicPolicyListener listener = data -> onUpdateThreadName.complete(Thread.currentThread().getName()); + systemTopicBasedTopicPoliciesService.registerListenerAsync(TOPIC1, listener).get(); + + // A live policy update flows through readMorePoliciesAsync -> notifyListener -> listener.onUpdate. + systemTopicBasedTopicPoliciesService.updateTopicPoliciesAsync(TOPIC1, false, false, topicPolicies -> + topicPolicies.setMaxConsumerPerTopic(20)).get(); + + String threadName = onUpdateThreadName.get(30, TimeUnit.SECONDS); + assertFalse("listener.onUpdate must not run on the shared broker-client reader thread, but ran on: " + + threadName, + threadName.contains("broker-client-shared-internal-executor")); + assertTrue("listener.onUpdate should run on the per-topic ordered executor, but ran on: " + threadName, + threadName.contains("broker-topic-workers")); + } + @Test public void testGetPolicy() throws Exception { From 0ac71ff034fd8adc67ea29ceb406a78863d058d4 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 18 Jun 2026 02:09:04 +0300 Subject: [PATCH 155/201] [fix][broker] Run the message expiry check off the topic policy update path (#26040) (cherry picked from commit 476312837b5660b23b5ea2c6ad112bdfba88f2ca) --- .../pulsar/broker/service/BrokerService.java | 1 + .../service/persistent/PersistentTopic.java | 26 +++++++++++++++++-- .../pulsar/broker/service/MessageTTLTest.java | 7 ++--- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 0664d9ff290b8..275d0a61a89fc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -285,6 +285,7 @@ public class BrokerService implements Closeable { .register(); private final SingleThreadNonConcurrentFixedRateScheduler inactivityMonitor; + @Getter private final SingleThreadNonConcurrentFixedRateScheduler messageExpiryMonitor; private final SingleThreadNonConcurrentFixedRateScheduler compactionMonitor; private final SingleThreadNonConcurrentFixedRateScheduler consumedLedgersMonitor; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 4dd2abf9ef8b2..6e85477a1da68 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -226,6 +226,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal private final TopicName shadowSourceTopic; public static final String DEDUPLICATION_CURSOR_NAME = "pulsar.dedup"; + private volatile int lastMessageTtlInSeconds; public static boolean isDedupCursorName(String name) { return DEDUPLICATION_CURSOR_NAME.equals(name); @@ -3719,8 +3720,9 @@ private List> applyUpdatedTopicPolicies() { }); producers.values().forEach(producer -> applyPoliciesFutureList.add( producer.checkPermissionsAsync().thenRun(producer::checkEncryption))); - // Check message expiry. - applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> checkMessageExpiry())); + + // Check message expiry in the background so that it doesn't block updating or loading topic policies. + maybeCheckMessageExpiryOnPolicyUpdateInBackground(); // Update rate limiters. applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateDispatchRateLimiter())); @@ -3745,6 +3747,26 @@ private List> applyUpdatedTopicPolicies() { return applyPoliciesFutureList; } + /** + * Triggers a message-expiry check on a policy update, but only when the message TTL actually changed, and + * runs it on the messageExpiryMonitor thread (the same thread the periodic expiry check uses) so it never + * blocks updating or loading topic policies. Unlike before, the expiry check is no longer part of the + * returned {@link #applyUpdatedTopicPolicies()} futures, so callers no longer wait for it; the periodic + * monitor provides eventual coverage and the check is idempotent. + */ + private void maybeCheckMessageExpiryOnPolicyUpdateInBackground() { + int messageTtlInSeconds = topicPolicies.getMessageTTLInSeconds().get(); + // don't check if the message expiry hasn't changed + if (messageTtlInSeconds > 0) { + if (lastMessageTtlInSeconds != messageTtlInSeconds) { + lastMessageTtlInSeconds = messageTtlInSeconds; + brokerService.getMessageExpiryMonitor().execute(this::checkMessageExpiry); + } + } else { + lastMessageTtlInSeconds = messageTtlInSeconds; + } + } + /** * * @return Backlog quota for topic diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java index beb054d86f5e6..699f5638aabd9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.service; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; +import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -138,12 +138,13 @@ public void testTTLPoliciesUpdate() throws Exception { Policies policies = admin.namespaces().getPolicies(namespace); policies.message_ttl_in_seconds = 10; topicRefMock.onPoliciesUpdate(policies); - verify(topicRefMock, times(1)).checkMessageExpiry(); + // checkMessageExpiry now runs asynchronously on the messageExpiryMonitor thread, so wait for it. + verify(topicRefMock, timeout(5000).times(1)).checkMessageExpiry(); TopicPolicies topicPolicies = new TopicPolicies(); topicPolicies.setMessageTTLInSeconds(5); topicRefMock.onUpdate(topicPolicies); - verify(topicRefMock, times(2)).checkMessageExpiry(); + verify(topicRefMock, timeout(5000).times(2)).checkMessageExpiry(); } @Test From c9d47a58dbfabf83baa8726043c302ea4c05e4ed Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 18 Jun 2026 02:44:55 +0300 Subject: [PATCH 156/201] [fix][test] Fix flaky testPrepareInitPoliciesCacheAsyncThrowExceptionAfterCreateReader (#26049) (cherry picked from commit 6f82106ea6c4d508a883ff27875ef279c7c4101c) --- ...temTopicBasedTopicPoliciesServiceTest.java | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 0cf2a8a03ff2d..9a05c51786851 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -49,6 +49,7 @@ import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.naming.NamespaceName; @@ -555,14 +556,28 @@ public void testPrepareInitPoliciesCacheAsyncThrowExceptionAfterCreateReader() t assertTrue(logFound); }); - - // Since cleanPoliciesCacheInitMap() is executed, should add the failed reader into readerCache again. - // Then in SystemTopicBasedTopicPoliciesService, readerCache has a closed reader, - // and policyCacheInitMap do not contain a future. - // To simulate the situation: when getTopicPolicy() execute, it will do prepareInitPoliciesCacheAsync() and - // use a closed reader to read the __change_event topic. Then throw exception - spyReaderCaches.put(NamespaceName.get(NAMESPACE5), readerCompletableFuture); + // The reader.close() above drives readMorePoliciesAsync() into cleanPoliciesCacheInitMap(), which logs + // "Closing the topic policies reader for" and removes the namespace from policyCacheInitMap and readerCaches. + // Now exercise a follow-up prepareInitPoliciesCacheAsync() that reuses a closed reader and must fail in + // initPolicesCache(). Two timing hazards made this flaky (#25081), so both are pinned deterministically: + // 1) A real closed reader's hasMoreEventsAsync() (Reader.hasMessageAvailableAsync()) can answer from cached + // state instead of failing with AlreadyClosedException, in which case initPolicesCache() reaches the end + // of the topic and completes successfully and the expected exception never happens. Use a spy whose + // hasMoreEventsAsync() always fails with AlreadyClosedException. + // 2) A background topic load can re-run prepareInitPoliciesCacheAsync() for the namespace and leave a + // completed init future behind; the next call would then short-circuit through the existing-future + // branch and never re-initialize. Drop any init future right before the call so it re-initializes, and + // stub createSystemTopicClient() so every reader created for this namespace is the failing spy — then + // whichever initialization wins the race fails in the same (asserted) way. + SystemTopicClient.Reader closedReader = Mockito.spy(reader); + Mockito.doReturn(CompletableFuture.failedFuture( + new PulsarClientException.AlreadyClosedException("Reader is already closed"))) + .when(closedReader).hasMoreEventsAsync(); + Mockito.doReturn(CompletableFuture.completedFuture(closedReader)) + .when(spyService).createSystemTopicClient(NamespaceName.get(NAMESPACE5)); + spyReaderCaches.put(NamespaceName.get(NAMESPACE5), CompletableFuture.completedFuture(closedReader)); FieldUtils.writeDeclaredField(spyService, "readerCaches", spyReaderCaches, true); + spyService.policyCacheInitMap.remove(NamespaceName.get(NAMESPACE5)); CompletableFuture prepareFuture = new CompletableFuture<>(); try { From 7ef83a0a809a0c34ed0fbb55e535975827f3a84d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 18 Jun 2026 10:29:17 +0300 Subject: [PATCH 157/201] [fix][broker] Prevent topic policy initialization race with a buffering listener wrapper (#26044) (cherry picked from commit b70b3a386c29374cdfa9c5367aacda8a730e4a71) --- .../pulsar/broker/service/AbstractTopic.java | 8 +- .../service/TopicPolicyListenerWrapper.java | 117 +++++++++++++++++ .../nonpersistent/NonPersistentTopic.java | 18 ++- .../service/persistent/PersistentTopic.java | 56 +++++--- .../TopicPolicyListenerWrapperTest.java | 124 ++++++++++++++++++ 5 files changed, 304 insertions(+), 19 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 1ab26994f81ee..17b2781c912c6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -557,14 +557,18 @@ protected boolean isProducersExceeded(boolean isRemote) { && maxProducers <= USER_CREATED_PRODUCER_COUNTER_UPDATER.get(this); } + protected TopicPolicyListener getTopicPolicyListener() { + return this; + } + protected void registerTopicPolicyListener() { brokerService.getPulsar().getTopicPoliciesService() - .registerListenerAsync(TopicName.getPartitionedTopicName(topic), this); + .registerListenerAsync(TopicName.getPartitionedTopicName(topic), getTopicPolicyListener()); } protected void unregisterTopicPolicyListener() { brokerService.getPulsar().getTopicPoliciesService() - .unregisterListener(TopicName.getPartitionedTopicName(topic), this); + .unregisterListener(TopicName.getPartitionedTopicName(topic), getTopicPolicyListener()); } protected boolean isSameAddressProducersExceeded(Producer producer) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java new file mode 100644 index 0000000000000..c04d5d30e12d0 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.service; + +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.policies.data.TopicPolicies; + +/** + * This TopicPolicyListener is used as a wrapper for the real TopicPolicyListener. + * This prevents a race condition in initialization where the topic policy state can change while the + * topic policy state is being applied to the topic in PersistentTopic#initTopicPolicy() method or in + * NonPersistentTopic#initialize method. The impact of the race conditions is that the topic policy state would + * be left in an inconsistent state until another update arrives. This is a rare corner case, but possible. + */ +@Slf4j +public class TopicPolicyListenerWrapper implements TopicPolicyListener { + private final TopicPolicyListener realTopicListener; + // The latest value received during initialization, per scope. A null reference means no update was + // received during initialization (the loaded value should be used); an Optional that is present holds the + // received policies, and an empty Optional records that a delete (onUpdate(null)) was received, so the + // loaded value must not be applied. Optional is used because the map-like field cannot itself hold null + // while still distinguishing "not received" (null) from "received a delete" (Optional.empty()). + private Optional latestGlobalPolicies; + private Optional latestLocalPolicies; + private boolean initialized; + private final long createdTimestampNanos = System.nanoTime(); + private static final long INITIALIZATION_WARNING_LOG_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30); + private int lastIntervalLogged; + + public TopicPolicyListenerWrapper(TopicPolicyListener realTopicListener) { + this.realTopicListener = realTopicListener; + } + + @Override + public synchronized void onUpdate(TopicPolicies data) { + if (initialized) { + realTopicListener.onUpdate(data); + return; + } + + maybeLogWarning(); + + // Record the latest value received during initialization so it can be applied (preferring it over the + // loaded value) in completeInitialization. A received value is stored as Optional.of(data) and a delete + // as Optional.empty(), so the delete is propagated downstream instead of being lost. + if (data == null) { + // A delete (onUpdate(null)) does not carry the global/local scope through the listener interface, + // so record it for both scopes; a later scoped update received during initialization still + // overrides its own scope. + latestGlobalPolicies = Optional.empty(); + latestLocalPolicies = Optional.empty(); + } else if (data.isGlobalPolicies()) { + latestGlobalPolicies = Optional.of(data); + } else { + latestLocalPolicies = Optional.of(data); + } + } + + /** + * Complete initialization of the TopicPolicyListenerWrapper and emit the latest policies to the real listener. + * @param loadedGlobalPolicies the loaded global policies + * @param loadedLocalPolicies the loaded local policies + */ + public synchronized void completeInitialization(TopicPolicies loadedGlobalPolicies, + TopicPolicies loadedLocalPolicies) { + // The listener might have received a newer value (or a delete) than the loaded one while the loading + // was happening; prefer the latest value received during initialization, falling back to the loaded + // value only when nothing was received for that scope. + emitInitialPolicies(latestGlobalPolicies, loadedGlobalPolicies); + emitInitialPolicies(latestLocalPolicies, loadedLocalPolicies); + latestGlobalPolicies = null; + latestLocalPolicies = null; + initialized = true; + } + + private void emitInitialPolicies(Optional latestReceived, TopicPolicies loaded) { + if (latestReceived != null) { + // A value (or a delete) was received during initialization; it supersedes the loaded value. + realTopicListener.onUpdate(latestReceived.orElse(null)); + } else if (loaded != null) { + realTopicListener.onUpdate(loaded); + } + } + + // warn if the initialization takes too long and updates have been received + // this helps detect issues where completeInitialization didn't get called after loading policies + private void maybeLogWarning() { + long durationNanos = System.nanoTime() - createdTimestampNanos; + int warningLogIntervalCount = (int) (durationNanos / INITIALIZATION_WARNING_LOG_INTERVAL_NANOS); + if (warningLogIntervalCount > lastIntervalLogged) { + log.warn("TopicPolicyUpdate buffered. TopicPolicyListenerWrapper initialization phase took too long. " + + "completeInitialization should have been called to complete the phase. " + + "topicPolicyListener={} sinceCreationMs={}", + realTopicListener, TimeUnit.NANOSECONDS.toMillis(durationNanos)); + lastIntervalLogged = warningLogIntervalCount; + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index d512d1379b3fe..18a9e7fc57109 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -66,6 +66,7 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.TopicAttributes; import org.apache.pulsar.broker.service.TopicPolicyListener; +import org.apache.pulsar.broker.service.TopicPolicyListenerWrapper; import org.apache.pulsar.broker.service.TransportCnx; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.service.schema.exceptions.NotExistSchemaException; @@ -129,6 +130,9 @@ protected TopicStats initialValue() { TOPIC_ATTRIBUTES_FIELD_UPDATER = AtomicReferenceFieldUpdater.newUpdater( NonPersistentTopic.class, TopicAttributes.class, "topicAttributes"); + // prevents race conditions in topic policy initialization + private final TopicPolicyListenerWrapper topicPolicyListener = new TopicPolicyListenerWrapper(this); + private static class TopicStats { public double averageMsgSize; public double aggMsgRateIn; @@ -181,7 +185,14 @@ public CompletableFuture initialize() { updatePublishRateLimiter(); updateResourceGroupLimiter(policies); return updateClusterMigrated(); - }, getPoliciesNotifyThread()); + }, getPoliciesNotifyThread()) + // Complete the topic-policy listener wrapper so buffered and future topic-level policy + // updates are forwarded to this topic. Without this the wrapper stays uninitialized forever + // and all topic-level policy updates are silently dropped. Unlike PersistentTopic, + // non-persistent topics don't load initial topic policies (matching the previous behavior), + // so the loaded values are passed as null. + .thenRunAsync(() -> topicPolicyListener.completeInitialization(null, null), + getPoliciesNotifyThread()); } @Override @@ -1290,4 +1301,9 @@ public TopicAttributes getTopicAttributes() { return TOPIC_ATTRIBUTES_FIELD_UPDATER.updateAndGet(this, old -> old != null ? old : new TopicAttributes(TopicName.get(topic))); } + + @Override + public TopicPolicyListener getTopicPolicyListener() { + return topicPolicyListener; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 6e85477a1da68..23274b77b8685 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -59,6 +59,7 @@ import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.BiConsumer; import java.util.function.BiFunction; +import java.util.function.Function; import lombok.Getter; import lombok.Value; import org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsException; @@ -135,6 +136,8 @@ import org.apache.pulsar.broker.service.SubscriptionOption; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.TopicPoliciesService; +import org.apache.pulsar.broker.service.TopicPolicyListener; +import org.apache.pulsar.broker.service.TopicPolicyListenerWrapper; import org.apache.pulsar.broker.service.TransportCnx; import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; @@ -294,6 +297,10 @@ protected TopicStatsHelper initialValue() { // Record the last time max read position is moved forward, unless it's a marker message. @Getter private volatile long lastMaxReadPositionMovedForwardTimestamp = 0; + + // prevents race conditions in topic policy initialization + private final TopicPolicyListenerWrapper topicPolicyListener = new TopicPolicyListenerWrapper(this); + @Getter private final ExecutorService orderedExecutor; @@ -4768,22 +4775,34 @@ protected CompletableFuture initTopicPolicy() { final var topicPoliciesService = brokerService.pulsar().getTopicPoliciesService(); final var partitionedTopicName = TopicName.getPartitionedTopicName(topic); - return topicPoliciesService.registerListenerAsync(partitionedTopicName, this).thenCompose(registered -> { - if (!registered) { - return CompletableFuture.completedFuture(null); - } - if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { - return CompletableFuture.completedFuture(null); - } - return topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, - TopicPoliciesService.GetType.GLOBAL_ONLY) - .thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), - getPoliciesNotifyThread()) - .thenCompose(__ -> topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, - TopicPoliciesService.GetType.LOCAL_ONLY)) - .thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), - getPoliciesNotifyThread()); - }); + return topicPoliciesService.registerListenerAsync(partitionedTopicName, topicPolicyListener) + .thenCompose(registered -> { + if (!registered) { + return CompletableFuture.completedFuture(null); + } + if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { + // Internal topics don't load topic-level policies, but the listener wrapper must + // still be initialized so any buffered/future updates are forwarded to the topic + // instead of being silently dropped. + return CompletableFuture.runAsync( + () -> topicPolicyListener.completeInitialization(null, null), + getPoliciesNotifyThread()); + } + // future for fetching global topic policies + CompletableFuture> globalPoliciesFuture = + topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, + TopicPoliciesService.GetType.GLOBAL_ONLY); + // future for fetching local topic policies + CompletableFuture> localPoliciesFuture = + topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, + TopicPoliciesService.GetType.LOCAL_ONLY); + return globalPoliciesFuture.thenCombine(localPoliciesFuture, (global, local) -> { + // finally update the topic policies with the latest value or loaded value + return CompletableFuture.runAsync(() -> + topicPolicyListener.completeInitialization(global.orElse(null), local.orElse(null)), + getPoliciesNotifyThread()); + }).thenCompose(Function.identity()); + }); } @VisibleForTesting @@ -4956,4 +4975,9 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { return future; } + + @Override + public TopicPolicyListener getTopicPolicyListener() { + return topicPolicyListener; + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java new file mode 100644 index 0000000000000..0e7553592741d --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import java.util.ArrayList; +import java.util.List; +import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class TopicPolicyListenerWrapperTest { + + private static TopicPolicies globalPolicies() { + return TopicPolicies.builder().isGlobal(true).build(); + } + + private static TopicPolicies localPolicies() { + return TopicPolicies.builder().isGlobal(false).build(); + } + + private static final class RecordingListener implements TopicPolicyListener { + final List updates = new ArrayList<>(); + + @Override + public void onUpdate(TopicPolicies data) { + updates.add(data); + } + } + + @Test + public void shouldBufferUpdatesUntilInitializedThenForwardLive() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + // Updates received before initialization are buffered, not forwarded. + TopicPolicies bufferedLocal = localPolicies(); + wrapper.onUpdate(bufferedLocal); + assertThat(real.updates).isEmpty(); + + // On completion, the buffered local value wins over the loaded local value; the loaded global value + // is applied since none was buffered. + TopicPolicies loadedGlobal = globalPolicies(); + wrapper.completeInitialization(loadedGlobal, localPolicies()); + assertThat(real.updates).containsExactly(loadedGlobal, bufferedLocal); + + // After initialization, updates are forwarded immediately. + TopicPolicies liveUpdate = localPolicies(); + wrapper.onUpdate(liveUpdate); + assertThat(real.updates).containsExactly(loadedGlobal, bufferedLocal, liveUpdate); + } + + @Test + public void shouldPreferBufferedOverLoadedForBothScopes() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + TopicPolicies bufferedGlobal = globalPolicies(); + TopicPolicies bufferedLocal = localPolicies(); + wrapper.onUpdate(bufferedGlobal); + wrapper.onUpdate(bufferedLocal); + + wrapper.completeInitialization(globalPolicies(), localPolicies()); + assertThat(real.updates).containsExactly(bufferedGlobal, bufferedLocal); + } + + @Test + public void shouldApplyLoadedWhenNothingBuffered() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + TopicPolicies loadedGlobal = globalPolicies(); + TopicPolicies loadedLocal = localPolicies(); + wrapper.completeInitialization(loadedGlobal, loadedLocal); + assertThat(real.updates).containsExactly(loadedGlobal, loadedLocal); + } + + @Test + public void shouldSuppressLoadedValuesWhenDeletedBeforeInitialization() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + // A delete (null) arriving before initialization must not NPE and must not be forwarded yet (#26037). + assertThatCode(() -> wrapper.onUpdate(null)).doesNotThrowAnyException(); + assertThat(real.updates).isEmpty(); + + // The delete supersedes the (now-stale) loaded values: they are not applied, and the delete (null) is + // propagated downstream instead. + wrapper.completeInitialization(globalPolicies(), localPolicies()); + assertThat(real.updates).containsExactly(null, null); + } + + @Test + public void shouldApplyLatestScopedUpdateOverEarlierDeleteDuringInitialization() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + // A delete records empty for both scopes, then a newer global update overrides only the global scope. + wrapper.onUpdate(null); + TopicPolicies newerGlobal = globalPolicies(); + wrapper.onUpdate(newerGlobal); + + wrapper.completeInitialization(globalPolicies(), localPolicies()); + // Global: the newer update wins; Local: the delete (null) wins over the loaded local value. + assertThat(real.updates).containsExactly(newerGlobal, null); + } +} From 7d8d4dcccf4ff2f4bba8ec4bc6ae55687f1445a6 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 18 Jun 2026 02:40:15 -0700 Subject: [PATCH 158/201] [fix][broker] Avoid blocking metadata read on the IO thread when redirecting migrated producers/consumers (#26051) (cherry picked from commit ac053c38fe40723bcee193c1aa6ab5e1d39d1779) --- .../service/AbstractBaseDispatcher.java | 3 +- .../pulsar/broker/service/AbstractTopic.java | 14 +- .../pulsar/broker/service/Consumer.java | 22 +-- .../pulsar/broker/service/ServerCnx.java | 164 +++++++++++------- .../nonpersistent/NonPersistentTopic.java | 30 ++-- .../service/persistent/PersistentTopic.java | 77 ++++---- 6 files changed, 174 insertions(+), 136 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index c5e001692f2f3..c4d99f1f6039d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -481,7 +481,8 @@ protected void checkAndApplyReachedEndOfTopicOrTopicMigration(List con public static void checkAndApplyReachedEndOfTopicOrTopicMigration(PersistentTopic topic, List consumers) { if (topic.isMigrated()) { - consumers.forEach(c -> c.topicMigrated(topic.getMigratedClusterUrl())); + topic.getMigratedClusterUrlAsync() + .thenAccept(clusterUrl -> consumers.forEach(c -> c.topicMigrated(clusterUrl))); } else { consumers.forEach(Consumer::reachedEndOfTopic); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 17b2781c912c6..78c4ba2746994 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -1349,8 +1349,8 @@ public void updateBrokerSubscribeRate() { subscribeRateInBroker(brokerService.pulsar().getConfiguration())); } - public Optional getMigratedClusterUrl() { - return getMigratedClusterUrl(brokerService.getPulsar(), topic); + public CompletableFuture> getMigratedClusterUrlAsync() { + return getMigratedClusterUrlAsync(brokerService.getPulsar(), topic); } public static CompletableFuture isClusterMigrationEnabled(PulsarService pulsar, @@ -1390,16 +1390,6 @@ private static CompletableFuture isNamespaceMigrationEnabledAsync(Pulsa .thenApply(policies -> policies.isPresent() && policies.get().migrated); } - public static Optional getMigratedClusterUrl(PulsarService pulsar, String topic) { - try { - return getMigratedClusterUrlAsync(pulsar, topic) - .get(pulsar.getPulsarResources().getClusterResources().getOperationTimeoutSec(), TimeUnit.SECONDS); - } catch (Exception e) { - log.warn("[{}] Failed to get migration cluster URL", topic, e); - } - return Optional.empty(); - } - public boolean isSystemCursor(String sub) { return COMPACTION_SUBSCRIPTION.equals(sub) || (additionalSystemCursorNames != null && additionalSystemCursorNames.contains(sub)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 8d897c80062f7..f7fca52065d9c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -914,20 +914,16 @@ public void topicMigrated(Optional clusterUrl) { } } - public boolean checkAndApplyTopicMigration() { - if (subscription.isSubscriptionMigrated()) { - Optional clusterUrl = AbstractTopic.getMigratedClusterUrl(cnx.getBrokerService().getPulsar(), - topicName); - if (clusterUrl.isPresent()) { - ClusterUrl url = clusterUrl.get(); - cnx.getCommandSender().sendTopicMigrated(ResourceType.Consumer, consumerId, url.getBrokerServiceUrl(), - url.getBrokerServiceUrlTls()); - // disconnect consumer after sending migrated cluster url - disconnect(); - return true; - } + public CompletableFuture checkAndApplyTopicMigrationAsync() { + if (!subscription.isSubscriptionMigrated()) { + return CompletableFuture.completedFuture(false); } - return false; + return AbstractTopic.getMigratedClusterUrlAsync(cnx.getBrokerService().getPulsar(), topicName) + .thenApply(clusterUrl -> { + // topicMigrated() sends the migrated cluster url and disconnects the consumer if present + topicMigrated(clusterUrl); + return clusterUrl.isPresent(); + }); } /** * Checks if consumer-blocking on unAckedMessages is allowed for below conditions:
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index de1e16a0d2fbb..b8b1fbb997021 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -26,7 +26,7 @@ import static org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.unsafeGetPartitionedTopicMetadataAsync; import static org.apache.pulsar.broker.lookup.TopicLookupBase.lookupTopicAsync; import static org.apache.pulsar.broker.service.ServerCnxThrottleTracker.ThrottleType; -import static org.apache.pulsar.broker.service.persistent.PersistentTopic.getMigratedClusterUrl; +import static org.apache.pulsar.broker.service.persistent.PersistentTopic.getMigratedClusterUrlAsync; import static org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage.ignoreUnrecoverableBKException; import static org.apache.pulsar.common.api.proto.ProtocolVersion.v5; import static org.apache.pulsar.common.naming.Constants.WEBSOCKET_DUMMY_ORIGINAL_PRINCIPLE; @@ -1490,8 +1490,9 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { } }); }) - .thenAcceptAsync(consumer -> { - if (consumer.checkAndApplyTopicMigration()) { + .thenComposeAsync(consumer -> consumer.checkAndApplyTopicMigrationAsync() + .thenAcceptAsync(migrated -> { + if (migrated) { log.info("[{}] Disconnecting consumer {} on migrated subscription on topic {} / {}", remoteAddress, consumerId, subscriptionName, topicName); consumers.remove(consumerId, consumerFuture); @@ -1524,7 +1525,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { consumers.remove(consumerId, consumerFuture); } - }, ctx.executor()) + }, ctx.executor()), ctx.executor()) .exceptionallyAsync(exception -> { if (exception.getCause() instanceof ConsumerBusyException) { if (log.isDebugEnabled()) { @@ -1535,23 +1536,12 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { exception.getCause().getMessage()); } } else if (exception.getCause() instanceof BrokerServiceException.TopicMigratedException) { - Optional clusterURL = getMigratedClusterUrl(service.getPulsar(), - topicName.toString()); - if (clusterURL.isPresent()) { - log.info("[{}] redirect migrated consumer to topic {}: " - + "consumerId={}, subName={}, {}", remoteAddress, - topicName, consumerId, subscriptionName, exception.getCause().getMessage()); - boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Consumer, consumerId, - clusterURL.get().getBrokerServiceUrl(), - clusterURL.get().getBrokerServiceUrlTls()); - if (!msgSent) { - log.info("consumer client doesn't support topic migration handling {}-{}-{}", - topicName, remoteAddress, consumerId); - } - consumers.remove(consumerId, consumerFuture); - closeConsumer(consumerId, Optional.empty()); - return null; - } + getMigratedClusterUrlAsync(service.getPulsar(), topicName.toString()) + .exceptionally(e -> Optional.empty()) + .thenAcceptAsync(clusterURL -> redirectOrFailMigratedConsumer(requestId, + consumerId, subscriptionName, topicName, consumerFuture, exception, + clusterURL), ctx.executor()); + return null; } else if (exception.getCause() instanceof BrokerServiceException) { log.warn("[{}][{}][{}] Failed to create consumer: consumerId={}, {}", remoteAddress, topicName, subscriptionName, @@ -1589,6 +1579,89 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { }, ctx.executor()); } + private void redirectOrFailMigratedConsumer(long requestId, long consumerId, String subscriptionName, + TopicName topicName, CompletableFuture consumerFuture, Throwable exception, + Optional clusterURL) { + if (clusterURL.isPresent()) { + log.info("[{}] redirect migrated consumer to topic {}: consumerId={}, subName={}, {}", remoteAddress, + topicName, consumerId, subscriptionName, exception.getCause().getMessage()); + boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Consumer, consumerId, + clusterURL.get().getBrokerServiceUrl(), + clusterURL.get().getBrokerServiceUrlTls()); + if (!msgSent) { + log.info("consumer client doesn't support topic migration handling {}-{}-{}", + topicName, remoteAddress, consumerId); + } + consumers.remove(consumerId, consumerFuture); + closeConsumer(consumerId, Optional.empty()); + } else { + // If client timed out, the future would have been completed by subsequent close. + // Send error back to client, only if not completed already. + if (consumerFuture.completeExceptionally(exception)) { + commandSender.sendErrorResponse(requestId, + BrokerServiceException.getClientErrorCode(exception.getCause()), + exception.getCause().getMessage()); + } + consumers.remove(consumerId, consumerFuture); + } + } + + private void redirectOrFailMigratedProducer(long requestId, long producerId, String producerName, + TopicName topicName, CompletableFuture producerFuture, Throwable exception, + Optional clusterURL) { + if (clusterURL.isPresent()) { + log.info("[{}] redirect migrated producer to topic {}: producerId={}, producerName = {}, {}", + remoteAddress, topicName, producerId, producerName, exception.getCause().getMessage()); + boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId, + clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); + if (!msgSent) { + log.info("client doesn't support topic migration handling {}-{}-{}", topicName, + remoteAddress, producerId); + } + producers.remove(producerId, producerFuture); + closeProducer(producerId, -1L, Optional.empty()); + } else { + log.error("[{}] Failed to create topic {}, producerId={}", remoteAddress, topicName, producerId, + exception); + if (producerFuture.completeExceptionally(exception)) { + commandSender.sendErrorResponse(requestId, + BrokerServiceException.getClientErrorCode(exception.getCause()), + exception.getCause().getMessage()); + } + producers.remove(producerId, producerFuture); + } + } + + private void redirectOrFailMigratedProducerInQueue(long requestId, long producerId, String producerName, + TopicName topicName, Topic topic, Producer producer, CompletableFuture producerFuture, + Throwable ex, Optional clusterURL) { + if (clusterURL.isPresent() && topic.shouldProducerMigrate()) { + log.info("[{}] redirect migrated producer to topic {}: producerId={}, producerName = {}, {}", + remoteAddress, topicName, producerId, producerName, ex.getCause().getMessage()); + boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId, + clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); + if (!msgSent) { + log.info("client doesn't support topic migration handling {}-{}-{}", topic, + remoteAddress, producerId); + } + closeProducer(producer); + } else { + if (clusterURL.isPresent()) { + log.info("Topic {} is migrated but replication backlog exist: " + + "producerId = {}, producerName = {}, {}", topicName, + producerId, producerName, ex.getCause().getMessage()); + } else { + log.warn("[{}] failed producer because migration url not configured topic {}: producerId={}, {}", + remoteAddress, topicName, producerId, ex.getCause().getMessage()); + } + producer.closeNow(true); + if (producerFuture.completeExceptionally(ex)) { + commandSender.sendErrorResponse(requestId, + BrokerServiceException.getClientErrorCode(ex), ex.getMessage()); + } + } + } + private SchemaData getSchema(Schema protocolSchema) { return SchemaData.builder() .data(protocolSchema.getSchemaData()) @@ -1822,21 +1895,11 @@ protected void handleProducer(final CommandProducer cmdProducer) { producers.remove(producerId, producerFuture); return null; } else if (cause instanceof BrokerServiceException.TopicMigratedException) { - Optional clusterURL = getMigratedClusterUrl(service.getPulsar(), topicName.toString()); - if (clusterURL.isPresent()) { - log.info("[{}] redirect migrated producer to topic {}: " - + "producerId={}, producerName = {}, {}", remoteAddress, - topicName, producerId, producerName, cause.getMessage()); - boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId, - clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); - if (!msgSent) { - log.info("client doesn't support topic migration handling {}-{}-{}", topicName, - remoteAddress, producerId); - } - producers.remove(producerId, producerFuture); - closeProducer(producerId, -1L, Optional.empty()); - return null; - } + getMigratedClusterUrlAsync(service.getPulsar(), topicName.toString()) + .exceptionally(e -> Optional.empty()) + .thenAcceptAsync(clusterURL -> redirectOrFailMigratedProducer(requestId, producerId, + producerName, topicName, producerFuture, exception, clusterURL), ctx.executor()); + return null; } // Do not print stack traces for expected exceptions @@ -1922,29 +1985,12 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ producers.remove(producerId, producerFuture); }, ctx.executor()).exceptionallyAsync(ex -> { if (ex.getCause() instanceof BrokerServiceException.TopicMigratedException) { - Optional clusterURL = getMigratedClusterUrl(service.getPulsar(), topic.getName()); - if (clusterURL.isPresent()) { - if (!topic.shouldProducerMigrate()) { - log.info("Topic {} is migrated but replication backlog exist: " - + "producerId = {}, producerName = {}, {}", topicName, - producerId, producerName, ex.getCause().getMessage()); - } else { - log.info("[{}] redirect migrated producer to topic {}: " - + "producerId={}, producerName = {}, {}", remoteAddress, - topicName, producerId, producerName, ex.getCause().getMessage()); - boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId, - clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); - if (!msgSent) { - log.info("client doesn't support topic migration handling {}-{}-{}", topic, - remoteAddress, producerId); - } - closeProducer(producer); - return null; - } - } else { - log.warn("[{}] failed producer because migration url not configured topic {}: producerId={}, {}", - remoteAddress, topicName, producerId, ex.getCause().getMessage()); - } + getMigratedClusterUrlAsync(service.getPulsar(), topic.getName()) + .exceptionally(e -> Optional.empty()) + .thenAcceptAsync(clusterURL -> redirectOrFailMigratedProducerInQueue(requestId, producerId, + producerName, topicName, topic, producer, producerFuture, ex, clusterURL), + ctx.executor()); + return null; } else if (ex.getCause() instanceof BrokerServiceException.ProducerFencedException) { if (log.isDebugEnabled()) { log.debug("[{}] Failed to add producer to topic {}: producerId={}, {}", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 18a9e7fc57109..5de7585d943f1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -83,7 +83,6 @@ import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; -import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.Policies; @@ -340,7 +339,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, DEFAULT_CONSUMER_EPOCH, schemaType); if (isMigrated()) { - consumer.topicMigrated(getMigratedClusterUrl()); + getMigratedClusterUrlAsync().thenAccept(consumer::topicMigrated); } addConsumerToSubscription(subscription, consumer).thenRun(() -> { @@ -1011,20 +1010,21 @@ public CompletableFuture checkClusterMigration() { return CompletableFuture.completedFuture(null); } - Optional url = getMigratedClusterUrl(); - if (url.isPresent()) { - this.migrated = true; - producers.forEach((__, producer) -> { - producer.topicMigrated(url); - }); - subscriptions.forEach((__, sub) -> { - sub.getConsumers().forEach((consumer) -> { - consumer.topicMigrated(url); + return getMigratedClusterUrlAsync().thenCompose(url -> { + if (url.isPresent()) { + this.migrated = true; + producers.forEach((__, producer) -> { + producer.topicMigrated(url); }); - }); - return disconnectReplicators().thenCompose(__ -> checkAndUnsubscribeSubscriptions()); - } - return CompletableFuture.completedFuture(null); + subscriptions.forEach((__, sub) -> { + sub.getConsumers().forEach((consumer) -> { + consumer.topicMigrated(url); + }); + }); + return disconnectReplicators().thenCompose(__ -> checkAndUnsubscribeSubscriptions()); + } + return CompletableFuture.completedFuture(null); + }); } private CompletableFuture checkAndUnsubscribeSubscriptions() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 23274b77b8685..87e9d0d28e0f1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -783,18 +783,23 @@ public synchronized void addFailed(ManagedLedgerException exception, Object ctx) // close all producers CompletableFuture disconnectProducersFuture; if (producers.size() > 0) { - List> futures = new ArrayList<>(); // send migration url metadata to producers before disconnecting them - if (isMigrated()) { - if (!shouldProducerMigrate()) { + CompletableFuture sendMigrationUrlFuture; + if (isMigrated() && shouldProducerMigrate()) { + sendMigrationUrlFuture = getMigratedClusterUrlAsync().thenAccept(clusterUrl -> + producers.forEach((__, producer) -> producer.topicMigrated(clusterUrl))); + } else { + if (isMigrated()) { log.info("Topic {} is migrated but replication-backlog exists or " + "subs not created. Closing producers.", topic); - } else { - producers.forEach((__, producer) -> producer.topicMigrated(getMigratedClusterUrl())); } + sendMigrationUrlFuture = CompletableFuture.completedFuture(null); } - producers.forEach((__, producer) -> futures.add(producer.disconnect())); - disconnectProducersFuture = FutureUtil.waitForAll(futures); + disconnectProducersFuture = sendMigrationUrlFuture.thenCompose(v -> { + List> futures = new ArrayList<>(); + producers.forEach((__, producer) -> futures.add(producer.disconnect())); + return FutureUtil.waitForAll(futures); + }); } else { disconnectProducersFuture = CompletableFuture.completedFuture(null); } @@ -3258,39 +3263,39 @@ public CompletableFuture checkClusterMigration() { return CompletableFuture.completedFuture(null); } - Optional clusterUrl = getMigratedClusterUrl(); - - if (!clusterUrl.isPresent()) { - return CompletableFuture.completedFuture(null); - } + return getMigratedClusterUrlAsync().thenCompose(clusterUrl -> { + if (!clusterUrl.isPresent()) { + return CompletableFuture.completedFuture(null); + } - if (isReplicated()) { - if (isReplicationBacklogExist()) { - if (!ledger.isMigrated()) { - log.info("{} applying migration with replication backlog", topic); - ledger.asyncMigrate(); - } - if (log.isDebugEnabled()) { - log.debug("{} has replication backlog and applied migration", topic); + if (isReplicated()) { + if (isReplicationBacklogExist()) { + if (!ledger.isMigrated()) { + log.info("{} applying migration with replication backlog", topic); + ledger.asyncMigrate(); + } + if (log.isDebugEnabled()) { + log.debug("{} has replication backlog and applied migration", topic); + } + return CompletableFuture.completedFuture(null); } - return CompletableFuture.completedFuture(null); } - } - return initMigration().thenCompose(subCreated -> { - migrationSubsCreated = true; - CompletableFuture migrated = !isMigrated() ? ledger.asyncMigrate() - : CompletableFuture.completedFuture(null); - return migrated.thenApply(__ -> { - subscriptions.forEach((name, sub) -> { - if (sub.isSubscriptionMigrated()) { - sub.getConsumers().forEach(Consumer::checkAndApplyTopicMigration); - } - }); - return null; - }).thenCompose(__ -> checkAndDisconnectReplicators()) - .thenCompose(__ -> checkAndUnsubscribeSubscriptions()) - .thenCompose(__ -> checkAndDisconnectProducers()); + return initMigration().thenCompose(subCreated -> { + migrationSubsCreated = true; + CompletableFuture migrated = !isMigrated() ? ledger.asyncMigrate() + : CompletableFuture.completedFuture(null); + return migrated.thenApply(__ -> { + subscriptions.forEach((name, sub) -> { + if (sub.isSubscriptionMigrated()) { + sub.getConsumers().forEach(consumer -> consumer.topicMigrated(clusterUrl)); + } + }); + return null; + }).thenCompose(__ -> checkAndDisconnectReplicators()) + .thenCompose(__ -> checkAndUnsubscribeSubscriptions()) + .thenCompose(__ -> checkAndDisconnectProducers()); + }); }); } From 07cf61377a5727f1997ff60a97c3cc683fe7b28b Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 18 Jun 2026 04:20:44 -0700 Subject: [PATCH 159/201] [fix][proxy] Avoid blocking the proxy IO thread on a cold broker cache (#26052) (cherry picked from commit 1d7ae01479cc225b9f588c6727d6c5d2e713a0e3) --- .../resources/MetadataStoreCacheLoader.java | 45 ++++++----- .../MetadataStoreCacheLoaderTest.java | 76 +++++++++++++++++++ 2 files changed, 98 insertions(+), 23 deletions(-) create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoaderTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoader.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoader.java index 29451148da447..7a0a41282824c 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoader.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoader.java @@ -25,7 +25,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.common.util.FutureUtil; @@ -46,6 +46,7 @@ public class MetadataStoreCacheLoader implements Closeable { private volatile List availableBrokers; private final FutureUtil.Sequencer sequencer; + private final AtomicBoolean refreshInProgress = new AtomicBoolean(false); private final OrderedScheduler orderedExecutor = OrderedScheduler.newSchedulerBuilder().numThreads(8) .name("pulsar-metadata-cache-loader-ordered-cache").build(); @@ -65,40 +66,38 @@ public MetadataStoreCacheLoader(PulsarResources pulsarResources, int operationTi * @throws Exception */ public void init() throws Exception { - Supplier> tryUpdate = () -> { - return loadReportResources.getChildrenAsync(LOADBALANCE_BROKERS_ROOT) - .thenComposeAsync(brokerNodes -> { - return updateBrokerList(brokerNodes).thenRun(() -> { - log.info("Successfully updated broker info {}", brokerNodes); - }); - }) - .exceptionally(ex -> { - log.warn("Error updating broker info after broker list changed", ex); - return null; - }); - }; loadReportResources.getStore().registerListener((n) -> { if (LOADBALANCE_BROKERS_ROOT.equals(n.getPath()) && NotificationType.ChildrenChanged.equals(n.getType())) { - sequencer.sequential(tryUpdate); + sequencer.sequential(this::reloadBrokers); } }); if (loadReportResources.getStore() instanceof MetadataStoreExtended) { ((MetadataStoreExtended) loadReportResources.getStore()).registerSessionListener(sessionEvent -> - sequencer.sequential(tryUpdate)); + sequencer.sequential(this::reloadBrokers)); } // Do initial fetch of brokers list - tryUpdate.get().get(operationTimeoutMs, TimeUnit.MILLISECONDS); + reloadBrokers().get(operationTimeoutMs, TimeUnit.MILLISECONDS); + } + + private CompletableFuture reloadBrokers() { + return loadReportResources.getChildrenAsync(LOADBALANCE_BROKERS_ROOT) + .thenComposeAsync(brokerNodes -> updateBrokerList(brokerNodes).thenRun(() -> + log.info("Successfully updated broker info {}", brokerNodes))) + .exceptionally(ex -> { + log.warn("Error updating broker info after broker list changed", ex); + return null; + }); } public List getAvailableBrokers() { - if (CollectionUtils.isEmpty(availableBrokers)) { - try { - updateBrokerList(loadReportResources.getChildren(LOADBALANCE_BROKERS_ROOT)); - } catch (Exception e) { - log.warn("Error updating broker from zookeeper.", e); - } + List brokers = availableBrokers; + if (CollectionUtils.isEmpty(brokers) && refreshInProgress.compareAndSet(false, true)) { + // Avoid blocking the caller (which may be a Netty IO thread): refresh the cache in the + // background and return the current snapshot. The cache is otherwise kept up to date by the + // metadata-store listener, so an empty snapshot means there are no active brokers. + sequencer.sequential(this::reloadBrokers).whenComplete((__, ex) -> refreshInProgress.set(false)); } - return availableBrokers; + return brokers == null ? new ArrayList<>() : brokers; } @Override diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoaderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoaderTest.java new file mode 100644 index 0000000000000..e8d4d28153c94 --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoaderTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.resources; + +import static org.apache.pulsar.broker.resources.MetadataStoreCacheLoader.LOADBALANCE_BROKERS_ROOT; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import lombok.Cleanup; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport; +import org.testng.annotations.Test; + +public class MetadataStoreCacheLoaderTest { + + private MetadataStoreCacheLoader newCacheLoader(LoadManagerReportResources loadReportResources) throws Exception { + MetadataStore store = mock(MetadataStore.class); + PulsarResources pulsarResources = mock(PulsarResources.class); + when(pulsarResources.getLoadReportResources()).thenReturn(loadReportResources); + when(loadReportResources.getStore()).thenReturn(store); + return new MetadataStoreCacheLoader(pulsarResources, 5000); + } + + @Test + public void testGetAvailableBrokersServesCacheWithoutBlocking() throws Exception { + LoadManagerReportResources loadReportResources = mock(LoadManagerReportResources.class); + LoadManagerReport report = mock(LoadManagerReport.class); + when(loadReportResources.getChildrenAsync(LOADBALANCE_BROKERS_ROOT)) + .thenReturn(CompletableFuture.completedFuture(List.of("broker-1"))); + when(loadReportResources.getAsync(LOADBALANCE_BROKERS_ROOT + "/broker-1")) + .thenReturn(CompletableFuture.completedFuture(Optional.of(report))); + + @Cleanup + MetadataStoreCacheLoader loader = newCacheLoader(loadReportResources); + + assertEquals(loader.getAvailableBrokers(), List.of(report)); + // The blocking, synchronous getChildren(...) must never be used: it could stall a Netty IO thread. + verify(loadReportResources, never()).getChildren(anyString()); + } + + @Test + public void testGetAvailableBrokersDoesNotBlockOnEmptyCache() throws Exception { + LoadManagerReportResources loadReportResources = mock(LoadManagerReportResources.class); + when(loadReportResources.getChildrenAsync(LOADBALANCE_BROKERS_ROOT)) + .thenReturn(CompletableFuture.completedFuture(List.of())); + + @Cleanup + MetadataStoreCacheLoader loader = newCacheLoader(loadReportResources); + + assertTrue(loader.getAvailableBrokers().isEmpty()); + verify(loadReportResources, never()).getChildren(anyString()); + } +} From 11bbb4328af4afd515b395567efdf4323722935c Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 18 Jun 2026 04:23:09 -0700 Subject: [PATCH 160/201] [fix][broker] Avoid blocking the dispatcher close path on delayed-delivery tracker close (#26053) (cherry picked from commit 82074aa80c5ca8d849b640a368c254e38e5163e2) --- .../delayed/DelayedDeliveryTracker.java | 10 +++++++ .../bucket/BucketDelayedDeliveryTracker.java | 27 ++++++++++++------- ...PersistentDispatcherMultipleConsumers.java | 8 +++--- ...entDispatcherMultipleConsumersClassic.java | 8 +++--- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java index 7c954879fe845..96e7151e7ea45 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java @@ -86,6 +86,16 @@ public interface DelayedDeliveryTracker extends AutoCloseable { */ void close(); + /** + * Close the subscription tracker and release all resources, completing the returned future once the + * tracker has finished closing. Prefer this over {@link #close()} on asynchronous paths (e.g. inside a + * {@link CompletableFuture} continuation) so the caller is not blocked. + */ + default CompletableFuture closeAsync() { + close(); + return CompletableFuture.completedFuture(null); + } + DelayedDeliveryTracker DISABLE = new DelayedDeliveryTracker() { @Override public boolean addMessage(long ledgerId, long entryId, long deliveryAt) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 67bc9d29360fc..69edf70dad786 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -728,17 +728,26 @@ public synchronized CompletableFuture clear() { } @Override - public synchronized void close() { - super.close(); - lastMutableBucket.close(); - sharedBucketPriorityQueue.close(); - try { - List> completableFutures = immutableBuckets.asMapOfRanges().values().stream() + public void close() { + // Block for AutoCloseable / synchronous callers; asynchronous callers should use closeAsync(). + closeAsync().join(); + } + + @Override + public CompletableFuture closeAsync() { + List> completableFutures; + synchronized (this) { + super.close(); + lastMutableBucket.close(); + sharedBucketPriorityQueue.close(); + completableFutures = immutableBuckets.asMapOfRanges().values().stream() .map(bucket -> bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE)).toList(); - FutureUtil.waitForAll(completableFutures).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (Exception e) { - log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e); } + return FutureUtil.waitForAll(completableFutures) + .exceptionally(e -> { + log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e); + return null; + }); } private CompletableFuture cleanImmutableBuckets() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index ce56ea3689d76..c855550069e09 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -621,11 +621,13 @@ public CompletableFuture close(boolean disconnectConsumers, this.delayedDeliveryTracker = Optional.empty(); } - delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::close); + CompletableFuture closeTrackerFuture = delayedDeliveryTracker + .map(DelayedDeliveryTracker::closeAsync) + .orElseGet(() -> CompletableFuture.completedFuture(null)); dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); - return disconnectConsumers - ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null); + return closeTrackerFuture.thenCompose(__ -> disconnectConsumers + ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null)); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java index 28dd52f89d818..59f2ce973dac9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java @@ -535,11 +535,13 @@ public CompletableFuture close(boolean disconnectConsumers, this.delayedDeliveryTracker = Optional.empty(); } - delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::close); + CompletableFuture closeTrackerFuture = delayedDeliveryTracker + .map(DelayedDeliveryTracker::closeAsync) + .orElseGet(() -> CompletableFuture.completedFuture(null)); dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); - return disconnectConsumers - ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null); + return closeTrackerFuture.thenCompose(__ -> disconnectConsumers + ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null)); } @Override From 8ae84a7c7570c82219633066c88ade8af290166f Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 18 Jun 2026 04:25:00 -0700 Subject: [PATCH 161/201] [fix][broker] Avoid blocking the bundle-throughput lookup on per-bundle metadata reads (#26054) (cherry picked from commit 561f373c1442a81364154aa9880fcb7ada9b89ef) --- .../loadbalance/ModularLoadManager.java | 14 +++ .../impl/ModularLoadManagerImpl.java | 89 ++++++++++--------- .../common/naming/NamespaceBundleFactory.java | 31 ++++--- .../impl/ModularLoadManagerImplTest.java | 2 +- .../namespace/NamespaceServiceTest.java | 2 +- 5 files changed, 84 insertions(+), 54 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java index d608bd6784f29..93059d722e734 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java @@ -138,8 +138,22 @@ default void writeBrokerDataOnZooKeeper(boolean force) { * * @param bundle * @return bundle data + * @deprecated use {@link #getBundleDataOrDefaultAsync(String)} instead; this method blocks on + * metadata-store reads and must not be called from async or event-loop threads. */ + @Deprecated BundleData getBundleDataOrDefault(String bundle); + /** + * Asynchronously fetch bundle's load report data. + * + * @param bundle + * @return future of the bundle data + */ + @SuppressWarnings("deprecation") + default CompletableFuture getBundleDataOrDefaultAsync(String bundle) { + return CompletableFuture.completedFuture(getBundleDataOrDefault(bundle)); + } + String setNamespaceBundleAffinity(String bundle, String broker); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index a9d7ddd78e07d..214d12e7ad2c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -381,46 +381,55 @@ private boolean checkBundleDataExistInNamespaceBundles(NamespaceBundles namespac // Attempt to local the data for the given bundle in metadata store // If it cannot be found, return the default bundle data. + /** + * @deprecated use {@link #getBundleDataOrDefaultAsync(String)} instead. + */ + @Deprecated @Override public BundleData getBundleDataOrDefault(final String bundle) { - BundleData bundleData = null; - try { - Optional optBundleData = - pulsarResources.getLoadBalanceResources().getBundleDataResources().getBundleData(bundle).join(); - if (optBundleData.isPresent()) { - return optBundleData.get(); - } + return getBundleDataOrDefaultAsync(bundle).join(); + } - Optional optQuota = pulsarResources.getLoadBalanceResources().getQuotaResources() - .getQuota(bundle).join(); - if (optQuota.isPresent()) { - ResourceQuota quota = optQuota.get(); - bundleData = new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES); - // Initialize from existing resource quotas if new API ZNodes do not exist. - final TimeAverageMessageData shortTermData = bundleData.getShortTermData(); - final TimeAverageMessageData longTermData = bundleData.getLongTermData(); - - shortTermData.setMsgRateIn(quota.getMsgRateIn()); - shortTermData.setMsgRateOut(quota.getMsgRateOut()); - shortTermData.setMsgThroughputIn(quota.getBandwidthIn()); - shortTermData.setMsgThroughputOut(quota.getBandwidthOut()); - - longTermData.setMsgRateIn(quota.getMsgRateIn()); - longTermData.setMsgRateOut(quota.getMsgRateOut()); - longTermData.setMsgThroughputIn(quota.getBandwidthIn()); - longTermData.setMsgThroughputOut(quota.getBandwidthOut()); - - // Assume ample history. - shortTermData.setNumSamples(NUM_SHORT_SAMPLES); - longTermData.setNumSamples(NUM_LONG_SAMPLES); - } - } catch (Exception e) { - log.warn("Error when trying to find bundle {} on metadata store: {}", bundle, e); - } - if (bundleData == null) { - bundleData = new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES, defaultStats); - } - return bundleData; + @Override + public CompletableFuture getBundleDataOrDefaultAsync(final String bundle) { + return pulsarResources.getLoadBalanceResources().getBundleDataResources().getBundleData(bundle) + .thenCompose(optBundleData -> { + if (optBundleData.isPresent()) { + return CompletableFuture.completedFuture(optBundleData.get()); + } + return pulsarResources.getLoadBalanceResources().getQuotaResources().getQuota(bundle) + .thenApply(optQuota -> { + if (optQuota.isEmpty()) { + return null; + } + ResourceQuota quota = optQuota.get(); + BundleData bundleData = new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES); + // Initialize from existing resource quotas if new API ZNodes do not exist. + final TimeAverageMessageData shortTermData = bundleData.getShortTermData(); + final TimeAverageMessageData longTermData = bundleData.getLongTermData(); + + shortTermData.setMsgRateIn(quota.getMsgRateIn()); + shortTermData.setMsgRateOut(quota.getMsgRateOut()); + shortTermData.setMsgThroughputIn(quota.getBandwidthIn()); + shortTermData.setMsgThroughputOut(quota.getBandwidthOut()); + + longTermData.setMsgRateIn(quota.getMsgRateIn()); + longTermData.setMsgRateOut(quota.getMsgRateOut()); + longTermData.setMsgThroughputIn(quota.getBandwidthIn()); + longTermData.setMsgThroughputOut(quota.getBandwidthOut()); + + // Assume ample history. + shortTermData.setNumSamples(NUM_SHORT_SAMPLES); + longTermData.setNumSamples(NUM_LONG_SAMPLES); + return bundleData; + }); + }) + .exceptionally(e -> { + log.warn("Error when trying to find bundle {} on metadata store: {}", bundle, e); + return null; + }) + .thenApply(bundleData -> bundleData != null ? bundleData + : new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES, defaultStats)); } // Use the Pulsar client to acquire the namespace bundle stats. @@ -561,7 +570,7 @@ private void updateBundleData() { } else { // Otherwise, attempt to find the bundle data on metadata store. // If it cannot be found, use the latest stats as the first sample. - BundleData currentBundleData = getBundleDataOrDefault(bundle); + BundleData currentBundleData = getBundleDataOrDefaultAsync(bundle).join(); currentBundleData.update(stats); bundleData.put(bundle, currentBundleData); } @@ -879,7 +888,7 @@ public Optional selectBrokerForAssignment(final ServiceUnitId serviceUni private void preallocateBundle(String bundle, String broker) { final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, - key -> getBundleDataOrDefault(bundle)); + key -> getBundleDataOrDefaultAsync(bundle).join()); loadData.getBrokerData().get(broker).getPreallocatedBundleData().put(bundle, data); preallocatedBundleToBroker.put(bundle, broker); @@ -893,7 +902,7 @@ Optional selectBroker(final ServiceUnitId serviceUnit) { synchronized (brokerCandidateCache) { final String bundle = serviceUnit.toString(); final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, - key -> getBundleDataOrDefault(bundle)); + key -> getBundleDataOrDefaultAsync(bundle).join()); brokerCandidateCache.clear(); LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, getAvailableBrokers(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java index bace2bf87a913..ef80e94a6bbd4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java @@ -56,6 +56,7 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.stats.CacheMetricsCollector; import org.apache.pulsar.common.util.Backoff; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.policies.data.loadbalancer.BundleData; import org.slf4j.Logger; @@ -246,19 +247,25 @@ public NamespaceBundle getBundle(TopicName topic) { public CompletableFuture getBundleWithHighestThroughputAsync(NamespaceName nsName) { LoadManager loadManager = pulsar.getLoadManager().get(); if (loadManager instanceof ModularLoadManagerWrapper) { - return getBundlesAsync(nsName).thenApply(bundles -> { - double maxMsgThroughput = -1; - NamespaceBundle bundleWithHighestThroughput = null; - for (NamespaceBundle bundle : bundles.getBundles()) { - BundleData bundleData = ((ModularLoadManagerWrapper) loadManager).getLoadManager() - .getBundleDataOrDefault(bundle.toString()); - if (bundleData.getTopics() > 0 - && bundleData.getLongTermData().totalMsgThroughput() > maxMsgThroughput) { - maxMsgThroughput = bundleData.getLongTermData().totalMsgThroughput(); - bundleWithHighestThroughput = bundle; + return getBundlesAsync(nsName).thenCompose(bundles -> { + List bundleList = bundles.getBundles(); + List> bundleDataFutures = bundleList.stream() + .map(bundle -> ((ModularLoadManagerWrapper) loadManager).getLoadManager() + .getBundleDataOrDefaultAsync(bundle.toString())) + .toList(); + return FutureUtil.waitForAll(bundleDataFutures).thenApply(__ -> { + double maxMsgThroughput = -1; + NamespaceBundle bundleWithHighestThroughput = null; + for (int i = 0; i < bundleList.size(); i++) { + BundleData bundleData = bundleDataFutures.get(i).join(); + if (bundleData.getTopics() > 0 + && bundleData.getLongTermData().totalMsgThroughput() > maxMsgThroughput) { + maxMsgThroughput = bundleData.getLongTermData().totalMsgThroughput(); + bundleWithHighestThroughput = bundleList.get(i); + } } - } - return bundleWithHighestThroughput; + return bundleWithHighestThroughput; + }); }); } return getBundleWithHighestTopicsAsync(nsName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java index 1bf40fab1a111..a245db16da844 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java @@ -1016,7 +1016,7 @@ public void testBundleDataDefaultValue() throws Exception { // get the bundleData of the first bundle range. // The default value of the bundleData be the same as resourceQuota because the resourceQuota is present. - BundleData defaultBundleData = lm.getBundleDataOrDefault(namespaceBundle.toString()); + BundleData defaultBundleData = lm.getBundleDataOrDefaultAsync(namespaceBundle.toString()).join(); TimeAverageMessageData shortTermData = defaultBundleData.getShortTermData(); TimeAverageMessageData longTermData = defaultBundleData.getLongTermData(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index a80bdbea51074..1f62718461d90 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -690,7 +690,7 @@ public void testSplitBundleWithHighestThroughput() throws Exception { LoadManager loadManager = pulsar.getLoadManager().get(); Awaitility.await().untilAsserted(() -> { BundleData targetBundleData = ((ModularLoadManagerWrapper) loadManager).getLoadManager() - .getBundleDataOrDefault(namespace + "/" + bundle); + .getBundleDataOrDefaultAsync(namespace + "/" + bundle).join(); assertEquals(targetBundleData.getTopics(), 10); }); From fdf14ee33d94f95dff52c1d40c30ccb2fc446774 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 20 Jun 2026 13:51:53 +0300 Subject: [PATCH 162/201] [fix][sec] Upgrade jline to 4.2.1 and picocli to 4.7.7, drop unused jline2 (#26068) (cherry picked from commit c536dcbd4ce598123dbbb967d1d37788c9a0b8a8) --- distribution/server/pom.xml | 6 ------ distribution/server/src/assemble/LICENSE.bin.txt | 7 +++---- distribution/shell/src/assemble/LICENSE.bin.txt | 6 +++--- pom.xml | 5 ++--- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 2703055d64c4c..a6d53e7c5b9d8 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -93,12 +93,6 @@ ${project.version} - - jline - jline - ${jline.version} - - ${project.groupId} pulsar-package-bookkeeper-storage diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 8aaf5c626c029..b42107c28918f 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -245,8 +245,8 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.82.jar * Picocli - - info.picocli-picocli-4.7.5.jar - - info.picocli-picocli-shell-jline3-4.7.5.jar + - info.picocli-picocli-4.7.7.jar + - info.picocli-picocli-shell-jline3-4.7.7.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - com.fasterxml.jackson.core-jackson-annotations-2.18.6.jar @@ -567,8 +567,7 @@ BSD 3-clause "New" or "Revised" License * LevelDB -- (included in org.rocksdb.*.jar) -- ../licenses/LICENSE-LevelDB.txt * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt * JSR305 -- jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt - * JLine -- jline-jline-2.14.6.jar -- ../licenses/LICENSE-JLine.txt - * JLine3 -- org.jline-jline-3.21.0.jar -- ../licenses/LICENSE-JLine.txt + * JLine3 -- org.jline-jline-4.2.1.jar -- ../licenses/LICENSE-JLine.txt * OW2 ASM - org.ow2.asm-asm-9.10.jar -- ../licenses/LICENSE-ASM.txt - org.ow2.asm-asm-commons-9.10.jar -- ../licenses/LICENSE-ASM.txt diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 9bb4a16dc8935..27c639f7877a5 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -310,8 +310,8 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Picocli - - picocli-4.7.5.jar - - picocli-shell-jline3-4.7.5.jar + - picocli-4.7.7.jar + - picocli-shell-jline3-4.7.7.jar * Jackson - jackson-annotations-2.18.6.jar - jackson-core-2.18.6.jar @@ -426,7 +426,7 @@ The Apache Software License, Version 2.0 * JSpecify -- jspecify-1.0.0.jar BSD 3-clause "New" or "Revised" License - * JLine3 -- jline-3.21.0.jar -- ../licenses/LICENSE-JLine.txt + * JLine3 -- jline-4.2.1.jar -- ../licenses/LICENSE-JLine.txt * JSR305 -- jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt MIT License diff --git a/pom.xml b/pom.xml index 84967564b449d..601b43328d543 100644 --- a/pom.xml +++ b/pom.xml @@ -274,8 +274,7 @@ flexible messaging model and an intuitive client API. 3.1.0 3.2.4 0.9.0 - 2.14.6 - 3.21.0 + 4.2.1 0.9.1 2.1.0 3.27.7 @@ -314,7 +313,7 @@ flexible messaging model and an intuitive client API. 2.21.0 ${opentelemetry.instrumentation.version}-alpha 1.37.0 - 4.7.5 + 4.7.7 1.8 0.3.6 3.3.2 From 3e6a848d9c71cb8248813ee50160e04ab9177ee8 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sat, 20 Jun 2026 14:00:29 -0700 Subject: [PATCH 163/201] [fix] functions: Run worker leader-election off the consumer event-listener thread (#26059) (cherry picked from commit ced1280a108709b9fddbb4f8a161fd288c3e2565) --- .../functions/worker/LeaderService.java | 28 ++++++++++++++++++- .../functions/worker/LeaderServiceTest.java | 8 ++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java index e7816f06aacc8..d15b917463550 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java @@ -18,6 +18,11 @@ */ package org.apache.pulsar.functions.worker; +import com.google.common.annotations.VisibleForTesting; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Consumer; @@ -42,6 +47,11 @@ public class LeaderService implements AutoCloseable, ConsumerEventListener { private final WorkerConfig workerConfig; private final PulsarClient pulsarClient; private volatile boolean isLeader = false; + // The consumer event listener callbacks (becameActive/becameInactive) run the blocking + // leader-election routines on this dedicated single-threaded executor so that the Pulsar client's + // shared consumer-listener thread is not blocked. The single thread also preserves event ordering. + private final ExecutorService executor = + Executors.newSingleThreadExecutor(new DefaultThreadFactory("function-worker-leader")); static final String COORDINATION_TOPIC_SUBSCRIPTION = "participants"; @@ -89,6 +99,12 @@ public void start() throws PulsarClientException { @Override public void becameActive(Consumer consumer, int partitionId) { + // Run the (blocking) become-leader routine on a dedicated executor so the consumer + // event-listener thread, which is shared with the Pulsar client, is not blocked. + executor.execute(() -> becameActiveInternal(consumer, partitionId)); + } + + private void becameActiveInternal(Consumer consumer, int partitionId) { synchronized (this) { if (isLeader) { return; @@ -148,7 +164,11 @@ public void becameActive(Consumer consumer, int partitionId) { } @Override - public synchronized void becameInactive(Consumer consumer, int partitionId) { + public void becameInactive(Consumer consumer, int partitionId) { + executor.execute(() -> becameInactiveInternal(consumer, partitionId)); + } + + private synchronized void becameInactiveInternal(Consumer consumer, int partitionId) { if (isLeader) { log.info("Worker {} lost the leadership.", consumerName); isLeader = false; @@ -176,10 +196,16 @@ public boolean isLeader() { return isLeader; } + @VisibleForTesting + void joinPendingEventTasks() throws InterruptedException, ExecutionException { + executor.submit(() -> { }).get(); + } + @Override public void close() throws PulsarClientException { if (consumer != null) { consumer.close(); } + executor.shutdown(); } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java index 5c10a59bd1388..03cbd772ab2e6 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java @@ -135,6 +135,7 @@ public void testLeaderService() throws Exception { verify(mockClient, times(1)).newConsumer(); listenerHolder.get().becameActive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertTrue(leaderService.isLeader()); verify(functionMetadataManager, times(1)).getIsInitialized(); @@ -150,6 +151,7 @@ public void testLeaderService() throws Exception { verify(schedulerManager, times((1))).initialize(any()); listenerHolder.get().becameInactive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertFalse(leaderService.isLeader()); verify(functionAssignmentTailer, times(1)).startFromMessage(messageId); @@ -165,6 +167,7 @@ public void testLeaderServiceNoNewScheduling() throws Exception { verify(mockClient, times(1)).newConsumer(); listenerHolder.get().becameActive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertTrue(leaderService.isLeader()); verify(functionMetadataManager, times(1)).acquireExclusiveWrite(any()); @@ -176,6 +179,7 @@ public void testLeaderServiceNoNewScheduling() throws Exception { verify(schedulerManager, times((1))).initialize(any()); listenerHolder.get().becameInactive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertFalse(leaderService.isLeader()); verify(functionAssignmentTailer, times(1)).start(); @@ -195,6 +199,7 @@ public void testAcquireScheduleManagerExclusiveProducerNotLeaderAnymore() throws when(schedulerManager.acquireExclusiveWrite(any())).thenThrow(new WorkerUtils.NotLeaderAnymore()); listenerHolder.get().becameActive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); // should have failed to become leader assertFalse(leaderService.isLeader()); @@ -211,6 +216,7 @@ public void testAcquireScheduleManagerExclusiveProducerNotLeaderAnymore() throws verify(schedulerManager, times((0))).initialize(any()); listenerHolder.get().becameInactive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertFalse(leaderService.isLeader()); verify(functionAssignmentTailer, times(0)).startFromMessage(messageId); @@ -231,6 +237,7 @@ public void testAcquireFunctionMetadataManagerExclusiveProducerNotLeaderAnymore( when(functionMetadataManager.acquireExclusiveWrite(any())).thenThrow(new WorkerUtils.NotLeaderAnymore()); listenerHolder.get().becameActive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); // should have failed to become leader assertFalse(leaderService.isLeader()); @@ -247,6 +254,7 @@ public void testAcquireFunctionMetadataManagerExclusiveProducerNotLeaderAnymore( verify(schedulerManager, times((0))).initialize(any()); listenerHolder.get().becameInactive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertFalse(leaderService.isLeader()); verify(functionAssignmentTailer, times(0)).startFromMessage(messageId); From dff997331bdb935cd8950e3238651c5f5d61724c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 22 Jun 2026 14:21:11 +0300 Subject: [PATCH 164/201] [fix][build][branch-4.2] Upgrade docker/setup-qemu-action to v4.1.0 - previous approved hash was removed by https://github.com/apache/infrastructure-actions/commit/8d7b8628 --- .github/workflows/pulsar-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index f8cb682896c08..bc572f31de8a1 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -588,7 +588,7 @@ jobs: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - name: Set up QEMU - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 + uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0 with: platforms: arm64 From 323ed2366e52ca840774386a80f5b9cd679deae8 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Tue, 23 Jun 2026 04:16:10 +0800 Subject: [PATCH 165/201] [fix][broker] Fix geo-replication stuck after a failed publish to the remote cluster (#26002) (cherry picked from commit 9b9e67a9d710f6e4cc4d0665a6cd8dab24c45973) --- .../persistent/PersistentReplicator.java | 3 ++ .../PersistentReplicatorInflightTaskTest.java | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index b125bd0306bdd..565efdc456cf7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -414,6 +414,9 @@ public void sendComplete(Throwable exception, OpSendMsgStats opSendMsgStats) { // cursor should be rewound since it was incremented when readMoreEntries replicator.beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding.Failed_Publishing); replicator.doRewindCursor(false); + // The failed send has completed from the producer queue perspective. The cursor rewind + // makes the entry readable again, so this in-flight task must release its permit. + inFlightTask.incCompletedEntries(); } else { if (log.isDebugEnabled()) { log.debug("[{}] Message persisted on remote broker", replicator.replicatorId, exception); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java index c825d49caa62e..0c22b14acf748 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service.persistent; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; @@ -43,8 +44,11 @@ import org.apache.pulsar.broker.service.BrokerServiceInternalMethodInvoker; import org.apache.pulsar.broker.service.OneWayReplicatorTestBase; import org.apache.pulsar.broker.service.persistent.PersistentReplicator.InFlightTask; +import org.apache.pulsar.broker.service.persistent.PersistentReplicator.ProducerSendCallback; +import org.apache.pulsar.broker.service.persistent.PersistentReplicator.ReasonOfWaitForCursorRewinding; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.awaitility.Awaitility; import org.mockito.invocation.InvocationOnMock; @@ -167,6 +171,34 @@ public void testReadEntriesFailedCompletesInFlightTaskAfterReplicatorTerminated( } } + @Test + public void testFailedPublishCompletesInFlightTask() throws Exception { + PersistentReplicator replicator = spy(getReplicator(topicName)); + doNothing().when(replicator).beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding.Failed_Publishing); + doNothing().when(replicator).doRewindCursor(false); + doNothing().when(replicator).readMoreEntries(); + + LinkedList inFlightTasks = replicator.inFlightTasks; + List originalTasks = new ArrayList<>(inFlightTasks); + inFlightTasks.clear(); + + try { + InFlightTask task = new InFlightTask(PositionFactory.create(1, 1), 1, replicator.getReplicatorId()); + task.setEntries(Collections.singletonList(mock(Entry.class))); + inFlightTasks.add(task); + assertEquals(replicator.getPermitsIfNoPendingRead(), 999); + + ProducerSendCallback callback = ProducerSendCallback.create(replicator, mock(Entry.class), null, task); + callback.sendComplete(new PulsarClientException.ProducerBlockedQuotaExceededException("mocked"), null); + + assertTrue(task.isDone()); + assertEquals(replicator.getPermitsIfNoPendingRead(), 1000); + } finally { + inFlightTasks.clear(); + inFlightTasks.addAll(originalTasks); + } + } + @Test public void testCreateOrRecycleInFlightTaskIntoQueue() throws Exception { log.info("Starting testCreateOrRecycleInFlightTaskIntoQueue"); From 6c68d9c61f6b4f49bc8854cf11a8b9fdd07d3a64 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 19 Jun 2026 02:41:53 +0800 Subject: [PATCH 166/201] [improve][test]Add test: test/testTopicPartitionCannotBeCreatedAfterTopicDeleted (#26038) (cherry picked from commit 8eaf5517ff788d0703f75f5deb45ac5b8edb0635) --- .../api/SimpleProducerConsumerTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index d5b10fb8347f2..dcafd157a9084 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -110,6 +110,7 @@ import org.apache.pulsar.client.impl.PartitionedProducerImpl; import org.apache.pulsar.client.impl.ProducerBase; import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.TopicMessageImpl; import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; @@ -5438,6 +5439,65 @@ public void testBacklogAfterCreatedSubscription(boolean trimLegderBeforeGetStats admin.topics().delete(topic, false); } + @Test + public void testTopicPartitionCannotBeCreatedAfterTopicDeleted() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createPartitionedTopic(topic, 1); + + // Inject an delay: delay to handle channel inactive event, to let the producer delay to reconnect. + ClientBuilderImpl clientBuilder = (ClientBuilderImpl) PulsarClient.builder() + .serviceUrl(lookupUrl.toString()) + .statsInterval(0, TimeUnit.SECONDS) + .connectionsPerBroker(1); + CountDownLatch countDownLatch = new CountDownLatch(1); + PulsarClientImpl pulsarClient = InjectedClientCnxClientBuilder.create(clientBuilder, (conf, eventLoopGroup) -> { + + return new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup) { + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + // Delay receiving the event, let producer will not reconnect immediately. + log.info("channel inactive"); + countDownLatch.await(); + super.channelInactive(ctx); + } + }; + }); + + // Producer connected. + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .create(); + PersistentTopic persistentTopic1 = (PersistentTopic) pulsar.getBrokerService() + .getTopic(topic + "-partition-0", false) + .get(5, TimeUnit.SECONDS).get(); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(persistentTopic1.getProducers().values().size(), 1); + }); + + // Make a network issue which leads to the connection breaks. + org.apache.pulsar.broker.service.Producer serviceProducer = persistentTopic1.getProducers().values() + .iterator().next(); + ServerCnx servercnx = (ServerCnx) serviceProducer.getCnx(); + servercnx.ctx().close(); + // After the connection is break, and before the producer reconnects, the partitioned topic can be deleted + // without "--force". + admin.topics().deletePartitionedTopic(topic); + Awaitility.await().untilAsserted(() -> { + assertTrue(persistentTopic1.isClosingOrDeleting()); + }); + + // Verify: the partition can not be loaded up once the partitioned topic was deleted. + countDownLatch.countDown(); + Thread.sleep(10_000); + assertFalse(producer.isConnected()); + assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic + "-partition-0")); + + // cleanup. + producer.close(); + pulsarClient.close(); + } + /** * The internal producer of replicator will resend messages after reconnected. This test guarantees that the * internal producer will continuously resent messages even though the client side encounters the following bugs. From ec447ab88ef91bddeb63e4d240878b55fb0cf300 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 5 May 2026 14:22:56 -0700 Subject: [PATCH 167/201] [fix][test] Make SameAuthParamsLookupAutoClusterFailoverTest less timing-sensitive (#25675) (cherry picked from commit 17c6b139915b4cc299d29a071839a128479f68c9) --- ...thParamsLookupAutoClusterFailoverTest.java | 89 +++++++++---------- 1 file changed, 40 insertions(+), 49 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java index a2e587fef395f..96f1e6421fa4b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -63,11 +63,11 @@ public Object[][] enabledTls () { }; } - // Each state-convergence phase below waits up to 2 minutes. With 3 phases plus - // cluster startup/teardown, the 240s overall timeout can be tight on slow CI agents - // (the probe timeout is 3s and recoverThreshold=5, so a single slow probe can - // stretch a phase to ~30s). Allow 8 minutes overall to match the per-phase budget. - @Test(dataProvider = "enabledTls", timeOut = 480 * 1000) + // Each state-convergence phase below waits up to 3 minutes. The probe timeout is 3s + // and recoverThreshold=5, so a transient probe failure during recovery resets the + // counter and a phase can need ~30s of healthy probes to recover. With 3 phases plus + // cluster startup/teardown, allow 12 minutes overall to absorb slow CI agents. + @Test(dataProvider = "enabledTls", timeOut = 720 * 1000) public void testAutoClusterFailover(boolean enabledTls) throws Exception { // Start clusters. setup(); @@ -108,63 +108,30 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { producer.send("0"); Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 0); - CompletableFuture checkStatesFuture1 = new CompletableFuture<>(); - executor.submit(() -> { - boolean res = stateArray[0] == PulsarServiceState.Healthy; - res = res & stateArray[1] == PulsarServiceState.Healthy; - res = res & stateArray[2] == PulsarServiceState.Healthy; - checkStatesFuture1.complete(res); - }); - Assert.assertTrue(checkStatesFuture1.join()); + assertStatesEqual(executor, stateArray, + PulsarServiceState.Healthy, PulsarServiceState.Healthy, PulsarServiceState.Healthy); // Test failover 0 --> 2. pulsar1.close(); - Awaitility.await().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { - CompletableFuture checkStatesFuture2 = new CompletableFuture<>(); - executor.submit(() -> { - boolean res = stateArray[0] == PulsarServiceState.Failed; - res = res & stateArray[1] == PulsarServiceState.Failed; - res = res & stateArray[2] == PulsarServiceState.Healthy; - checkStatesFuture2.complete(res); - }); - Assert.assertTrue(checkStatesFuture2.join()); - producer.send("0->2"); - Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 2); - }); + awaitStatesAndIndex(executor, stateArray, failover, 2, + PulsarServiceState.Failed, PulsarServiceState.Failed, PulsarServiceState.Healthy); + producer.send("0->2"); // Test recover 2 --> 1. executor.execute(() -> { urlArray[1] = url2; }); - Awaitility.await().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { - CompletableFuture checkStatesFuture3 = new CompletableFuture<>(); - executor.submit(() -> { - boolean res = stateArray[0] == PulsarServiceState.Failed; - res = res & stateArray[1] == PulsarServiceState.Healthy; - res = res & stateArray[2] == PulsarServiceState.Healthy; - checkStatesFuture3.complete(res); - }); - Assert.assertTrue(checkStatesFuture3.join()); - producer.send("2->1"); - Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 1); - }); + awaitStatesAndIndex(executor, stateArray, failover, 1, + PulsarServiceState.Failed, PulsarServiceState.Healthy, PulsarServiceState.Healthy); + producer.send("2->1"); // Test recover 1 --> 0. executor.execute(() -> { urlArray[0] = url2; }); - Awaitility.await().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { - CompletableFuture checkStatesFuture4 = new CompletableFuture<>(); - executor.submit(() -> { - boolean res = stateArray[0] == PulsarServiceState.Healthy; - res = res & stateArray[1] == PulsarServiceState.Healthy; - res = res & stateArray[2] == PulsarServiceState.Healthy; - checkStatesFuture4.complete(res); - }); - Assert.assertTrue(checkStatesFuture4.join()); - producer.send("1->0"); - Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 0); - }); + awaitStatesAndIndex(executor, stateArray, failover, 0, + PulsarServiceState.Healthy, PulsarServiceState.Healthy, PulsarServiceState.Healthy); + producer.send("1->0"); // cleanup. producer.close(); @@ -186,6 +153,30 @@ public void testInitializeCanOnlyBeCalledOnce() throws Exception { } } + /** + * Wait for the state machine to converge to the expected per-index states and current index. + * The state read happens on the failover executor to avoid races with the periodic check task, + * and producer/lookup operations are kept out of the polling loop so a slow message send does + * not consume the convergence budget. + */ + private static void awaitStatesAndIndex(EventLoopGroup executor, PulsarServiceState[] stateArray, + SameAuthParamsLookupAutoClusterFailover failover, + int expectedIndex, + PulsarServiceState... expectedStates) { + Awaitility.await().atMost(180, TimeUnit.SECONDS).untilAsserted(() -> { + assertStatesEqual(executor, stateArray, expectedStates); + Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), expectedIndex); + }); + } + + private static void assertStatesEqual(EventLoopGroup executor, PulsarServiceState[] stateArray, + PulsarServiceState... expected) throws Exception { + CompletableFuture snapshot = new CompletableFuture<>(); + executor.submit(() -> snapshot.complete(stateArray.clone())); + PulsarServiceState[] actual = snapshot.get(10, TimeUnit.SECONDS); + Assert.assertEquals(actual, expected); + } + @Override protected void cleanupPulsarResources() { // Nothing to do. From 46c8b643a34c82245c2fd617c9432e897b150f89 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 22 Jun 2026 07:11:03 -0700 Subject: [PATCH 168/201] [fix][client] Run the failover health probe off the Netty event-loop thread (#26064) (cherry picked from commit f4fc00c197ffaf09a42f9cb0ab0d89515a88a61e) --- ...meAuthParamsLookupAutoClusterFailoverTest.java | 8 ++++---- .../SameAuthParamsLookupAutoClusterFailover.java | 15 +++++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java index 96f1e6421fa4b..eb091ff9f1f4f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -21,11 +21,11 @@ import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.CA_CERT_FILE_PATH; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.getTlsFileForClient; import static org.apache.pulsar.client.impl.SameAuthParamsLookupAutoClusterFailover.PulsarServiceState; -import io.netty.channel.EventLoopGroup; import java.net.ServerSocket; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.service.NetworkErrorTestBase; import org.apache.pulsar.broker.service.OneWayReplicatorTestBase; @@ -98,7 +98,7 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { .tlsTrustCertsFilePath(CA_CERT_FILE_PATH); } final PulsarClient client = clientBuilder.build(); - final EventLoopGroup executor = WhiteboxImpl.getInternalState(failover, "executor"); + final ScheduledExecutorService executor = WhiteboxImpl.getInternalState(failover, "executor"); final PulsarServiceState[] stateArray = WhiteboxImpl.getInternalState(failover, "pulsarServiceStateArray"); @@ -159,7 +159,7 @@ public void testInitializeCanOnlyBeCalledOnce() throws Exception { * and producer/lookup operations are kept out of the polling loop so a slow message send does * not consume the convergence budget. */ - private static void awaitStatesAndIndex(EventLoopGroup executor, PulsarServiceState[] stateArray, + private static void awaitStatesAndIndex(ScheduledExecutorService executor, PulsarServiceState[] stateArray, SameAuthParamsLookupAutoClusterFailover failover, int expectedIndex, PulsarServiceState... expectedStates) { @@ -169,7 +169,7 @@ private static void awaitStatesAndIndex(EventLoopGroup executor, PulsarServiceSt }); } - private static void assertStatesEqual(EventLoopGroup executor, PulsarServiceState[] stateArray, + private static void assertStatesEqual(ScheduledExecutorService executor, PulsarServiceState[] stateArray, PulsarServiceState... expected) throws Exception { CompletableFuture snapshot = new CompletableFuture<>(); executor.submit(() -> snapshot.complete(stateArray.clone())); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java index 69c74bd28d92a..b137e37996efa 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java @@ -19,10 +19,11 @@ package org.apache.pulsar.client.impl; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import io.netty.channel.EventLoopGroup; -import io.netty.util.concurrent.ScheduledFuture; import java.util.Arrays; import java.util.HashSet; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -35,7 +36,6 @@ import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.netty.EventLoopUtil; /** * A service URL provider that probes multiple Pulsar service URLs with the same authentication @@ -50,7 +50,7 @@ public class SameAuthParamsLookupAutoClusterFailover implements ServiceUrlProvider { private PulsarClientImpl pulsarClient; - private EventLoopGroup executor; + private ScheduledExecutorService executor; private volatile boolean closed; private ScheduledFuture scheduledCheckTask; @Getter @@ -79,9 +79,12 @@ public synchronized void initialize(PulsarClient client) { } this.currentPulsarServiceIndex = 0; this.pulsarClient = (PulsarClientImpl) client; - this.executor = EventLoopUtil.newEventLoopGroup(1, false, + this.executor = Executors.newSingleThreadScheduledExecutor( new ExecutorProvider.ExtendedThreadFactory("broker-service-url-check")); - scheduledCheckTask = executor.scheduleAtFixedRate(() -> { + // Use fixed-delay (not fixed-rate) scheduling: a probe can block up to its timeout, and with a + // plain single-threaded scheduled executor fixed-rate runs would otherwise pile up back-to-back + // and monopolize the thread. Fixed-delay leaves a gap after each check completes. + scheduledCheckTask = executor.scheduleWithFixedDelay(() -> { try { if (closed) { return; From 951f39846ada6675d3ea31a72a7ca390165f16b9 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 22 Jun 2026 07:12:54 -0700 Subject: [PATCH 169/201] [fix][meta] Run ledger-underreplication notification callbacks off the metadata-store listener thread (#26065) (cherry picked from commit cc96d1c385b0bc8e930579adb9aea742feedd191) --- .../PulsarLedgerUnderreplicationManager.java | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java index bfd9171a391aa..a299f664e3763 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java @@ -30,6 +30,7 @@ import com.google.common.base.Joiner; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.TextFormat; +import io.netty.util.concurrent.DefaultThreadFactory; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; @@ -45,6 +46,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeoutException; import java.util.function.Predicate; import java.util.regex.Matcher; @@ -118,6 +121,11 @@ long getLedgerNodeVersion() { private final List> lostBookieRecoveryDelayCallbacks = new ArrayList<>(); + // Registered callbacks can perform synchronous metadata-store reads, so run them on a dedicated + // single-threaded executor instead of the metadata-store notification thread (and outside the lock). + private final ExecutorService notificationCallbackExecutor = + Executors.newSingleThreadExecutor(new DefaultThreadFactory("pulsar-underreplication-notification")); + private static class PulsarUnderreplicatedLedger extends UnderreplicatedLedger { PulsarUnderreplicatedLedger(long ledgerId) { super(ledgerId); @@ -249,13 +257,15 @@ private void handleNotification(Notification n) { callbackList = new ArrayList<>(lostBookieRecoveryDelayCallbacks); lostBookieRecoveryDelayCallbacks.clear(); } - for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { - try { - callback.operationComplete(0, null); - } catch (Exception e) { - log.warn("lostBookieRecoveryDelayCallbacks handle error", e); + notificationCallbackExecutor.execute(() -> { + for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { + try { + callback.operationComplete(0, null); + } catch (Exception e) { + log.warn("lostBookieRecoveryDelayCallbacks handle error", e); + } } - } + }); return; } if (replicationDisablePath.equals(n.getPath()) && n.getType() == NotificationType.Deleted) { @@ -266,13 +276,15 @@ private void handleNotification(Notification n) { callbackList = new ArrayList<>(replicationEnabledCallbacks); replicationEnabledCallbacks.clear(); } - for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { - try { - callback.operationComplete(0, null); - } catch (Exception e) { - log.warn("replicationEnabledCallbacks handle error", e); + notificationCallbackExecutor.execute(() -> { + for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { + try { + callback.operationComplete(0, null); + } catch (Exception e) { + log.warn("replicationEnabledCallbacks handle error", e); + } } - } + }); } } } @@ -690,6 +702,7 @@ public void close() throws ReplicationException.UnavailableException { if (log.isDebugEnabled()) { log.debug("close()"); } + notificationCallbackExecutor.shutdownNow(); try { for (Map.Entry e : heldLocks.entrySet()) { store.delete(e.getValue().getLockPath(), Optional.empty()) From 515e8d5bdcf68fc3f3508e0aa2972ac09f650425 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 23 Jun 2026 01:04:48 +0300 Subject: [PATCH 170/201] [fix][broker] Avoid attaching a consumer to a migrated non-persistent topic on subscribe (#26075) (cherry picked from commit d351b6b223be299448cc6512714bd518d4e5f493) --- .../nonpersistent/NonPersistentTopic.java | 66 ++++++++------ .../nonpersistent/NonPersistentTopicTest.java | 89 +++++++++++++++++++ 2 files changed, 127 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 5de7585d943f1..bba7bafed0ed0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -338,37 +338,47 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, DEFAULT_CONSUMER_EPOCH, schemaType); - if (isMigrated()) { - getMigratedClusterUrlAsync().thenAccept(consumer::topicMigrated); - } + consumer.checkAndApplyTopicMigrationAsync().thenCompose(migrated -> { + if (migrated) { + // The topic is migrated: checkAndApplyTopicMigrationAsync() has already sent the + // TopicMigrated redirect and disconnected the consumer (which also released the usage + // count taken by handleConsumerAdded above). Skip addConsumerToSubscription so the consumer + // is never attached to the subscription on the old cluster. Sequencing the migration check + // through thenCompose (instead of the previous fire-and-forget thenAccept that raced with + // addConsumerToSubscription) is what guarantees the consumer is not added once migrated. + log.info("[{}][{}] Skipped subscription on migrated topic; consumer {} was redirected", + topic, subscriptionName, consumerId); + future.complete(consumer); + return CompletableFuture.completedFuture(null); + } + return addConsumerToSubscription(subscription, consumer).thenRun(() -> { + if (!cnx.isActive()) { + try { + consumer.close(); + } catch (BrokerServiceException e) { + if (e instanceof ConsumerBusyException) { + log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, + consumerId, consumerName); + } else if (e instanceof SubscriptionBusyException) { + log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); + } - addConsumerToSubscription(subscription, consumer).thenRun(() -> { - if (!cnx.isActive()) { - try { - consumer.close(); - } catch (BrokerServiceException e) { - if (e instanceof ConsumerBusyException) { - log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, - consumerName); - } else if (e instanceof SubscriptionBusyException) { - log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); + decrementUsageCount(); + future.completeExceptionally(e); + return; } - - decrementUsageCount(); - future.completeExceptionally(e); - return; - } - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, - consumer.consumerName(), currentUsageCount()); + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, + consumer.consumerName(), currentUsageCount()); + } + future.completeExceptionally( + new BrokerServiceException.ConnectionClosedException( + "Connection was closed while the opening the cursor ")); + } else { + log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId); + future.complete(consumer); } - future.completeExceptionally( - new BrokerServiceException.ConnectionClosedException( - "Connection was closed while the opening the cursor ")); - } else { - log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId); - future.complete(consumer); - } + }); }).exceptionally(e -> { Throwable throwable = e.getCause(); if (throwable instanceof ConsumerBusyException) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java index 12b6cd2761a19..d09f3068c2f09 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import java.lang.reflect.Field; +import java.util.Collections; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -30,7 +31,9 @@ import lombok.Cleanup; import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.PulsarCommandSender; import org.apache.pulsar.broker.service.SubscriptionOption; +import org.apache.pulsar.broker.service.TransportCnx; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -39,7 +42,9 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionMode; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; import org.mockito.Mockito; @@ -126,6 +131,90 @@ public void testCreateNonExistentPartitions() throws PulsarAdminException { } + /** + * Regression test for the migration-redirect race in {@code NonPersistentTopic.internalSubscribe} + * (introduced by PR #26051, which turned a blocking, ordered migration redirect into a fire-and-forget + * async one). When the topic is migrated, the migration check must complete before + * {@code addConsumerToSubscription}, and the consumer must NOT be attached to the subscription on the + * old cluster. The previous code ran {@code getMigratedClusterUrlAsync().thenAccept(consumer::topicMigrated)} + * concurrently with {@code addConsumerToSubscription}, so the consumer could be added before the redirect + * and disconnect ran. The fix sequences the check through {@link org.apache.pulsar.broker.service.Consumer + * #checkAndApplyTopicMigrationAsync()} and skips the add when migrated. + */ + @Test + public void testSubscribeOnMigratedTopicSkipsAddingConsumer() throws Exception { + final String topicName = "non-persistent://prop/ns-abc/migration-race-" + UUID.randomUUID(); + final String subName = "migration-sub"; + + // Materialize the real topic on the broker and acquire namespace-bundle ownership via a client lookup. + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + NonPersistentTopic realTopic = + (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); + + // Mark the local cluster as migrated so AbstractTopic.getMigratedClusterUrlAsync() (used by + // Consumer.checkAndApplyTopicMigrationAsync()) resolves to a present migrated-cluster URL. + ClusterUrl migratedUrl = new ClusterUrl("http://migrated:8080", "https://migrated:8443", + "pulsar://migrated:6650", "pulsar+ssl://migrated:6651"); + admin.clusters().updateClusterMigration(conf.getClusterName(), true, migratedUrl); + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> + AbstractTopic.getMigratedClusterUrlAsync(pulsar, topicName).get().isPresent()); + + // Spy the topic and force isMigrated() so the subscription is considered migrated. + NonPersistentTopic spyTopic = Mockito.spy(realTopic); + Mockito.doReturn(true).when(spyTopic).isMigrated(); + + // Pre-install a spy subscription so we can assert addConsumer is never invoked when migrated. + NonPersistentSubscription spySubscription = + Mockito.spy(new NonPersistentSubscription(spyTopic, subName, Collections.emptyMap())); + spyTopic.getSubscriptions().put(subName, spySubscription); + + // An active transport connection that records the migration redirect. + PulsarCommandSender commandSender = Mockito.mock(PulsarCommandSender.class); + TransportCnx cnx = Mockito.mock(TransportCnx.class); + Mockito.doReturn(true).when(cnx).isActive(); + Mockito.doReturn(true).when(cnx).isBatchMessageCompatibleVersion(); + Mockito.doReturn("test-role").when(cnx).getAuthRole(); + Mockito.doReturn(pulsar.getBrokerService()).when(cnx).getBrokerService(); + Mockito.doReturn(commandSender).when(cnx).getCommandSender(); + + SubscriptionOption option = SubscriptionOption.builder() + .cnx(cnx) + .subscriptionName(subName) + .consumerId(1L) + .subType(CommandSubscribe.SubType.Shared) + .priorityLevel(0) + .consumerName("consumer-1") + .isDurable(false) + .startMessageId(null) + .metadata(Collections.emptyMap()) + .readCompacted(false) + .initialPosition(CommandSubscribe.InitialPosition.Latest) + .startMessageRollbackDurationSec(0) + .replicatedSubscriptionStateArg(false) + .keySharedMeta(null) + .subscriptionProperties(Optional.empty()) + .build(); + + long usageBefore = spyTopic.currentUsageCount(); + + org.apache.pulsar.broker.service.Consumer consumer = + spyTopic.subscribe(option).get(10, TimeUnit.SECONDS); + assertNotNull(consumer); + + // The migration redirect must have been sent to the client. + Mockito.verify(commandSender).sendTopicMigrated(Mockito.any(), Mockito.eq(1L), + Mockito.eq(migratedUrl.getBrokerServiceUrl()), Mockito.eq(migratedUrl.getBrokerServiceUrlTls())); + + // The core regression assertion: a migrated topic must never attach the consumer to the + // subscription. The buggy code added it before the async redirect/disconnect could run. + Mockito.verify(spySubscription, Mockito.never()).addConsumer(Mockito.any()); + assertTrue(spySubscription.getConsumers().isEmpty()); + + // No usage-count leak: handleConsumerAdded's increment is balanced by the disconnect's removeConsumer. + assertEquals(spyTopic.currentUsageCount(), usageBefore); + } + @Test public void testSubscriptionsOnNonPersistentTopic() throws Exception { final String topicName = "non-persistent://prop/ns-abc/topic_" + UUID.randomUUID(); From 20561a4a007ec07c7ab72e367dbc2b83dca5e64a Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 23 Jun 2026 16:33:40 +0800 Subject: [PATCH 171/201] [feat][broker] Expose managed ledger properties via topic internal stats (#26079) (cherry picked from commit 6d0d99b7c4241b376e4e532183691f8ee48df9ba) --- .../mledger/impl/ManagedLedgerImpl.java | 4 +- .../service/persistent/PersistentTopic.java | 1 + .../pulsar/broker/admin/AdminApi2Test.java | 38 +++++++++++++++++++ .../data/ManagedLedgerInternalStats.java | 3 ++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index b1a7e199740c7..d5462bd76ebb7 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -4978,7 +4978,9 @@ public CompletableFuture getManagedLedgerInternalSta stats.lastConfirmedEntry = this.getLastConfirmedEntry().toString(); stats.state = this.getState().toString(); - stats.cursors = new HashMap(); + stats.properties = new HashMap<>(propertiesMap); + + stats.cursors = new HashMap<>(); this.getCursors().forEach(c -> { ManagedCursorImpl cursor = (ManagedCursorImpl) c; PersistentTopicInternalStats.CursorStats cs = new PersistentTopicInternalStats.CursorStats(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 87e9d0d28e0f1..08a8dc54196b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3064,6 +3064,7 @@ public CompletableFuture getInternalStats(boolean stats.lastConfirmedEntry = ledgerInternalStats.getLastConfirmedEntry(); stats.state = ledgerInternalStats.getState(); stats.ledgers = ledgerInternalStats.ledgers; + stats.properties = ledgerInternalStats.getProperties(); // Add ledger info for compacted topic ledger if exist. LedgerInfo info = new LedgerInfo(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 774c195b7fbb9..bcc61dec08a0a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -69,7 +69,9 @@ import lombok.Cleanup; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.lang3.reflect.FieldUtils; @@ -1440,6 +1442,42 @@ public void testUpdatePropertiesOnNonExistentTopic() throws Exception { } } + @Test + public void testGetInternalStatsWithProperties() throws Exception { + final var namespace = newUniqueName(defaultTenant + "/ns2"); + final var topicName = "persistent://" + namespace + "/testGetInternalStatsWithProperties"; + admin.namespaces().createNamespace(namespace); + + final var topicProperties = Map.of("key1", "value1", "key2", "value2"); + admin.topics().createNonPartitionedTopic(topicName, topicProperties); + + var stats = admin.topics().getInternalStats(topicName); + assertEquals(stats.properties, topicProperties); + + var persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get() + .orElseThrow(); + final var future = new CompletableFuture>(); + persistentTopic.getManagedLedger().asyncSetProperty("new-key", "new-value", + new AsyncCallbacks.UpdatePropertiesCallback() { + @Override + public void updatePropertiesComplete(Map properties, Object ctx) { + future.complete(properties); + } + + @Override + public void updatePropertiesFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, null); + assertEquals(future.get(), Map.of("key1", "value1", "key2", "value2", "new-key", "new-value")); + + admin.namespaces().unload(namespace); + persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, true).get() + .orElseThrow(); + stats = admin.topics().getInternalStats(topicName); + assertEquals(stats.properties, Map.of("key1", "value1", "key2", "value2", "new-key", "new-value")); + } + @Test public void testNonPersistentTopics() throws Exception { final String namespace = newUniqueName(defaultTenant + "/ns2"); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java index 87228fa37362e..c109e26920211 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java @@ -71,6 +71,9 @@ public class ManagedLedgerInternalStats { /** The list of all cursors on this topic. Each subscription in the topic stats has a cursor. */ public Map cursors; + /** The properties map of the managed ledger. */ + public Map properties; + /** * Ledger information. */ From f0d1f048ed6e6db83fa202aba7b42bafb6ce49d3 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 24 Jun 2026 17:32:21 +0800 Subject: [PATCH 172/201] [fix][broker]Do not trigger topic GC if replication is still active (#25915) (cherry picked from commit 4fd22bbe80b1812b7840cefe577d2377fd1b5472) --- .../pulsar/broker/ServiceConfiguration.java | 14 ++ .../broker/service/AbstractReplicator.java | 14 +- .../pulsar/broker/service/AbstractTopic.java | 27 +++ .../pulsar/broker/service/BrokerService.java | 12 ++ .../pulsar/broker/service/Replicator.java | 4 - .../NonPersistentReplicator.java | 6 + .../persistent/GeoPersistentReplicator.java | 7 +- .../persistent/PersistentReplicator.java | 83 ++++++-- .../service/persistent/PersistentTopic.java | 90 +------- .../ReplicatedSubscriptionsController.java | 4 - .../service/persistent/ShadowReplicator.java | 7 +- .../broker/service/OneWayReplicatorTest.java | 194 ++++++++++++++++++ .../service/OneWayReplicatorTestBase.java | 11 +- ...yReplicatorUsingGlobalPartitionedTest.java | 10 + .../OneWayReplicatorUsingGlobalZKTest.java | 78 +++++++ .../broker/service/PersistentTopicTest.java | 25 ++- .../service/ReplicationTopicGcTest.java | 8 + .../pulsar/broker/service/ReplicatorTest.java | 33 ++- .../PersistentReplicatorInflightTaskTest.java | 13 ++ 19 files changed, 488 insertions(+), 152 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 5e977b5b09f28..80f2d479f748d 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -776,6 +776,20 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece ) private Integer brokerDeleteInactiveTopicsMaxInactiveDurationSeconds = null; + @FieldContext( + category = CATEGORY_POLICIES, + dynamic = true, + doc = "Time in seconds that a persistent geo-replication replicator may stay idle before the broker" + + " disconnects its replication producer. A replicator is eligible only when it has no backlog and" + + " has not read entries for replication processing for longer than this threshold. Disconnecting" + + " only releases the idle producer; the replicator and its cursor remain available, and the" + + " producer is recreated automatically when new messages need to be replicated. Set this value to" + + " 0 or a negative value to disable idle-replicator disconnection. The check runs with the" + + " inactive-topic monitor, whose interval is brokerDeleteInactiveTopicsFrequencySeconds, and only" + + " when brokerDeleteInactiveTopicsEnabled is true. The default is 86400 seconds (24 hours)." + ) + private int brokerReplicationInactiveThresholdSeconds = 24 * 3600; + @FieldContext( category = CATEGORY_POLICIES, dynamic = true, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index bb1d1e08f12a5..0d41d7353ade8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -85,6 +85,11 @@ public abstract class AbstractReplicator implements Replicator { private static final AtomicReferenceFieldUpdater ATTRIBUTES_UPDATER = AtomicReferenceFieldUpdater.newUpdater(AbstractReplicator.class, Attributes.class, "attributes"); + protected volatile long latestPublishTime = System.currentTimeMillis(); + + // The estimated time when the producer connection is successful, "0" means it will be connected immediately. + protected volatile long estimatedTimeStampProducerConnected = 0; + public enum State { /** * This enum has two mean meanings： @@ -169,7 +174,7 @@ protected CompletableFuture prepareCreateProducer() { return CompletableFuture.completedFuture(null); } - public void startProducer() { + protected void startProducer() { // Guarantee only one task call "producerBuilder.createAsync()". Pair setStartingRes = compareSetAndGetState(State.Disconnected, State.Starting); if (!setStartingRes.getLeft()) { @@ -203,12 +208,14 @@ public void startProducer() { builderImpl.getConf().setNonPartitionedTopicExpected(true); builderImpl.getConf().setReplProducer(true); return producerBuilder.createAsync().thenAccept(producer -> { + estimatedTimeStampProducerConnected = 0; setProducerAndTriggerReadEntries(producer); }); }).exceptionally(ex -> { Pair setDisconnectedRes = compareSetAndGetState(State.Starting, State.Disconnected); if (setDisconnectedRes.getLeft()) { long waitTimeMs = backOff.next().toMillis(); + estimatedTimeStampProducerConnected = System.currentTimeMillis() + waitTimeMs; log.warn("[{}] Failed to create remote producer ({}), retrying in {} s", replicatorId, ex.getMessage(), waitTimeMs / 1000.0); // BackOff before retrying @@ -311,8 +318,7 @@ protected CompletableFuture isLocalTopicActive() { /** * This method only be used by {@link PersistentTopic#checkGC} now. */ - @Override - public CompletableFuture disconnect() { + protected CompletableFuture disconnect() { long backlog = getNumberOfEntriesInBacklog(); if (backlog > 0) { CompletableFuture disconnectFuture = new CompletableFuture<>(); @@ -390,8 +396,6 @@ protected CompletableFuture closeProducerAsync(boolean closeTheStartingPro Pair setDisconnectedRes = compareSetAndGetState(State.Disconnecting, State.Disconnected); if (setDisconnectedRes.getLeft()) { this.producer = null; - // deactivate further read - disableReplicatorRead(); return; } if (setDisconnectedRes.getRight() == State.Terminating diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 78c4ba2746994..da2000abc2743 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -67,6 +67,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.ProducerFencedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicMigratedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException; +import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.plugin.EntryFilter; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; @@ -669,6 +670,19 @@ protected Consumer getActiveConsumer(Subscription subscription) { return null; } + protected boolean hasProducersActive() { + return !producers.isEmpty(); + } + + protected boolean hasActiveReplicators() { + for (Replicator replicator : getReplicators().values()) { + if (replicator.isConnected()) { + return true; + } + } + return false; + } + protected boolean hasLocalProducers() { if (producers.isEmpty()) { return false; @@ -681,6 +695,19 @@ protected boolean hasLocalProducers() { return false; } + public void disconnectReplicatorsIfNoTrafficAndBacklog() { + for (Replicator replicator : getReplicators().values()) { + if (replicator instanceof PersistentReplicator persistentReplicator) { + persistentReplicator.disconnectIfNoTrafficAndBacklog(); + } + } + for (Replicator replicator : getShadowReplicators().values()) { + if (replicator instanceof PersistentReplicator persistentReplicator) { + persistentReplicator.disconnectIfNoTrafficAndBacklog(); + } + } + } + @Override public String toString() { return MoreObjects.toStringHelper(this).add("topic", topic).toString(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 275d0a61a89fc..73b8aef9d2873 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -725,6 +725,10 @@ protected void startInactivityMonitor() { int interval = pulsar().getConfiguration().getBrokerDeleteInactiveTopicsFrequencySeconds(); inactivityMonitor.scheduleAtFixedRateNonConcurrently(() -> checkGC(), interval, interval, TimeUnit.SECONDS); + if (pulsar().getConfig().getBrokerReplicationInactiveThresholdSeconds() > 0) { + inactivityMonitor.scheduleAtFixedRateNonConcurrently(() -> checkInactiveReplication(), interval, + interval, TimeUnit.SECONDS); + } } // Deduplication info checker @@ -2330,6 +2334,14 @@ public void checkGC() { forEachTopic(Topic::checkGC); } + public void checkInactiveReplication() { + forEachTopic(topic -> { + if (topic instanceof AbstractTopic abstractTopic) { + abstractTopic.disconnectReplicatorsIfNoTrafficAndBacklog(); + } + }); + } + public void checkClusterMigration() { forEachTopic(Topic::checkClusterMigration); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java index 86e2b6e74de89..1f781f0e98dcf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java @@ -25,16 +25,12 @@ public interface Replicator { - void startProducer(); - Topic getLocalTopic(); ReplicatorStatsImpl computeStats(); CompletableFuture terminate(); - CompletableFuture disconnect(); - void updateRates(); String getRemoteCluster(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java index 38320e5be70c5..c3695f978d162 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java @@ -70,6 +70,11 @@ protected String getProducerName() { return getReplicatorName(replicatorPrefix, localCluster) + REPL_PRODUCER_NAME_DELIMITER + remoteCluster; } + @Override + public void startProducer() { + super.startProducer(); + } + @Override protected void setProducerAndTriggerReadEntries(Producer producer) { this.producer = (ProducerImpl) producer; @@ -88,6 +93,7 @@ protected void setProducerAndTriggerReadEntries(Producer producer) { } public void sendMessage(Entry entry) { + latestPublishTime = System.currentTimeMillis(); if ((STATE_UPDATER.get(this) == State.Started) && isWritable()) { int length = entry.getLength(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java index 922a0e42c3ddb..7345e938ad4b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java @@ -148,8 +148,7 @@ protected boolean replicateEntries(List entries, final InFlightTask inFli try { // This flag is set to true when we skip at least one local message, // in order to skip remaining local messages. - boolean isLocalMessageSkippedOnce = false; - boolean skipRemainingMessages = inFlightTask.isSkipReadResultDueToCursorRewind(); + boolean skipRemainingMessages = false; for (int i = 0; i < entries.size(); i++) { Entry entry = entries.get(i); // Skip the messages since the replicator need to fetch the schema info to replicate the schema to the @@ -230,14 +229,14 @@ protected boolean replicateEntries(List entries, final InFlightTask inFli continue; } - if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) { + if (STATE_UPDATER.get(this) != State.Started || inFlightTask.isSkipReadResultDueToCursorRewind()) { // The producer is not ready yet after having stopped/restarted. Drop the message because it will // recover when the producer is ready if (log.isDebugEnabled()) { log.debug("[{}] Dropping read message at {} because producer is not ready", replicatorId, entry.getPosition()); } - isLocalMessageSkippedOnce = true; + skipRemainingMessages = true; inFlightTask.incCompletedEntries(); entry.release(); msg.recycle(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 565efdc456cf7..392f5eb98db43 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.apache.pulsar.broker.service.AbstractReplicator.State.Disconnected; +import static org.apache.pulsar.broker.service.AbstractReplicator.State.Disconnecting; import static org.apache.pulsar.broker.service.AbstractReplicator.State.Started; import static org.apache.pulsar.broker.service.AbstractReplicator.State.Starting; import static org.apache.pulsar.broker.service.AbstractReplicator.State.Terminated; @@ -51,6 +53,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.CursorAlreadyClosedException; import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; @@ -148,13 +151,6 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man @Override protected void setProducerAndTriggerReadEntries(Producer producer) { - // Repeat until there are no read operations in progress - if (STATE_UPDATER.get(this) == State.Starting && hasPendingRead() && !cursor.cancelPendingReadRequest()) { - brokerService.getPulsar().getExecutor() - .schedule(() -> setProducerAndTriggerReadEntries(producer), 10, TimeUnit.MILLISECONDS); - return; - } - /** * 1. Try change state to {@link Started}. * 2. Atoms modify multiple properties if change state success, to avoid another thread get a null value @@ -176,8 +172,6 @@ protected void setProducerAndTriggerReadEntries(Producer producer) { // activate cursor: so, entries can be cached. this.cursor.setActive(); - // Rewind the cursor to be sure to read again all non-acked messages sent while restarting - cursor.rewind(); // read entries readMoreEntries(); } else { @@ -286,6 +280,30 @@ private AvailablePermits getRateLimiterAvailablePermits(int availablePermits) { return new AvailablePermits((int) availablePermitsOnMsg, availablePermitsOnByte); } + public void disconnectIfNoTrafficAndBacklog() { + // Disabled the feature. + int threshold = brokerService.getPulsar().getConfig().getBrokerReplicationInactiveThresholdSeconds(); + if (threshold <= 0) { + return; + } + // Has backlog. + long backlog = getNumberOfEntriesInBacklog(); + if (backlog > 0) { + return; + } + // Already disconnected. + if (state != Started) { + return; + } + + // Disconnect if no backlog and no traffic for a long time. + if (System.currentTimeMillis() - latestPublishTime > threshold * 1000L) { + log.info("Disconnecting replication producers since no producer is active for a long time." + + " brokerReplicationInactiveThresholdSeconds: {}", threshold); + disconnect(); + } + } + protected void readMoreEntries() { if (state.equals(Terminated) || state.equals(Terminating)) { return; @@ -351,6 +369,48 @@ public void readEntriesComplete(List entries, Object ctx) { log.debug("[{}] Read entries complete of {} messages", replicatorId, entries.size()); } InFlightTask inFlightTask = (InFlightTask) ctx; + + latestPublishTime = System.currentTimeMillis(); + // Release memory if terminated. + if (state == State.Terminated || state == State.Terminating + || inFlightTask.isSkipReadResultDueToCursorRewind()) { + for (Entry entry : entries) { + inFlightTask.incCompletedEntries(); + entry.release(); + } + return; + } + + // Retry to trigger read completes if it is not started. + ManagedLedgerImpl ml = (ManagedLedgerImpl) cursor.getManagedLedger(); + Runnable retryReplicateEntries = () -> { + long estimatedTimeStampProducerConnected = this.estimatedTimeStampProducerConnected; + long delayMillis; + if (estimatedTimeStampProducerConnected > System.currentTimeMillis()) { + delayMillis = (estimatedTimeStampProducerConnected - System.currentTimeMillis()) + 100; + } else { + delayMillis = 100; + } + ml.getScheduledExecutor().schedule(() -> { + ml.getExecutor().execute(() -> { + readEntriesComplete(entries, ctx); + }); + }, delayMillis, TimeUnit.MILLISECONDS); + }; + + // Retry. + if (state == Disconnecting || state == Starting) { + retryReplicateEntries.run(); + return; + } + // Start producer and retry. + if (state == Disconnected) { + startProducer(); + retryReplicateEntries.run(); + return; + } + + // After set entries, the next reading can be start. inFlightTask.setEntries(entries); // After the replicator starts, the speed will be gradually increased. @@ -932,15 +992,10 @@ protected CompletableFuture beforeDisconnect() { .TopicBusyException("Cannot close a replicator with backlog")); } } - beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding.Disconnecting); return CompletableFuture.completedFuture(null); } } - protected void afterDisconnected() { - doRewindCursor(false); - } - protected void beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding reason) { synchronized (inFlightTasks) { boolean hasCanceledPendingRead = cursor.cancelPendingReadRequest(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 08a8dc54196b4..26fe918a1440f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -29,7 +29,6 @@ import com.carrotsearch.hppc.ObjectObjectHashMap; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import io.netty.util.concurrent.FastThreadLocal; import java.time.Clock; @@ -835,9 +834,7 @@ public CompletableFuture> addProducer(Producer producer, CompletableFuture producerQueuedFuture) { return super.addProducer(producer, producerQueuedFuture).thenCompose(topicEpoch -> { messageDeduplication.producerAdded(producer.getProducerName()); - - // Start replication producers if not already - return startReplProducers().thenApply(__ -> topicEpoch); + return CompletableFuture.completedFuture(topicEpoch); }); } @@ -884,46 +881,6 @@ private boolean hasRemoteProducers() { return false; } - public CompletableFuture startReplProducers() { - // read repl-cluster from policies to avoid restart of replicator which are in process of disconnect and close - return brokerService.pulsar().getPulsarResources().getNamespaceResources() - .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) - .thenAcceptAsync(optPolicies -> { - if (optPolicies.isPresent()) { - if (optPolicies.get().replication_clusters != null) { - Set configuredClusters = Sets.newTreeSet(optPolicies.get().replication_clusters); - replicators.forEach((region, replicator) -> { - if (configuredClusters.contains(region)) { - replicator.startProducer(); - } - }); - } - } else { - replicators.forEach((region, replicator) -> replicator.startProducer()); - } - }, getOrderedExecutor()).exceptionally(ex -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Error getting policies while starting repl-producers {}", topic, ex.getMessage()); - } - replicators.forEach((region, replicator) -> replicator.startProducer()); - return null; - }); - } - - public CompletableFuture stopReplProducers() { - List> closeFutures = new ArrayList<>(); - replicators.forEach((region, replicator) -> closeFutures.add(replicator.terminate())); - shadowReplicators.forEach((__, replicator) -> closeFutures.add(replicator.terminate())); - return FutureUtil.waitForAll(closeFutures); - } - - private synchronized CompletableFuture closeReplProducersIfNoBacklog() { - List> closeFutures = new ArrayList<>(); - replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect())); - shadowReplicators.forEach((__, replicator) -> closeFutures.add(replicator.disconnect())); - return FutureUtil.waitForAll(closeFutures); - } - @Override protected void handleProducerRemoved(Producer producer) { super.handleProducerRemoved(producer); @@ -3248,7 +3205,7 @@ public boolean isActive(InactiveTopicDeleteMode deleteMode) { break; } // no local producers - return hasLocalProducers(); + return hasProducersActive() || hasActiveReplicators(); } private boolean hasBacklogs(boolean getPreciseBacklog) { @@ -3454,47 +3411,8 @@ public void checkGC() { // Topic activity is still within the retention period return; } else { - CompletableFuture replCloseFuture = new CompletableFuture<>(); - - // Close repl producers first. - // Once all repl producers are closed, we can delete the topic, - // provided no remote producers connected to the broker. - if (log.isDebugEnabled()) { - log.debug("[{}] Topic inactive for {} seconds, closing repl producers.", topic, - maxInactiveDurationInSec); - } - /** - * There is a race condition that may cause a NPE: - * - task 1: a callback of "replicator.cursor.asyncRead" will trigger a replication. - * - task 2: "closeReplProducersIfNoBacklog" called by current thread will make the variable - * "replicator.producer" to a null value. - * Race condition: task 1 will get a NPE when it tries to send messages using the variable - * "replicator.producer", because task 2 will set this variable to "null". - * TODO Create a seperated PR to fix it. - */ - closeReplProducersIfNoBacklog().thenRun(() -> { - if (hasRemoteProducers()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Topic has connected remote producers. Not a candidate for GC", - topic); - } - replCloseFuture - .completeExceptionally(new TopicBusyException("Topic has connected remote producers")); - } else { - log.info("[{}] Topic inactive for {} seconds, closed repl producers", topic, - maxInactiveDurationInSec); - replCloseFuture.complete(null); - } - }).exceptionally(e -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Topic has replication backlog. Not a candidate for GC", topic); - } - replCloseFuture.completeExceptionally(e.getCause()); - return null; - }); - - replCloseFuture.thenCompose(v -> delete(deleteMode == InactiveTopicDeleteMode.delete_when_no_subscriptions, - deleteMode == InactiveTopicDeleteMode.delete_when_subscriptions_caught_up, false)) + delete(deleteMode == InactiveTopicDeleteMode.delete_when_no_subscriptions, + deleteMode == InactiveTopicDeleteMode.delete_when_subscriptions_caught_up, false) .thenCompose((res) -> tryToDeletePartitionedMetadata()) .thenRun(() -> log.info("[{}] Topic deleted successfully due to inactivity", topic)) .exceptionally(e -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java index a156566d0feaa..f19570afcd6e8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java @@ -145,7 +145,6 @@ public void localSubscriptionUpdated(String subscriptionName, } private void receivedSnapshotRequest(ReplicatedSubscriptionsSnapshotRequest request) { - // if replicator producer is already closed, restart it to send snapshot response Replicator replicator = topic.getReplicators().get(request.getSourceCluster()); if (replicator == null) { log.warn("[{}] Received replicated subscription snapshot request {} from cluster {}, but no replicator is" @@ -153,9 +152,6 @@ private void receivedSnapshotRequest(ReplicatedSubscriptionsSnapshotRequest requ topic.getName(), request.getSnapshotId(), request.getSourceCluster()); return; } - if (!replicator.isConnected()) { - topic.startReplProducers(); - } // Send response containing the current last written message id. The response // marker we're publishing locally and then replicating will have a higher diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java index 2e5e91fb9a633..08c1f10fa3025 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java @@ -63,8 +63,7 @@ protected boolean replicateEntries(List entries, InFlightTask inFlightTas try { // This flag is set to true when we skip at least one local message, // in order to skip remaining local messages. - boolean isLocalMessageSkippedOnce = false; - boolean skipRemainingMessages = inFlightTask.isSkipReadResultDueToCursorRewind(); + boolean skipRemainingMessages = false; for (int i = 0; i < entries.size(); i++) { Entry entry = entries.get(i); // Skip the messages since the replicator need to fetch the schema info to replicate the schema to the @@ -101,14 +100,14 @@ protected boolean replicateEntries(List entries, InFlightTask inFlightTas continue; } - if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) { + if (STATE_UPDATER.get(this) != State.Started || inFlightTask.isSkipReadResultDueToCursorRewind()) { // The producer is not ready yet after having stopped/restarted. Drop the message because it will // recovered when the producer is ready if (log.isDebugEnabled()) { log.debug("[{}] Dropping read message at {} because producer is not ready", replicatorId, entry.getPosition()); } - isLocalMessageSkippedOnce = true; + skipRemainingMessages = true; inFlightTask.incCompletedEntries(); entry.release(); msg.recycle(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index 06c4411aae034..673dd082ba491 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -55,6 +55,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -75,6 +78,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerTest; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.resources.ClusterResources; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentReplicator; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; @@ -120,6 +124,8 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.impl.DualMetadataStore; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; import org.awaitility.Awaitility; import org.awaitility.reflect.WhiteboxImpl; import org.glassfish.jersey.client.JerseyClient; @@ -147,6 +153,11 @@ public void cleanup() throws Exception { super.cleanup(); } + protected void setConfigDefaults(ServiceConfiguration config, String clusterName, + LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { + super.setConfigDefaults(config, clusterName, bookkeeperEnsemble, brokerConfigZk); + } + @Test(timeOut = 45 * 1000) public void testReceiverSideReplicationStats() throws Exception { final String topic = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); @@ -226,6 +237,189 @@ public void testDeleteTopicWhenReplicating() throws Exception { }); } + @DataProvider + public Object[][] paramsDisconnectReplicator() { + // Binary way replication. + // local producers on cluster-1 registered. + // local producers on cluster-1 have traffic. + // replicator producer from cluster-2 has traffic. + // replicator producer from cluster-2 is present. + return new Object[][] { + {true, true, false, true, true}, // verify-cluster-2: no replicator terminate occurs. + {true, true, false, false, true}, // verify-cluster-2: replicator terminated and resumed. + {true, true, false, false, false}, // verify-cluster-2: replicator terminated and resumed. + {true, false, false, true, true}, // verify-cluster-2: no replicator terminate occurs. + {true, false, false, false, true}, // verify-cluster-2: replicator terminated and resumed. + {true, false, false, false, false}, // verify-cluster-2: replicator terminated and resumed. + + {false, false, false, false, false}, // verify-cluster-2: replicator terminated and resumed. + {false, true, false, false, false} // verify-cluster-2: replicator terminated and resumed. + }; + } + + @Test(timeOut = 240 * 1000, dataProvider = "paramsDisconnectReplicator") + public void testDisconnectAndReconnectReplicator(boolean binaryWayRepl, + boolean hasLocalProducerRegistered, + boolean localProducerHasTraffic, + boolean hasRemoteProducerTraffic, + boolean hasRemoteProducerRegistered) throws Exception { + ScheduledExecutorService executor1 = Executors.newScheduledThreadPool(1); + ScheduledExecutorService executor2 = Executors.newScheduledThreadPool(1); + ScheduledFuture checkInactiveTopic = executor1.scheduleWithFixedDelay(() -> { + pulsar1.getBrokerService().checkInactiveReplication(); + }, 10, 10, TimeUnit.SECONDS); + // local cluster: let inactive replicator check faster. + int replicationInactiveThresholdSeconds1 = pulsar1.getConfig().getBrokerReplicationInactiveThresholdSeconds(); + pulsar1.getConfig().setBrokerReplicationInactiveThresholdSeconds(30); + // remote cluster: let inactive topic deletion never occur. + int replicationInactiveThresholdSeconds2 = pulsar2.getConfig().getBrokerReplicationInactiveThresholdSeconds(); + pulsar2.getConfig().setBrokerReplicationInactiveThresholdSeconds(3600 * 24); + // Lat topic GC does not execute. + int inactiveTopicsMaxInactiveDurationSeconds = pulsar1.getConfig() + .getBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(); + pulsar1.getConfig().setBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(3600 * 24); + + // Check params. + if (hasRemoteProducerTraffic && !hasRemoteProducerRegistered) { + throw new Exception("If has traffic from remote cluster, the param \"hasRemoteProducer\" can not be false"); + } + // Check params. + if (localProducerHasTraffic && !hasLocalProducerRegistered) { + throw new Exception("If has local traffic, the param \"localProducerEmpty\" can not be true"); + } + + ScheduledFuture scheduledPublish1 = null; + ScheduledFuture scheduledPublish2 = null; + final String topic = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + + // Init by params: local producers. + final Producer producer1A = client1.newProducer(Schema.STRING).topic(topic).create(); + Producer producer1B = null; + if (!hasLocalProducerRegistered) { + producer1A.close(); + } + // Init by params: local producer traffic. + if (localProducerHasTraffic) { + AtomicInteger msgCount = new AtomicInteger(); + scheduledPublish1 = executor1.scheduleWithFixedDelay(() -> { + producer1A.sendAsync(msgCount.incrementAndGet() + ""); + }, 1, 1, TimeUnit.SECONDS); + } + // Init by params: binary way replication. + waitReplicatorStarted(topic, pulsar2); + if (binaryWayRepl) { + admin2.topics().setReplicationClusters(topic, Arrays.asList(cluster1, cluster2)); + waitReplicatorStarted(topic, pulsar1); + } + final PersistentTopic persistentTopic1 = (PersistentTopic) broker1.getTopic(topic, false).join().get(); + final PersistentTopic persistentTopic2 = (PersistentTopic) broker2.getTopic(topic, false).join().get(); + // Init by params: remote producer traffic. + final Producer producer2 = client2.newProducer(Schema.STRING).topic(topic).create(); + if (hasRemoteProducerTraffic) { + AtomicInteger msgCount = new AtomicInteger(); + scheduledPublish2 = executor2.scheduleWithFixedDelay(() -> { + producer2.sendAsync(msgCount.incrementAndGet() + ""); + }, 1, 1, TimeUnit.SECONDS); + } + // Init by params: remote producers. + if (binaryWayRepl && !hasRemoteProducerTraffic && !hasRemoteProducerRegistered) { + persistentTopic2.getReplicators().get(cluster1).terminate(); + } + + // Verify: all states match params. + Thread.sleep(3000); + // All states match: local producers. + if (!hasLocalProducerRegistered) { + assertFalse(persistentTopic1.getProducers().values().stream() + .filter(p -> !p.isRemote()).findAny().isPresent()); + } else { + Optional serviceProducer1 = persistentTopic1.getProducers() + .values().stream().filter(p -> !p.isRemote()).findAny(); + assertTrue(serviceProducer1.isPresent()); + } + // All states match: remote producers. + if (binaryWayRepl) { + if (!hasRemoteProducerRegistered) { + assertFalse(persistentTopic1.getProducers().values().stream() + .filter(p -> p.isRemote()).findAny().isPresent()); + } else { + Optional serviceProducer1 = persistentTopic1.getProducers() + .values().stream().filter(p -> p.isRemote()).findAny(); + assertTrue(serviceProducer1.isPresent()); + } + } + + // Verify: replicator terminated or not. + if (hasRemoteProducerTraffic || localProducerHasTraffic) { + long verifyStartTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - verifyStartTime < 100_000) { + assertFalse(persistentTopic1.getReplicators().isEmpty()); + PersistentReplicator persistentReplicator = + (PersistentReplicator) persistentTopic1.getReplicators().get(cluster2); + assertTrue(persistentReplicator.isConnected()); + assertEquals(persistentReplicator.getState(), AbstractReplicator.State.Started); + Thread.sleep(1000); + } + } else { + Thread.sleep(100_000); + assertFalse(persistentTopic1.getReplicators().isEmpty()); + PersistentReplicator persistentReplicatorA = + (PersistentReplicator) persistentTopic1.getReplicators().get(cluster2); + assertFalse(persistentReplicatorA.isConnected()); + assertEquals(persistentReplicatorA.getState(), AbstractReplicator.State.Disconnected); + + // Verify: resume. + if (hasRemoteProducerRegistered && !hasRemoteProducerTraffic) { + producer2.send("msg-remote"); + } + if (!hasLocalProducerRegistered) { + producer1B = client1.newProducer(Schema.STRING).topic(topic).create(); + producer1B.send("msg-local"); + } else { + producer1A.send("msg-local"); + } + Awaitility.await().untilAsserted(() -> { + assertFalse(persistentTopic1.getReplicators().isEmpty()); + PersistentReplicator persistentReplicatorB = + (PersistentReplicator) persistentTopic1.getReplicators().get(cluster2); + assertTrue(persistentReplicatorB.isConnected()); + assertEquals(persistentReplicatorB.getState(), AbstractReplicator.State.Started); + }); + } + + // cleanup. + pulsar1.getConfig().setBrokerReplicationInactiveThresholdSeconds(replicationInactiveThresholdSeconds1); + pulsar2.getConfig().setBrokerReplicationInactiveThresholdSeconds(replicationInactiveThresholdSeconds2); + pulsar1.getConfig().setBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds( + inactiveTopicsMaxInactiveDurationSeconds); + if (scheduledPublish1 != null) { + scheduledPublish1.cancel(true); + } + if (scheduledPublish2 != null) { + scheduledPublish2.cancel(true); + } + checkInactiveTopic.cancel(true); + if (producer1A.isConnected()) { + producer1A.close(); + } + if (producer1B != null && producer1B.isConnected()) { + producer1B.close(); + } + if (producer2.isConnected()) { + producer2.close(); + } + if (binaryWayRepl) { + admin2.topics().setReplicationClusters(topic, Arrays.asList(cluster2)); + waitReplicatorStopped(pulsar2, pulsar1, topic); + } + cleanupTopics(() -> { + admin1.topics().delete(topic); + admin2.topics().delete(topic); + }); + executor1.shutdown(); + executor2.shutdown(); + } + @Test(timeOut = 45 * 1000) public void testReplicatorProducerStatInTopic() throws Exception { final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java index a5983cecf9cdf..c07704bef931e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java @@ -44,6 +44,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; import org.apache.pulsar.broker.service.persistent.GeoPersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -55,6 +56,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.ClusterData; @@ -407,8 +409,13 @@ protected void waitReplicatorStarted(String topicName, PulsarService remoteClust Awaitility.await().untilAsserted(() -> { Optional topicOptional2 = remoteCluster.getBrokerService().getTopic(topicName, false).get(); assertTrue(topicOptional2.isPresent()); - PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); - assertFalse(persistentTopic2.getProducers().isEmpty()); + if (TopicName.get(topicName).getDomain().equals(TopicDomain.persistent)) { + PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); + assertFalse(persistentTopic2.getProducers().isEmpty()); + } else { + NonPersistentTopic nonPersistentTopic2 = (NonPersistentTopic) topicOptional2.get(); + assertFalse(nonPersistentTopic2.getProducers().isEmpty()); + } }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java index 32ac30fece8c0..16a1686ce6e06 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java @@ -84,6 +84,16 @@ public void testDeleteTopicWhenReplicating() throws Exception { super.testDeleteTopicWhenReplicating(); } + @Test(enabled = false) + public void testDisconnectAndReconnectReplicator(boolean binaryWayRepl, + boolean hasLocalProducerRegistered, + boolean localProducerHasTraffic, + boolean hasRemoteProducerTraffic, + boolean hasRemoteProducerRegistered) throws Exception { + super.testDisconnectAndReconnectReplicator(binaryWayRepl, hasLocalProducerRegistered, localProducerHasTraffic, + hasRemoteProducerTraffic, hasRemoteProducerRegistered); + } + @Override @Test(enabled = false) public void testReplicatorProducerStatInTopic() throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java index aaab17d006467..0af1dc0fa3088 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java @@ -54,6 +54,8 @@ import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; +import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; import org.apache.pulsar.common.policies.data.NamespaceIsolationData; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TopicPolicies; @@ -818,6 +820,82 @@ public void testSystemTopicCreationWithDifferentTopicCreationRule(int localSyste admin2.topics().delete(tp, false); } + @Test(enabled = false) + public void testDisconnectAndReconnectReplicator(boolean binaryWayRepl, + boolean hasLocalProducerRegistered, + boolean localProducerHasTraffic, + boolean hasRemoteProducerTraffic, + boolean hasRemoteProducerRegistered) throws Exception { + super.testDisconnectAndReconnectReplicator(binaryWayRepl, hasLocalProducerRegistered, localProducerHasTraffic, + hasRemoteProducerTraffic, hasRemoteProducerRegistered); + } + + @Test + public void testTopicGCDoesNotDisconnectReplicatorWhenRemoteProducerIsActive() throws Exception { + int replicationInactiveThresholdSeconds = pulsar1.getConfig().getBrokerReplicationInactiveThresholdSeconds(); + pulsar1.getConfig().setBrokerReplicationInactiveThresholdSeconds(3600); + final String topic = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + admin1.topics().createNonPartitionedTopic(topic); + Producer producer1 = client1.newProducer(Schema.STRING).topic(topic).create(); + + try { + producer1.send("msg-1"); + waitReplicatorStarted(topic, pulsar1); + waitReplicatorStarted(topic, pulsar2); + PersistentTopic persistentTopic2 = (PersistentTopic) broker2.getTopic(topic, false) + .join().get(); + + // Set inactive policies. + InactiveTopicPolicies inactiveTopicPolicies = new InactiveTopicPolicies(); + inactiveTopicPolicies.setInactiveTopicDeleteMode(InactiveTopicDeleteMode.delete_when_no_subscriptions); + inactiveTopicPolicies.setMaxInactiveDurationSeconds(10); + inactiveTopicPolicies.setDeleteWhileInactive(true); + admin2.topicPolicies().setInactiveTopicPolicies(topic, inactiveTopicPolicies); + + // Ensure policies were set successfully. + Awaitility.await().untilAsserted(() -> { + assertFalse(persistentTopic2.getProducers().values().stream() + .anyMatch(producer -> !producer.isRemote())); + assertTrue(persistentTopic2.getSubscriptions().isEmpty()); + assertTrue(persistentTopic2.getInactiveTopicPolicies().isDeleteWhileInactive()); + assertEquals(persistentTopic2.getInactiveTopicPolicies().getMaxInactiveDurationSeconds(), 10); + + Replicator replicator = persistentTopic2.getReplicators().get(cluster1); + assertNotNull(replicator); + assertTrue(replicator.isConnected()); + assertEquals(replicator.getNumberOfEntriesInBacklog(), 0); + }); + + // Trigger GC. + persistentTopic2.disconnectReplicatorsIfNoTrafficAndBacklog(); + persistentTopic2.checkGC(); + Thread.sleep(15 * 1000); + persistentTopic2.disconnectReplicatorsIfNoTrafficAndBacklog(); + persistentTopic2.checkGC(); + + // Verify: the replication is not disconnected due to Topic GC. + Replicator replicator = persistentTopic2.getReplicators().get(cluster1); + assertNotNull(replicator); + assertTrue(replicator.isConnected()); + + // Verify: the replication still works. + producer1.send("msg-2"); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin2.topics().getStats(topic).getReplication().get(cluster1).getReplicationBacklog(), 0); + }); + + } finally { + pulsar1.getConfig().setBrokerReplicationInactiveThresholdSeconds(replicationInactiveThresholdSeconds); + producer1.close(); + admin1.topics().setReplicationClusters(topic, Arrays.asList(cluster1)); + admin2.topics().setReplicationClusters(topic, Arrays.asList(cluster2)); + waitReplicatorStopped(topic, pulsar1, pulsar2, false); + waitReplicatorStopped(topic, pulsar2, pulsar1, false); + admin1.topics().delete(topic, false); + admin2.topics().delete(topic, false); + } + } + @Test public void testUpdateNamespacePolicies() throws Exception { // Create a namespace and allow both clusters to access. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 5d7f88fb3bd5e..6ce87968155b8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -1676,17 +1676,19 @@ private PulsarAdmin mockReplicationAdmin() { } /** - * NonPersistentReplicator.removeReplicator doesn't remove replicator in atomic way and does in multiple step: - * 1. disconnect replicator producer + * PersistentReplicator.removeReplicator doesn't remove replicator in atomic way and does in multiple step: + * 1. Turn off replication. *

- * 2. close cursor + * 2. Broker will do two things: + * 2-1. terminate replication. + * 2-2. delete cursor that named "repl.x" *

- * 3. remove from replicator-list. + * 3. remove the terminated replicator from "topic.replicators". *

* - * If we try to startReplicationProducer before step-c finish then it should not avoid restarting repl-producer. - * - * @throws Exception + * Test: + * do: try to restart replicator producer before step-2-2 finish. + * verify: the replicator producer will not be started. */ @Test public void testAtomicReplicationRemoval() throws Exception { @@ -1743,9 +1745,14 @@ public CompletableFuture createAsync() { // step-2 now, policies doesn't have removed replication cluster so, it should not invoke "startProducer" of the // replicator // try to start replicator again - topic.startReplProducers().join(); + Awaitility.await().untilAsserted(() -> { + assertEquals(replicator.getState(), AbstractReplicator.State.Terminated); + }); + replicator.startProducer(); + Thread.sleep(10_000); // verify: replicator.startProducer is not invoked - verify(replicator, Mockito.times(1)).startProducer(); + assertEquals(replicator.getState(), AbstractReplicator.State.Terminated); + assertFalse(replicator.isConnected()); // step-3 : complete the callback to remove replicator from the list ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteCursorCallback.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTopicGcTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTopicGcTest.java index c2b6d4281a9b0..c72d2bcef53d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTopicGcTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTopicGcTest.java @@ -28,6 +28,8 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.persistent.GeoPersistentReplicator; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; @@ -77,6 +79,7 @@ protected void setConfigDefaults(ServiceConfiguration config, String clusterName config.setBrokerDeleteInactiveTopicsFrequencySeconds(5); config.setBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(5); config.setReplicationPolicyCheckDurationSeconds(1); + config.setBrokerReplicationInactiveThresholdSeconds(5); } @Test(dataProvider = "topicTypes") @@ -144,6 +147,7 @@ public void testRemoteClusterStillConsumeAfterCurrentClusterGc(TopicType topicTy // Wait for replicator started. Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).create(); waitReplicatorStarted(subTopic); + PersistentTopic persistentTopic1 = (PersistentTopic) broker1.getTopic(subTopic, false).get().get(); admin2.topics().createSubscription(topicName, subscription, MessageId.earliest); if (usingGlobalZK) { @@ -159,6 +163,10 @@ public void testRemoteClusterStillConsumeAfterCurrentClusterGc(TopicType topicTy // Trigger GC through close all clients. producer1.close(); + // Manually skip the check "brokerReplicationInactiveThresholdSeconds". + GeoPersistentReplicator geoPersistentReplicator1 = + (GeoPersistentReplicator) persistentTopic1.getReplicators().get(cluster2); + geoPersistentReplicator1.disconnect(); // Verify: the topic was removed on the source cluster. Awaitility.await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { // sub topic. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 605e1cd7ae39f..0db4050aa6471 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -71,7 +71,9 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; +import org.apache.pulsar.broker.service.persistent.GeoPersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; +import org.apache.pulsar.broker.service.persistent.PersistentReplicatorInflightTaskTest; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.stats.OpenTelemetryReplicatorStats; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -1812,21 +1814,26 @@ public void testReplicatorWithTTL() throws Exception { waitReplicateFinish(topic, admin1); // Pause replicator + List resumeReplicatorFunctions = new ArrayList<>(); persistentTopic.getReplicators().forEach((cluster, replicator) -> { PersistentReplicator persistentReplicator = (PersistentReplicator) replicator; - pauseReplicator(persistentReplicator); + resumeReplicatorFunctions.add(PersistentReplicatorInflightTaskTest.pauseReplicator(persistentReplicator)); }); // Send V2 and V3 messages, then let them expire. These messages will not be replicated to the remote cluster. persistentProducer1.send("V2".getBytes()); persistentProducer1.send("V3".getBytes()); - Thread.sleep(1000); - admin1.topics().expireMessagesForAllSubscriptions(topic.toString(), 1); + Thread.sleep(2000); + GeoPersistentReplicator persistentReplicator = + (GeoPersistentReplicator) persistentTopic.getReplicators().values().iterator().next(); + persistentReplicator.expireMessages(1); + waitReplicateFinish(topic, admin1); // Start replicator + for (Runnable r : resumeReplicatorFunctions) { + r.run(); + } persistentTopic.getReplicators().forEach((cluster, replicator) -> { - PersistentReplicator persistentReplicator = (PersistentReplicator) replicator; - resumeReplicator(persistentReplicator); Awaitility.await().untilAsserted(() -> { CompletableFuture> topic2 = pulsar2.getBrokerService().getTopic(topic.toString(), false); @@ -1903,7 +1910,7 @@ public void testReplicationMetrics() throws Exception { .stream() .map(PersistentReplicator.class::cast) .toList(); - persistentReplicators.forEach(this::pauseReplicator); + persistentReplicators.forEach(PersistentReplicatorInflightTaskTest::pauseReplicator); producer1.produce(5); Awaitility.await().untilAsserted(() -> { persistentReplicators.forEach(repl -> repl.expireMessages(1)); @@ -1983,18 +1990,4 @@ public void testEnableReplicationWithNamespaceAllowedClustersPolices() throws Ex assertTrue(replicator.isConnected()); }); } - - private void pauseReplicator(PersistentReplicator replicator) { - Awaitility.await().untilAsserted(() -> { - assertTrue(replicator.isConnected()); - }); - Awaitility.await().until(() -> { - replicator.disconnect().join(); - return true; - }); - } - - private void resumeReplicator(PersistentReplicator replicator) { - replicator.startProducer(); - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java index 0c22b14acf748..83529969df417 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java @@ -496,4 +496,17 @@ public void testAcquirePermitsIfNotFetchingSchema() throws Exception { inFlightTasks.addAll(originalTasks); } } + + public static Runnable pauseReplicator(PersistentReplicator replicator) { + Awaitility.await().untilAsserted(() -> { + assertTrue(replicator.isConnected()); + }); + replicator.beforeTerminateOrCursorRewinding(PersistentReplicator.ReasonOfWaitForCursorRewinding.Disconnecting); + replicator.doRewindCursor(false); + InFlightTask inFlightTask = replicator.createOrRecycleInFlightTaskIntoQueue(PositionFactory.create(1, 1), 1); + return () -> { + inFlightTask.setEntries(Collections.emptyList()); + replicator.readMoreEntries(); + }; + } } From 1a7fe8bc7c81282d8b8e67ea6c81dd2d2a980a31 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Fri, 26 Jun 2026 19:28:47 +0800 Subject: [PATCH 173/201] [fix][sec][branch-4.2] Upgrade Jackson version to 2.18.8 (#26099) --- .../server/src/assemble/LICENSE.bin.txt | 22 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 22 +++++++++---------- pom.xml | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index b42107c28918f..527baae4dcf0c 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -249,17 +249,17 @@ The Apache Software License, Version 2.0 - info.picocli-picocli-shell-jline3-4.7.7.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.18.6.jar - - com.fasterxml.jackson.core-jackson-core-2.18.6.jar - - com.fasterxml.jackson.core-jackson-databind-2.18.6.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.18.6.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.18.6.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.18.6.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.18.6.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.18.6.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.18.6.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.6.jar - - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.6.jar + - com.fasterxml.jackson.core-jackson-annotations-2.18.8.jar + - com.fasterxml.jackson.core-jackson-core-2.18.8.jar + - com.fasterxml.jackson.core-jackson-databind-2.18.8.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.18.8.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.18.8.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.18.8.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.18.8.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.18.8.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.18.8.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.8.jar + - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.8.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-3.2.4.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Fastutil -- it.unimi.dsi-fastutil-8.5.16.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 27c639f7877a5..bd805d92f66f4 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -313,17 +313,17 @@ The Apache Software License, Version 2.0 - picocli-4.7.7.jar - picocli-shell-jline3-4.7.7.jar * Jackson - - jackson-annotations-2.18.6.jar - - jackson-core-2.18.6.jar - - jackson-databind-2.18.6.jar - - jackson-dataformat-yaml-2.18.6.jar - - jackson-jaxrs-base-2.18.6.jar - - jackson-jaxrs-json-provider-2.18.6.jar - - jackson-module-jaxb-annotations-2.18.6.jar - - jackson-module-jsonSchema-2.18.6.jar - - jackson-datatype-jdk8-2.18.6.jar - - jackson-datatype-jsr310-2.18.6.jar - - jackson-module-parameter-names-2.18.6.jar + - jackson-annotations-2.18.8.jar + - jackson-core-2.18.8.jar + - jackson-databind-2.18.8.jar + - jackson-dataformat-yaml-2.18.8.jar + - jackson-jaxrs-base-2.18.8.jar + - jackson-jaxrs-json-provider-2.18.8.jar + - jackson-module-jaxb-annotations-2.18.8.jar + - jackson-module-jsonSchema-2.18.8.jar + - jackson-datatype-jdk8-2.18.8.jar + - jackson-datatype-jsr310-2.18.8.jar + - jackson-module-parameter-names-2.18.8.jar * Caffeine -- caffeine-3.2.4.jar * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar * Gson diff --git a/pom.xml b/pom.xml index 601b43328d543..c80e28c04adf5 100644 --- a/pom.xml +++ b/pom.xml @@ -211,7 +211,7 @@ flexible messaging model and an intuitive client API. 2.0.11 2.0.6 2.0.1 - 2.18.6 + 2.18.8 8.5.16 4.0.5 0.10.2 From ba906cbaff5fb449f38cb4fd8e7ef765310d535f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=81=93=E5=90=9B-=20Tao=20Jiuming?= Date: Tue, 23 Jun 2026 15:26:13 +0800 Subject: [PATCH 174/201] [improve][broker] Trim orphaned bucket snapshots when ledgers are deleted (#25984) (cherry picked from commit e45b425f927c63b4de4093ee6f9f0218b188b701) --- .../bucket/BucketDelayedDeliveryTracker.java | 110 ++++++++- .../BucketDelayedDeliveryTrackerTest.java | 213 ++++++++++++++++++ 2 files changed, 312 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 69edf70dad786..7a1b95a3a4a73 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -38,6 +38,7 @@ import java.util.Optional; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -48,6 +49,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.commons.collections4.CollectionUtils; @@ -112,6 +114,8 @@ public static record SnapshotKey(long ledgerId, long entryId) {} private CompletableFuture pendingLoad = null; + private volatile CompletableFuture trimFuture; + public BucketDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, @@ -387,8 +391,15 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver afterCreateImmutableBucket(immutableBucketDelayedIndexPair, createStartTime); lastMutableBucket.resetLastMutableBucketRange(); - if (maxNumBuckets > 0 && immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { - asyncMergeBucketSnapshot(); + if (maxNumBuckets > 0 && immutableBuckets.asMapOfRanges().size() > maxNumBuckets + && (trimFuture == null || trimFuture.isDone())) { + trimFuture = asyncTrimImmutableBuckets() + .thenCompose(ignore -> asyncMergeBucketSnapshot()) + .whenComplete((ignore, t) -> { + if (t != null) { + log.warn("Failed to trim or merge bucket snapshots", t); + } + }); } } @@ -452,6 +463,10 @@ private synchronized List selectMergedBuckets(final List asyncMergeBucketSnapshot() { List immutableBucketList = immutableBuckets.asMapOfRanges().values().stream().toList(); + if (maxNumBuckets <= 0 || immutableBucketList.size() <= maxNumBuckets) { + return CompletableFuture.completedFuture(null); + } + List toBeMergeImmutableBuckets = selectMergedBuckets(immutableBucketList, MAX_MERGE_NUM); if (toBeMergeImmutableBuckets.isEmpty()) { @@ -605,6 +620,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) } long cutoffTime = getCutoffTime(); + Long firstLiveLedgerId = firstActiveLedgerId(); lastMutableBucket.moveScheduledMessageToSharedQueue(cutoffTime, sharedBucketPriorityQueue); @@ -613,13 +629,19 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) while (n > 0 && !sharedBucketPriorityQueue.isEmpty()) { long timestamp = sharedBucketPriorityQueue.peekN1(); + long ledgerId = sharedBucketPriorityQueue.peekN2(); + long entryId = sharedBucketPriorityQueue.peekN3(); + if (firstLiveLedgerId != null && ledgerId < firstLiveLedgerId) { + sharedBucketPriorityQueue.pop(); + if (removeIndexBit(ledgerId, entryId)) { + numberDelayedMessages.decrementAndGet(); + } + continue; + } if (timestamp > cutoffTime) { break; } - long ledgerId = sharedBucketPriorityQueue.peekN2(); - long entryId = sharedBucketPriorityQueue.peekN3(); - SnapshotKey snapshotKey = new SnapshotKey(ledgerId, entryId); ImmutableBucket bucket = snapshotSegmentLastIndexMap.get(snapshotKey); @@ -719,12 +741,26 @@ public boolean shouldPauseAllDeliveries() { @Override public synchronized CompletableFuture clear() { - CompletableFuture future = cleanImmutableBuckets(); - sharedBucketPriorityQueue.clear(); - lastMutableBucket.clear(); - snapshotSegmentLastIndexMap.clear(); - numberDelayedMessages.set(0); - return future; + // Wait for any in-flight trim+merge to settle, then clear. + // Reuse trimFuture to block new triggers until the clear chain completes. + CompletableFuture before = trimFuture != null && !trimFuture.isDone() + ? trimFuture : CompletableFuture.completedFuture(null); + trimFuture = before + .exceptionally(t -> { + log.warn("Trim/merge buckets failed, but still clear", t); + return null; + }) + .thenCompose(__ -> { + synchronized (BucketDelayedDeliveryTracker.this) { + CompletableFuture future = cleanImmutableBuckets(); + sharedBucketPriorityQueue.clear(); + lastMutableBucket.clear(); + snapshotSegmentLastIndexMap.clear(); + numberDelayedMessages.set(0); + return future; + } + }); + return trimFuture; } @Override @@ -791,4 +827,56 @@ public Map genTopicMetricMap() { stats.recordBucketSnapshotSizeBytes(totalSnapshotLength.longValue()); return stats.genTopicMetricMap(); } + + /** + * Delete orphaned bucket snapshots whose ledger range lies entirely before the earliest + * surviving ledger. Buckets are deleted sequentially; the chain stops on first failure + * to avoid wasted work when storage is unavailable. + */ + private synchronized CompletableFuture asyncTrimImmutableBuckets() { + Long firstLedgerId = firstActiveLedgerId(); + if (null == firstLedgerId) { + return CompletableFuture.completedFuture(null); + } + ManagedLedger ledger = dispatcher.getCursor().getManagedLedger(); + + Map, ImmutableBucket> toBeDeletedBuckets = + new HashMap<>(immutableBuckets.subRangeMap(Range.lessThan(firstLedgerId)).asMapOfRanges()); + + if (toBeDeletedBuckets.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + + String ledgerName = ledger.getName(); + CompletableFuture chain = CompletableFuture.completedFuture(null); + for (Map.Entry, ImmutableBucket> entry : toBeDeletedBuckets.entrySet()) { + chain = chain.thenCompose(__ -> + deleteBucketSnapshot(ledgerName, entry.getKey(), entry.getValue())); + } + return chain; + } + + private CompletableFuture deleteBucketSnapshot(String ledgerName, + Range range, ImmutableBucket bucket) { + return bucket.asyncDeleteBucketSnapshot(stats) + .handle((__, t) -> { + if (t != null) { + log.warn("Failed to delete bucket snapshot, LedgerName: {}, BucketKey: {}", + ledgerName, bucket.bucketKey()); + throw new CompletionException(t); + } + synchronized (this) { + snapshotSegmentLastIndexMap.entrySet().removeIf(entry -> entry.getValue() == bucket); + immutableBuckets.remove(range); + numberDelayedMessages.addAndGet(-bucket.getNumberBucketDelayedMessages()); + } + return null; + }); + } + + private Long firstActiveLedgerId() { + ManagedCursor cursor = dispatcher.getCursor(); + Position mdp = cursor.getMarkDeletedPosition(); + return mdp == null ? null : mdp.getLedgerId(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 6ff98fa7f7004..ee0ec0d92945a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -40,13 +40,16 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.proto.ManagedLedgerInfo.LedgerInfo; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.pulsar.broker.delayed.AbstractDeliveryTrackerTest; import org.apache.pulsar.broker.delayed.MockBucketSnapshotStorage; @@ -463,4 +466,214 @@ public void testClear(BucketDelayedDeliveryTracker tracker) tracker.close(); } + + private static class TrackerWithStorage { + final BucketDelayedDeliveryTracker tracker; + final MockBucketSnapshotStorage storage; + final AtomicLong clockTime; + + TrackerWithStorage(BucketDelayedDeliveryTracker tracker, MockBucketSnapshotStorage storage, + AtomicLong clockTime) { + this.tracker = tracker; + this.storage = storage; + this.clockTime = clockTime; + } + + void close() throws Exception { + tracker.close(); + storage.close(); + } + } + + private static class BlockingDeleteStorage extends MockBucketSnapshotStorage { + final CompletableFuture firstDeleteFuture = new CompletableFuture<>(); + final AtomicLong deleteCalls = new AtomicLong(); + + @Override + public CompletableFuture deleteBucketSnapshot(long bucketId) { + if (deleteCalls.incrementAndGet() <= 4) { + return firstDeleteFuture; + } + return super.deleteBucketSnapshot(bucketId); + } + } + + private TrackerWithStorage createTrackerWithMockLedger(long firstLedgerId, int maxNumBuckets) + throws Exception { + return createTrackerWithMockLedger(firstLedgerId, maxNumBuckets, new MockBucketSnapshotStorage()); + } + + private TrackerWithStorage createTrackerWithMockLedger(long firstLedgerId, int maxNumBuckets, + MockBucketSnapshotStorage storage) + throws Exception { + storage.start(); + + ManagedLedger mockLedger = mock(ManagedLedger.class); + NavigableMap ledgerInfo = new TreeMap<>(); + ledgerInfo.put(firstLedgerId, mock(LedgerInfo.class)); + when(mockLedger.getLedgersInfo()).thenReturn(ledgerInfo); + when(mockLedger.getName()).thenReturn("test-ledger"); + + ManagedCursor mockCursor = new MockManagedCursor("test-cursor") { + @Override + public ManagedLedger getManagedLedger() { + return mockLedger; + } + + @Override + public Position getMarkDeletedPosition() { + return PositionFactory.create(firstLedgerId, -1); + } + }; + + AbstractPersistentDispatcherMultipleConsumers disp = + mock(AbstractPersistentDispatcherMultipleConsumers.class); + Clock mockClock = mock(Clock.class); + AtomicLong mockClockTime = new AtomicLong(); + when(mockClock.millis()).then(x -> mockClockTime.get()); + doReturn(mockCursor).when(disp).getCursor(); + doReturn("persistent://public/default/testDelay" + " / " + mockCursor.getName()).when(disp).getName(); + + BucketDelayedDeliveryTracker tracker = new BucketDelayedDeliveryTracker(disp, mock(Timer.class), + 100000, mockClock, true, storage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, maxNumBuckets); + return new TrackerWithStorage(tracker, storage, mockClockTime); + } + + @Test + public void testTrimRemovesOrphanedBuckets() throws Exception { + long firstLedgerId = 31L; + int messageCount = 36; + TrackerWithStorage ts = createTrackerWithMockLedger(firstLedgerId, 5); + + for (int i = 1; i <= messageCount; i++) { + ts.tracker.addMessage(i, i, i * 10); + } + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + int bucketCount = ts.tracker.getImmutableBuckets().asMapOfRanges().size(); + assertTrue(bucketCount <= 5, + "Bucket count " + bucketCount + " should be <= maxNumBuckets=5 after trim+merge"); + + ts.tracker.getImmutableBuckets().asMapOfRanges().forEach((range, bucket) -> + assertTrue(range.lowerEndpoint() >= firstLedgerId, + "Remaining bucket range " + range + " should be >= " + firstLedgerId)); + + long messagesAfterTrim = ts.tracker.getNumberOfDelayedMessages(); + ts.clockTime.set(messageCount * 10); + NavigableSet scheduledMessages = ts.tracker.getScheduledMessages(1); + assertTrue(scheduledMessages.stream().noneMatch(position -> position.getLedgerId() < firstLedgerId), + "Trimmed ledgers should not be returned from the loaded shared queue"); + assertEquals(ts.tracker.getNumberOfDelayedMessages(), messagesAfterTrim - scheduledMessages.size()); + + ts.close(); + } + + @Test + public void testTrimHandlesDeleteFailure() throws Exception { + long firstLedgerId = 50L; + int messageCount = 31; + TrackerWithStorage ts = createTrackerWithMockLedger(firstLedgerId, 5); + + // MaxRetryTimes=3 means the first trim delete attempt plus 3 retries = 4 exceptions consumed. + for (int i = 0; i < 4; i++) { + ts.storage.injectDeleteException( + new BucketSnapshotPersistenceException("Delete failed")); + } + + for (int i = 1; i <= messageCount; i++) { + ts.tracker.addMessage(i, i, i * 10); + } + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + Awaitility.await().untilAsserted(() -> + assertTrue(ts.storage.deleteExceptionQueue.isEmpty(), + "Delete exception should have been consumed")); + + // Trim failed on the first orphaned bucket; the sequential chain stopped, so all + // 6 orphaned buckets remain in immutableBuckets. + assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().size() > 0, + "Orphaned buckets should remain when trim delete fails"); + ts.tracker.getImmutableBuckets().asMapOfRanges().forEach((range, bucket) -> + assertTrue(range.upperEndpoint() < firstLedgerId, + "Remaining bucket " + range + " should be an orphaned bucket")); + + // numberDelayedMessages is unchanged because failed deletes do not decrement the count. + assertEquals(ts.tracker.getNumberOfDelayedMessages(), messageCount); + + ts.close(); + } + + @Test + public void testClearRunsAfterInFlightTrimFailure() throws Exception { + long firstLedgerId = 50L; + int messageCount = 31; + BlockingDeleteStorage storage = new BlockingDeleteStorage(); + TrackerWithStorage ts = createTrackerWithMockLedger(firstLedgerId, 5, storage); + + for (int i = 1; i <= messageCount; i++) { + ts.tracker.addMessage(i, i, i * 10); + } + Awaitility.await().untilAsserted(() -> + assertTrue(storage.deleteCalls.get() > 0, "Trim delete should be in flight")); + + CompletableFuture clearFuture = ts.tracker.clear(); + storage.firstDeleteFuture.completeExceptionally(new BucketSnapshotPersistenceException("Delete failed")); + + clearFuture.get(1, TimeUnit.MINUTES); + assertEquals(ts.tracker.getNumberOfDelayedMessages(), 0); + assertEquals(ts.tracker.getImmutableBuckets().asMapOfRanges().size(), 0); + assertEquals(ts.tracker.getLastMutableBucket().size(), 0); + assertEquals(ts.tracker.getSharedBucketPriorityQueue().size(), 0); + + ts.close(); + } + + @Test + public void testTrimWithNoOrphanedBuckets() throws Exception { + TrackerWithStorage ts = createTrackerWithMockLedger(0L, 5); + + for (int i = 1; i <= 31; i++) { + ts.tracker.addMessage(i, i, i * 10); + } + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + int bucketCount = ts.tracker.getImmutableBuckets().asMapOfRanges().size(); + assertTrue(bucketCount <= 5, + "Bucket count " + bucketCount + " should be <= maxNumBuckets=5"); + assertTrue(bucketCount > 0, "Should have at least one bucket after merge"); + + ts.close(); + } + + @Test + public void testMergeEarlyReturnWhenWithinLimit() throws Exception { + TrackerWithStorage ts = createTrackerWithMockLedger(0L, 50); + + for (int i = 1; i <= 30; i++) { + ts.tracker.addMessage(i, i, i * 10); + } + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + int bucketCount = ts.tracker.getImmutableBuckets().asMapOfRanges().size(); + assertTrue(bucketCount < 50, + "Bucket count " + bucketCount + " should be well below maxNumBuckets=50"); + + long msgsBefore = ts.tracker.getNumberOfDelayedMessages(); + ts.tracker.addMessage(200, 200, 200 * 10); + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + assertEquals(ts.tracker.getNumberOfDelayedMessages(), msgsBefore + 1); + + ts.close(); + } } From 964a723c7e07a3a07c93e01c819e08326ba89ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=81=93=E5=90=9B-=20Tao=20Jiuming?= Date: Wed, 24 Jun 2026 18:02:51 +0800 Subject: [PATCH 175/201] [fix][broker] Guard BucketDelayedDeliveryTracker.nextDeliveryTime against empty queues (#26080) (cherry picked from commit 51e2b9a6bc9f739d3fc33b24821df15c588d4055) --- .../bucket/BucketDelayedDeliveryTracker.java | 5 ++ .../BucketDelayedDeliveryTrackerTest.java | 46 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 7a1b95a3a4a73..4b6121fd3e4a0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -593,6 +593,11 @@ protected synchronized long nextDeliveryTime() { return sharedBucketPriorityQueue.peekN1(); } else if (sharedBucketPriorityQueue.isEmpty() && !lastMutableBucket.isEmpty()) { return lastMutableBucket.nextDeliveryTime(); + } else if (lastMutableBucket.isEmpty() && sharedBucketPriorityQueue.isEmpty()) { + // numberDelayedMessages can be > 0 while both queues are empty (e.g. remaining + // messages live in not-yet-loaded snapshot segments). Returning Long.MAX_VALUE + // signals "no imminent delivery" without throwing on the empty queues. + return Long.MAX_VALUE; } long timestamp = lastMutableBucket.nextDeliveryTime(); long bucketTimestamp = sharedBucketPriorityQueue.peekN1(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index ee0ec0d92945a..4becc00ab3f31 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -676,4 +676,50 @@ public void testMergeEarlyReturnWhenWithinLimit() throws Exception { ts.close(); } + + @Test + public void testGetScheduledMessagesWhenAllOrphaned() throws Exception { + // Reproduces IAE in nextDeliveryTime: when every delayed message lies below the + // mark-delete position, the filter in getScheduledMessages pops the in-memory + // messages without returning them. If the immutable bucket has additional messages + // still in storage (later snapshot segments), numberDelayedMessages stays > 0 + // while both the mutable bucket and the shared priority queue are empty. + // The trailing updateTimer -> nextDeliveryTime must not throw. + long firstLedgerId = 50L; + TrackerWithStorage ts = createTrackerWithMockLedger(firstLedgerId, 50); + + // Five delayed messages on the same orphaned ledger (ledgerId < firstLedgerId). + // They share a mutable bucket because seal requires a strictly greater ledgerId. + // Timestamps are 100ms apart so each lands in its own snapshot segment + // (timeStep=10ms); only the first segment is loaded into the shared queue at seal. + for (int i = 1; i <= 5; i++) { + ts.tracker.addMessage(1, i, i * 100); + } + // A new orphaned ledgerId triggers the seal, producing immutable bucket [1..1] + // with 5 messages across 5 segments; shared queue holds just the first segment. + ts.tracker.addMessage(2, 1, 600); + + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + // In strict deliver-at mode getCutoffTime() is just clock.millis(), so advancing the + // clock past the trigger message's deliverAt (600) is enough for + // moveScheduledMessageToSharedQueue to flush the mutable bucket into the shared queue. + ts.clockTime.set(700); + + // Both queues end up empty (filter pops the two in-memory messages), but + // numberDelayedMessages is still 4 (segments 2..5 remain in storage). + NavigableSet scheduledMessages = ts.tracker.getScheduledMessages(10); + assertTrue(scheduledMessages.isEmpty(), + "Orphaned messages should be filtered out, not returned"); + assertTrue(ts.tracker.getNumberOfDelayedMessages() > 0, + "Remaining storage-only messages should keep the counter > 0"); + + // hasMessageAvailable calls nextDeliveryTime while numberDelayedMessages > 0; + // it must not throw IAE. + assertFalse(ts.tracker.hasMessageAvailable()); + + ts.close(); + } } From 9cf8a613f76a8e97678b7961ee63fc5d6e96e447 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Thu, 25 Jun 2026 03:34:33 +0800 Subject: [PATCH 176/201] [improve][broker] Improve dispatch performance by summing entry bytes with a loop (#26055) (cherry picked from commit b2f37560b356a011bcd9bdb7d37d8ce01d95ea98) --- ...PersistentDispatcherMultipleConsumers.java | 10 +++ ...PersistentDispatcherMultipleConsumers.java | 2 +- ...entDispatcherMultipleConsumersClassic.java | 2 +- .../PersistentDispatcherTotalBytesTest.java | 78 +++++++++++++++++++ 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherTotalBytesTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java index 79d365b9fee21..3a63dc6594704 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java @@ -18,8 +18,10 @@ */ package org.apache.pulsar.broker.service.persistent; +import java.util.List; import java.util.Map; import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers; @@ -64,4 +66,12 @@ public AbstractPersistentDispatcherMultipleConsumers(Subscription subscription, public abstract Map getBucketDelayedIndexStats(); public abstract boolean isClassic(); + + static long getTotalBytesSize(List entries) { + long totalBytesSize = 0; + for (int i = 0, entriesSize = entries.size(); i < entriesSize; i++) { + totalBytesSize += entries.get(i).getLength(); + } + return totalBytesSize; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index c855550069e09..71e822e6dd6a9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -707,7 +707,7 @@ public final synchronized void readEntriesComplete(List entries, Object c log.debug("[{}] Distributing {} messages to {} consumers", name, entries.size(), consumerList.size()); } - long totalBytesSize = entries.stream().mapToLong(Entry::getLength).sum(); + long totalBytesSize = getTotalBytesSize(entries); updatePendingBytesToDispatch(totalBytesSize); // dispatch messages to a separate thread, but still in order for this subscription diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java index 59f2ce973dac9..7c35e626dc4db 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java @@ -621,7 +621,7 @@ public final synchronized void readEntriesComplete(List entries, Object c log.debug("[{}] Distributing {} messages to {} consumers", name, entries.size(), consumerList.size()); } - long size = entries.stream().mapToLong(Entry::getLength).sum(); + long size = getTotalBytesSize(entries); updatePendingBytesToDispatch(size); // dispatch messages to a separate thread, but still in order for this subscription diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherTotalBytesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherTotalBytesTest.java new file mode 100644 index 0000000000000..239a698ab011b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherTotalBytesTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service.persistent; + +import static org.testng.Assert.assertEquals; +import java.util.ArrayList; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.impl.EntryImpl; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class PersistentDispatcherTotalBytesTest { + + @DataProvider(name = "entryCounts") + public Object[][] entryCounts() { + return new Object[][] { + {0}, + {1}, + {32}, + {1024} + }; + } + + @Test(dataProvider = "entryCounts") + public void testGetTotalBytesSize(int entryCount) { + EntriesAndExpectedSize entries = entriesWithVaryingPayloadSizes(entryCount); + try { + assertEquals(AbstractPersistentDispatcherMultipleConsumers.getTotalBytesSize(entries.entries), + entries.expectedSize); + } finally { + entries.release(); + } + } + + private static EntriesAndExpectedSize entriesWithVaryingPayloadSizes(int entryCount) { + EntriesAndExpectedSize entries = new EntriesAndExpectedSize(entryCount); + for (int i = 0; i < entryCount; i++) { + int payloadSize = payloadSize(i); + entries.entries.add(EntryImpl.create(1, i, new byte[payloadSize])); + entries.expectedSize += payloadSize; + } + return entries; + } + + private static int payloadSize(int index) { + return index % 97 == 0 ? 4096 : 1 + ((index * 31) & 1023); + } + + private static final class EntriesAndExpectedSize { + private final ArrayList entries; + private long expectedSize; + + private EntriesAndExpectedSize(int entryCount) { + this.entries = new ArrayList<>(entryCount); + } + + private void release() { + entries.forEach(Entry::release); + } + } +} From 86256f15cfa415fe3febc47bf0bd840fa48242ad Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Tue, 30 Jun 2026 00:23:12 +0800 Subject: [PATCH 177/201] [fix][broker] Fix replicator getting stuck under rate limiter throttling and honor readBatchSize/maxReadSizeBytes on the default read path (#26005) Co-authored-by: Lari Hotari (cherry picked from commit abca10f9ce93da79a2842455dafda707042d42fa) --- .../persistent/GeoPersistentReplicator.java | 6 +- .../persistent/PersistentReplicator.java | 185 +++++------- .../broker/service/OneWayReplicatorTest.java | 80 ++++++ .../PersistentReplicatorInflightTaskTest.java | 265 +++++++++++------- 4 files changed, 321 insertions(+), 215 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java index 7345e938ad4b4..2d20ccb79704e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java @@ -255,9 +255,9 @@ protected boolean replicateEntries(List entries, final InFlightTask inFli * Explain the result of the race-condition between: * - {@link #readMoreEntries} * - {@link #beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding)} - * Since {@link #acquirePermitsIfNotFetchingSchema} and - * {@link #beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding)} acquire the - * same lock, it is safe. + * Since the read scheduling path in {@link #readMoreEntries()} and + * {@link #beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding)} update in-flight + * read state under the same lock, it is safe. */ beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding.Fetching_Schema); inFlightTask.incCompletedEntries(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 392f5eb98db43..ed599cd220595 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -38,7 +38,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -91,7 +90,7 @@ public abstract class PersistentReplicator extends AbstractReplicator protected Optional dispatchRateLimiter = Optional.empty(); private final Object dispatchRateLimiterLock = new Object(); - private int readBatchSize; + private volatile int readBatchSize; private final int readMaxSizeBytes; private final int producerQueueThreshold; @@ -138,10 +137,8 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man this.expiryMonitor = new PersistentMessageExpiryMonitor(localTopic, Codec.decode(cursor.getName()), cursor, null); - readBatchSize = Math.min( - producerQueueSize, - localTopic.getBrokerService().pulsar().getConfiguration().getDispatcherMaxReadBatchSize()); - readMaxSizeBytes = localTopic.getBrokerService().pulsar().getConfiguration().getDispatcherMaxReadSizeBytes(); + readBatchSize = getMaxReadBatchSize(); + readMaxSizeBytes = brokerService.pulsar().getConfiguration().getDispatcherMaxReadSizeBytes(); producerQueueThreshold = (int) (producerQueueSize * 0.9); this.initializeDispatchRateLimiterIfNeeded(); @@ -149,6 +146,10 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man startProducer(); } + private int getMaxReadBatchSize() { + return Math.min(producerQueueSize, brokerService.pulsar().getConfiguration().getDispatcherMaxReadBatchSize()); + } + @Override protected void setProducerAndTriggerReadEntries(Producer producer) { /** @@ -212,72 +213,62 @@ protected void disableReplicatorRead() { this.cursor.setInactive(); } - @Data - @AllArgsConstructor - private static class AvailablePermits { - private int messages; - private long bytes; - - /** - * messages, bytes - * 0, O: Producer queue is full, no permits. - * -1, -1: Rate Limiter reaches limit. - * >0, >0: available permits for read entries. - */ - public boolean isExceeded() { - return messages == -1 && bytes == -1; - } - + private record ReadLimits(int messages, long bytes) { public boolean isReadable() { return messages > 0 && bytes > 0; } } /** - * Calculate available permits for read entries. + * Calculate read limits for a read operation. Takes the rate limiter into account if it's enabled. + * Also limits to current readBatchSize and readMaxSizeBytes. */ - private AvailablePermits getRateLimiterAvailablePermits(int availablePermits) { + private ReadLimits getReadLimits(int permits) { // return 0, if Producer queue is full, it will pause read entries. - if (availablePermits <= 0) { + if (permits <= 0) { if (log.isDebugEnabled()) { - log.debug("[{}] Producer queue is full, availablePermits: {}, pause reading", - replicatorId, availablePermits); + log.debug("[{}] Producer queue is full, permits: {}, pause reading", replicatorId, permits); } - return new AvailablePermits(0, 0); + return new ReadLimits(0, 0); } - long availablePermitsOnMsg = -1; - long availablePermitsOnByte = -1; + long readLimitOnMsg; + long readLimitOnByte; // handle rate limit if (dispatchRateLimiter.isPresent() && dispatchRateLimiter.get().isDispatchRateLimitingEnabled()) { DispatchRateLimiter rateLimiter = dispatchRateLimiter.get(); - // if dispatch-rate is in msg then read only msg according to available permit - availablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg(); - availablePermitsOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte(); - // no permits from rate limit - if (availablePermitsOnByte == 0 || availablePermitsOnMsg == 0) { + // rateLimiter returns -1 if there is no rate limit configured + readLimitOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg(); + readLimitOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte(); + // no permits from rate limit when either limit is 0 + if (readLimitOnByte == 0 || readLimitOnMsg == 0) { if (log.isDebugEnabled()) { - log.debug("[{}] message-read exceeded topic replicator message-rate {}/{}," - + " schedule after a {}", + log.debug("[{}] Message-read exceeded topic replicator rate limit," + + " dispatchRateOnMsg: {}, dispatchRateOnByte: {}," + + " readLimitOnMsg: {}, readLimitOnByte: {}", replicatorId, rateLimiter.getDispatchRateOnMsg(), rateLimiter.getDispatchRateOnByte(), - MESSAGE_RATE_BACKOFF_MS); + readLimitOnMsg, + readLimitOnByte); } - return new AvailablePermits(-1, -1); + return new ReadLimits(-1, -1); } + // use given permits if no rate limit configured, otherwise limit to returned rate limiter permits + readLimitOnMsg = readLimitOnMsg == -1 ? permits : Math.min(permits, readLimitOnMsg); + // use readMaxSizeBytes if no rate limit configured, otherwise limit to returned rate limiter permits + readLimitOnByte = readLimitOnByte == -1 ? readMaxSizeBytes : Math.min(readMaxSizeBytes, readLimitOnByte); + } else { + readLimitOnMsg = permits; + readLimitOnByte = readMaxSizeBytes; } - availablePermitsOnMsg = - availablePermitsOnMsg == -1 ? availablePermits : Math.min(availablePermits, availablePermitsOnMsg); - availablePermitsOnMsg = Math.min(availablePermitsOnMsg, readBatchSize); - - availablePermitsOnByte = - availablePermitsOnByte == -1 ? readMaxSizeBytes : Math.min(readMaxSizeBytes, availablePermitsOnByte); + // limit messages to current read batch size + readLimitOnMsg = Math.min(readLimitOnMsg, readBatchSize); - return new AvailablePermits((int) availablePermitsOnMsg, availablePermitsOnByte); + return new ReadLimits((int) readLimitOnMsg, readLimitOnByte); } public void disconnectIfNoTrafficAndBacklog() { @@ -308,8 +299,39 @@ protected void readMoreEntries() { if (state.equals(Terminated) || state.equals(Terminating)) { return; } - // Acquire permits and check state of producer. - InFlightTask newInFlightTask = acquirePermitsIfNotFetchingSchema(); + InFlightTask newInFlightTask = null; + ReadLimits readLimits = null; + synchronized (inFlightTasks) { + if (hasPendingRead()) { + log.debug("Skip the reading because there is a pending read task"); + } else if (waitForCursorRewindingRefCnf > 0) { + log.debug("Skip the reading due to new detected schema"); + } else if (state != Started) { + log.debug("Skip the reading because producer has not started"); + } else { + int permits = getPermitsIfNoPendingRead(); + if (permits > 0) { + if (!isWritable()) { + log.debug("Throttling replication traffic to a single message permit because producer is not " + + "writable"); + // Minimize the read size if the producer is disconnected or the window is already full. + permits = 1; + } + + readLimits = getReadLimits(permits); + if (readLimits.isReadable()) { + newInFlightTask = createOrRecycleInFlightTaskIntoQueue(cursor.getReadPosition(), + readLimits.messages); + } else { + // no rate limiter permits from rate limit + if (log.isDebugEnabled()) { + log.debug("[{}] Throttling replication traffic. Messages To Read {}, Bytes To Read {}", + replicatorId, readLimits.messages, readLimits.bytes); + } + } + } + } + } if (newInFlightTask == null) { // no permits from rate limit if (log.isDebugEnabled()) { @@ -318,48 +340,14 @@ protected void readMoreEntries() { if (!hasPendingRead()) { topic.getBrokerService().executor().schedule( () -> readMoreEntries(), MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS); - return; - } else { - return; - } - } - // If disabled RateLimiter. - if (!dispatchRateLimiter.isPresent() || !dispatchRateLimiter.get().isDispatchRateLimitingEnabled()) { - cursor.asyncReadEntriesOrWait(newInFlightTask.readingEntries, -1, this, - newInFlightTask/* Context object */, topic.getMaxReadPosition()); - return; - } - // No permits of RateLimiter. - AvailablePermits availablePermits = getRateLimiterAvailablePermits(newInFlightTask.readingEntries); - if (!availablePermits.isReadable()) { - // no rate limiter permits from rate limit - if (log.isDebugEnabled()) { - log.debug("[{}] Throttling replication traffic. Messages To Read {}, Bytes To Read {}", - replicatorId, availablePermits.getMessages(), availablePermits.getBytes()); } - topic.getBrokerService().executor().schedule( - () -> readMoreEntries(), MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS); return; } - // Has permits of RateLimiter. - int messagesToRead = availablePermits.getMessages(); - long bytesToRead = availablePermits.getBytes(); - if (!isWritable()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Throttling replication traffic because producer is not writable", replicatorId); - } - // Minimize the read size if the producer is disconnected or the window is already full - messagesToRead = 1; - } - // Update acquired permits exceeds limitation. - if (messagesToRead < newInFlightTask.readingEntries) { - newInFlightTask.setReadingEntries(messagesToRead); - } if (log.isDebugEnabled()) { - log.debug("[{}] Schedule read of {} messages or {} bytes", replicatorId, newInFlightTask.readingEntries, - bytesToRead); + log.debug("[{}] Schedule read of {} messages or {} bytes", replicatorId, + newInFlightTask.readingEntries, readLimits.bytes); } - cursor.asyncReadEntriesOrWait(newInFlightTask.readingEntries, bytesToRead, this, + cursor.asyncReadEntriesOrWait(newInFlightTask.readingEntries, readLimits.bytes, this, newInFlightTask/* Context object */, topic.getMaxReadPosition()); } @@ -414,7 +402,7 @@ public void readEntriesComplete(List entries, Object ctx) { inFlightTask.setEntries(entries); // After the replicator starts, the speed will be gradually increased. - int maxReadBatchSize = topic.getBrokerService().pulsar().getConfiguration().getDispatcherMaxReadBatchSize(); + int maxReadBatchSize = getMaxReadBatchSize(); if (readBatchSize < maxReadBatchSize) { int newReadBatchSize = Math.min(readBatchSize * 2, maxReadBatchSize); if (log.isDebugEnabled()) { @@ -573,7 +561,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { } // Reduce read batch size to avoid flooding bookies with retries - readBatchSize = topic.getBrokerService().pulsar().getConfiguration().getDispatcherMinReadBatchSize(); + readBatchSize = brokerService.pulsar().getConfiguration().getDispatcherMinReadBatchSize(); long waitTimeMillis = readFailureBackoff.next().toMillis(); @@ -930,29 +918,6 @@ InFlightTask createOrRecycleInFlightTaskIntoQueue(Position readPos, int readingE } } - protected InFlightTask acquirePermitsIfNotFetchingSchema() { - synchronized (inFlightTasks) { - if (hasPendingRead()) { - log.info("[{}] Skip the reading because there is a pending read task", replicatorId); - return null; - } - if (waitForCursorRewindingRefCnf > 0) { - log.info("[{}] Skip the reading due to new detected schema", replicatorId); - return null; - } - if (state != Started) { - log.info("[{}] Skip the reading because producer has not started [{}]", replicatorId, state); - return null; - } - // Guarantee that there is a unique cursor reading task. - int permits = getPermitsIfNoPendingRead(); - if (permits == 0) { - return null; - } - return createOrRecycleInFlightTaskIntoQueue(cursor.getReadPosition(), permits); - } - } - protected int getPermitsIfNoPendingRead() { synchronized (inFlightTasks) { for (InFlightTask task : inFlightTasks) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index 673dd082ba491..67151479ed22c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -111,6 +111,7 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; import org.apache.pulsar.common.policies.data.PublishRate; import org.apache.pulsar.common.policies.data.ReplicatorStats; @@ -2213,6 +2214,85 @@ public void testReplicatorsInflightTaskListIsEmptyAfterReplicationFinished() thr ensureNoBacklogByInflightTask(getReplicator(topicName)); } + @DataProvider + public Object[][] replicatorDispatchRateLimits() { + return new Object[][] { + {1, -1L}, + {-1, 1L} + }; + } + + @Test(timeOut = 90_000, dataProvider = "replicatorDispatchRateLimits") + public void testReplicatorContinuesAfterRateLimiterHasNoPermits(int messageRate, long byteRate) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + final String subscriptionName = "sub"; + final List messages = Arrays.asList("msg-0", "msg-1", "msg-2"); + DispatchRate dispatchRate = DispatchRate.builder() + .dispatchThrottlingRateInMsg(messageRate) + .dispatchThrottlingRateInByte(byteRate) + .ratePeriodInSecond(2) + .build(); + Producer producer = null; + Consumer consumer = null; + boolean topicCreated = false; + boolean dispatchRateConfigured = false; + try { + admin1.topics().createNonPartitionedTopic(topicName); + topicCreated = true; + admin1.topicPolicies().setReplicatorDispatchRate(topicName, dispatchRate); + dispatchRateConfigured = true; + GeoPersistentReplicator replicator = getReplicator(topicName); + Awaitility.await().untilAsserted(() -> { + assertTrue(replicator.getRateLimiter().isPresent()); + assertEquals(replicator.getRateLimiter().get().getDispatchRateOnMsg(), messageRate); + assertEquals(replicator.getRateLimiter().get().getDispatchRateOnByte(), byteRate); + }); + consumer = client2.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subscriptionName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + producer = client1.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(false) + .create(); + + for (String message : messages) { + producer.send(message); + } + + Set expected = new HashSet<>(messages); + Set received = new HashSet<>(); + Consumer subscribedConsumer = consumer; + Awaitility.await().atMost(Duration.ofSeconds(60)).untilAsserted(() -> { + Message message = subscribedConsumer.receive(1, TimeUnit.SECONDS); + if (message != null) { + received.add(message.getValue()); + subscribedConsumer.acknowledge(message); + } + assertEquals(received, expected); + }); + waitForReplicationTaskFinish(topicName); + ensureNoBacklogByInflightTask(replicator); + } finally { + if (producer != null) { + producer.close(); + } + if (consumer != null) { + consumer.close(); + } + if (dispatchRateConfigured) { + admin1.topicPolicies().removeReplicatorDispatchRate(topicName); + } + if (topicCreated) { + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1)); + waitReplicatorStopped(topicName, false); + admin1.topics().delete(topicName, true); + admin2.topics().delete(topicName, true); + } + } + } + @DataProvider public Object[] isPartitioned() { return new Object[]{ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java index 83529969df417..773f2933ce7c0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java @@ -18,44 +18,64 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import io.netty.channel.EventLoopGroup; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerTest; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.AbstractReplicator; -import org.apache.pulsar.broker.service.BrokerServiceInternalMethodInvoker; +import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.OneWayReplicatorTestBase; import org.apache.pulsar.broker.service.persistent.PersistentReplicator.InFlightTask; import org.apache.pulsar.broker.service.persistent.PersistentReplicator.ProducerSendCallback; import org.apache.pulsar.broker.service.persistent.PersistentReplicator.ReasonOfWaitForCursorRewinding; +import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.awaitility.Awaitility; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Slf4j @@ -199,6 +219,63 @@ public void testFailedPublishCompletesInFlightTask() throws Exception { } } + @DataProvider + public Object[][] readSchedulingLimits() { + return new Object[][] { + {"message permits exhausted", 0, -1L, true, false, 0, 0L}, + {"byte permits exhausted", -1, 0L, true, false, 0, 0L}, + {"message permits limit read batch", 5, -1L, true, true, 5, 1024L}, + {"byte permits limit read size", -1, 512L, true, true, 100, 512L}, + {"non-writable producer limits read batch", 5, 512L, false, true, 1, 512L} + }; + } + + @Test(dataProvider = "readSchedulingLimits") + public void testReadMoreEntriesSchedulesCursorReadWithReadLimits(String scenario, + long availableMessages, + long availableBytes, + boolean writable, + boolean expectRead, + int expectedMessages, + long expectedBytes) throws Exception { + TestReplicatorFixture fixture = newTestReplicatorFixture(writable); + PersistentReplicator replicator = fixture.replicator; + DispatchRateLimiter rateLimiter = mock(DispatchRateLimiter.class); + when(rateLimiter.isDispatchRateLimitingEnabled()).thenReturn(true); + when(rateLimiter.getAvailableDispatchRateLimitOnMsg()).thenReturn(availableMessages); + when(rateLimiter.getAvailableDispatchRateLimitOnByte()).thenReturn(availableBytes); + replicator.dispatchRateLimiter = Optional.of(rateLimiter); + + replicator.readMoreEntries(); + + if (expectRead) { + assertEquals(replicator.inFlightTasks.size(), 1, scenario); + InFlightTask inFlightTask = replicator.inFlightTasks.peek(); + verify(fixture.cursor).asyncReadEntriesOrWait(eq(expectedMessages), eq(expectedBytes), + same(replicator), same(inFlightTask), any(Position.class)); + assertEquals(inFlightTask.getReadingEntries(), expectedMessages, scenario); + verify(fixture.executor, never()).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + } else { + verify(fixture.cursor, never()).asyncReadEntriesOrWait(anyInt(), anyLong(), any(), any(), any()); + assertTrue(replicator.inFlightTasks.isEmpty(), scenario); + verify(fixture.executor).schedule(any(Runnable.class), eq((long) PersistentTopic.MESSAGE_RATE_BACKOFF_MS), + eq(TimeUnit.MILLISECONDS)); + } + } + + @Test + public void testReadMoreEntriesSkipsReadWhenPendingReadExists() throws Exception { + TestReplicatorFixture fixture = newTestReplicatorFixture(true); + PersistentReplicator replicator = fixture.replicator; + replicator.inFlightTasks.add(new InFlightTask(PositionFactory.create(1, 1), 5, replicator.getReplicatorId())); + + replicator.readMoreEntries(); + + verify(fixture.cursor, never()).asyncReadEntriesOrWait(anyInt(), anyLong(), any(), any(), any()); + verify(fixture.executor, never()).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + assertEquals(replicator.inFlightTasks.size(), 1); + } + @Test public void testCreateOrRecycleInFlightTaskIntoQueue() throws Exception { log.info("Starting testCreateOrRecycleInFlightTaskIntoQueue"); @@ -389,111 +466,94 @@ public void testGetPermitsIfNoPendingRead() throws Exception { } } - @Test - public void testAcquirePermitsIfNotFetchingSchema() throws Exception { - log.info("Starting testAcquirePermitsIfNotFetchingSchema"); - // Get the replicator for the test topic - PersistentReplicator replicator = getReplicator(topicName); - Assert.assertNotNull(replicator, "Replicator should not be null"); + @SuppressWarnings("unchecked") + private TestReplicatorFixture newTestReplicatorFixture(boolean writable) throws Exception { + ServiceConfiguration configuration = new ServiceConfiguration(); + configuration.setClusterName("local"); + configuration.setReplicationProducerQueueSize(1000); + configuration.setDispatcherMaxReadBatchSize(100); + configuration.setDispatcherMaxReadSizeBytes(1024); + + PulsarService pulsar = mock(PulsarService.class); + when(pulsar.getConfiguration()).thenReturn(configuration); + when(pulsar.getConfig()).thenReturn(configuration); + when(pulsar.getClient()).thenReturn(mock(PulsarClientImpl.class)); + when(pulsar.getAdminClient()).thenReturn(mock(PulsarAdmin.class)); + + BrokerService brokerService = mock(BrokerService.class); + EventLoopGroup executor = mock(EventLoopGroup.class); + when(brokerService.pulsar()).thenReturn(pulsar); + when(brokerService.getPulsar()).thenReturn(pulsar); + when(brokerService.executor()).thenReturn(executor); + + ProducerBuilder producerBuilder = mock(ProducerBuilder.class); + when(producerBuilder.topic(anyString())).thenReturn(producerBuilder); + when(producerBuilder.messageRoutingMode(any())).thenReturn(producerBuilder); + when(producerBuilder.enableBatching(anyBoolean())).thenReturn(producerBuilder); + when(producerBuilder.sendTimeout(anyInt(), any(TimeUnit.class))).thenReturn(producerBuilder); + when(producerBuilder.maxPendingMessages(anyInt())).thenReturn(producerBuilder); + when(producerBuilder.producerName(anyString())).thenReturn(producerBuilder); + + PulsarClientImpl replicationClient = mock(PulsarClientImpl.class); + when(replicationClient.newProducer(any(Schema.class))).thenReturn(producerBuilder); + + PersistentTopic topic = mock(PersistentTopic.class); + when(topic.getName()).thenReturn("persistent://prop/ns/test-read-scheduling"); + when(topic.getReplicatorPrefix()).thenReturn("pulsar.repl"); + when(topic.getBrokerService()).thenReturn(brokerService); + when(topic.getMaxReadPosition()).thenReturn(PositionFactory.create(1, 100)); + + ManagedCursor cursor = mock(ManagedCursor.class); + when(cursor.getName()).thenReturn("pulsar.repl.remote"); + when(cursor.getReadPosition()).thenReturn(PositionFactory.create(1, 1)); + + TestPersistentReplicator replicator = new TestPersistentReplicator(topic, cursor, brokerService, + replicationClient, mock(PulsarAdmin.class), writable); + return new TestReplicatorFixture(replicator, cursor, executor); + } - // Get access to the inFlightTasks list for setup - LinkedList inFlightTasks = replicator.inFlightTasks; - Assert.assertNotNull(inFlightTasks, "InFlightTasks list should not be null"); + private static class TestReplicatorFixture { + final TestPersistentReplicator replicator; + final ManagedCursor cursor; + final EventLoopGroup executor; - // Save original tasks and clear for testing - List originalTasks = new ArrayList<>(inFlightTasks); - inFlightTasks.clear(); + TestReplicatorFixture(TestPersistentReplicator replicator, ManagedCursor cursor, EventLoopGroup executor) { + this.replicator = replicator; + this.cursor = cursor; + this.executor = executor; + } + } - // Save original state - int originalWaitForCursorRewinding = replicator.waitForCursorRewindingRefCnf; - AbstractReplicator.State originalState = replicator.getState(); + private static class TestPersistentReplicator extends PersistentReplicator { + private final boolean writable; - try { - // Test Case 1: Normal case - no pending read, not waiting for cursor rewinding, state is Started - // Should return a new InFlightTask - // First, check the current permits available - int expectedPermits = replicator.getPermitsIfNoPendingRead(); - Assert.assertTrue(expectedPermits > 0, "Should have available permits for the test"); - InFlightTask task1 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNotNull(task1, "Should return a new InFlightTask in normal case"); - Assert.assertNotNull(task1.getReadPos(), "Task should have a read position"); - Assert.assertEquals(task1.getReadingEntries(), expectedPermits, - "Task readingEntries should equal the number of permits available"); - Assert.assertTrue(inFlightTasks.contains(task1), - "Task should be added to the inFlightTasks list"); - - // Test Case 2: With pending read - should return null - inFlightTasks.clear(); - Position position1 = PositionFactory.create(1, 1); - InFlightTask pendingReadTask = new InFlightTask(position1, 5, ""); - // Don't set readoutEntries to simulate pending read - inFlightTasks.add(pendingReadTask); - InFlightTask task2 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNull(task2, "Should return null when there is a pending read"); + TestPersistentReplicator(PersistentTopic topic, ManagedCursor cursor, BrokerService brokerService, + PulsarClientImpl replicationClient, PulsarAdmin replicationAdmin, boolean writable) + throws PulsarServerException { + super("local", topic, cursor, "remote", topic.getName(), brokerService, replicationClient, + replicationAdmin); + this.writable = writable; + this.state = State.Started; + } - // Test Case 3: With waitForCursorRewinding=true - should return null - inFlightTasks.clear(); - replicator.waitForCursorRewindingRefCnf = 1; - InFlightTask task3 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNull(task3, "Should return null when waiting for cursor rewinding"); - // Reset for next test - replicator.waitForCursorRewindingRefCnf = 0; - - // Test Case 4: With state != Started - should return null - // We need to use reflection to modify the state since it's protected by AtomicReferenceFieldUpdater - BrokerServiceInternalMethodInvoker.replicatorSetState(replicator, AbstractReplicator.State.Starting); - InFlightTask task4 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNull(task4, "Should return null when state is not Started"); - // Reset state for next test - BrokerServiceInternalMethodInvoker.replicatorSetState(replicator, AbstractReplicator.State.Started); - - // Test Case 5: With limited permits - verify readingEntries is set correctly - inFlightTasks.clear(); - // Add a task with some in-flight messages to reduce available permits - Position positionLimited = PositionFactory.create(10, 10); - InFlightTask limitedTask = new InFlightTask(positionLimited, 5, ""); - // Add enough entries to leave just a small number of permits (e.g., 10) - List limitedEntries = new ArrayList<>(); - int entriesCount = 990; - for (int j = 0; j < entriesCount; j++) { - limitedEntries.add(mock(Entry.class)); - } - limitedTask.setEntries(limitedEntries); - inFlightTasks.add(limitedTask); - // Check that we have limited permits available - int limitedPermits = replicator.getPermitsIfNoPendingRead(); - Assert.assertTrue(limitedPermits > 0 && limitedPermits < 20, - "Should have a small number of permits available for testing"); - // Now acquire permits and verify readingEntries matches the limited permits - InFlightTask task5 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNotNull(task5, "Should return a task with limited permits"); - Assert.assertEquals(task5.getReadingEntries(), limitedPermits, - "Task readingEntries should equal the limited number of permits available"); - - // Test Case 6: With permits=0 - should return null - inFlightTasks.clear(); - // Add tasks that will make getPermitsIfNoPendingRead() return 0 - // We need enough in-flight messages to equal producerQueueSize - for (int i = 0; i < 10; i++) { - Position position = PositionFactory.create(i, i); - InFlightTask task = new InFlightTask(position, 5, ""); - List entries = new ArrayList<>(); - for (int j = 0; j < 100; j++) { - entries.add(mock(Entry.class)); - } - task.setEntries(entries); - inFlightTasks.add(task); - } - InFlightTask task6 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNull(task6, "Should return null when permits is 0"); - log.info("Completed testAcquirePermitsIfNotFetchingSchema"); - } finally { - // Restore original state - replicator.waitForCursorRewindingRefCnf = originalWaitForCursorRewinding; - BrokerServiceInternalMethodInvoker.replicatorSetState(replicator, originalState); - // Restore original tasks - inFlightTasks.clear(); - inFlightTasks.addAll(originalTasks); + @Override + protected void startProducer() { + // No-op for scheduling behavior tests. + } + + @Override + protected String getProducerName() { + return "test-replicator"; + } + + @Override + protected boolean isWritable() { + return writable; + } + + @Override + protected boolean replicateEntries(List entries, InFlightTask inFlightTask) { + return true; } } @@ -503,7 +563,8 @@ public static Runnable pauseReplicator(PersistentReplicator replicator) { }); replicator.beforeTerminateOrCursorRewinding(PersistentReplicator.ReasonOfWaitForCursorRewinding.Disconnecting); replicator.doRewindCursor(false); - InFlightTask inFlightTask = replicator.createOrRecycleInFlightTaskIntoQueue(PositionFactory.create(1, 1), 1); + InFlightTask inFlightTask = + replicator.createOrRecycleInFlightTaskIntoQueue(PositionFactory.create(1, 1), 1); return () -> { inFlightTask.setEntries(Collections.emptyList()); replicator.readMoreEntries(); From fe64a9ec1d68e17f29066760c1ace746b1554d67 Mon Sep 17 00:00:00 2001 From: sinan liu Date: Tue, 30 Jun 2026 00:58:18 +0800 Subject: [PATCH 178/201] [fix][broker] Forward topic policy updates after init failures (#26110) (cherry picked from commit b14524eb672181124f8834a2048b8ed6a278385d) --- .../service/persistent/PersistentTopic.java | 22 ++++++--- .../persistent/PersistentTopicTest.java | 46 +++++++++++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 26fe918a1440f..f015e7240b5a8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -4720,12 +4720,22 @@ protected CompletableFuture initTopicPolicy() { CompletableFuture> localPoliciesFuture = topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, TopicPoliciesService.GetType.LOCAL_ONLY); - return globalPoliciesFuture.thenCombine(localPoliciesFuture, (global, local) -> { - // finally update the topic policies with the latest value or loaded value - return CompletableFuture.runAsync(() -> - topicPolicyListener.completeInitialization(global.orElse(null), local.orElse(null)), - getPoliciesNotifyThread()); - }).thenCompose(Function.identity()); + CompletableFuture initialPoliciesFuture = + globalPoliciesFuture.thenCombine(localPoliciesFuture, (global, local) -> { + // finally update the topic policies with the latest value or loaded value + return CompletableFuture.runAsync(() -> + topicPolicyListener.completeInitialization(global.orElse(null), + local.orElse(null)), + getPoliciesNotifyThread()); + }).thenCompose(Function.identity()); + return initialPoliciesFuture.exceptionallyCompose(ex -> + // The topic load path logs and continues when initial policy loading fails. Make sure the + // already-registered wrapper is not left buffering future live updates forever. + CompletableFuture.runAsync( + () -> topicPolicyListener.completeInitialization(null, null), + getPoliciesNotifyThread()) + .thenCompose(__ -> FutureUtil.failedFuture( + FutureUtil.unwrapCompletionException(ex)))); }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 5b3c705688c7f..6b7a29ba3c0b8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -64,6 +64,7 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; @@ -74,6 +75,7 @@ import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.TopicPoliciesService; +import org.apache.pulsar.broker.service.TopicPolicyListener; import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -97,6 +99,7 @@ import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; +import org.mockito.ArgumentCaptor; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -733,6 +736,49 @@ public void testCheckPersistencePolicies() throws Exception { TimeUnit.MINUTES.toMillis(1)); } + @Test + public void testTopicPolicyListenerForwardsLiveUpdatesAfterInitialLoadFailure() throws Exception { + class RecordingPersistentTopic extends PersistentTopic { + final List receivedUpdates = new ArrayList<>(); + + RecordingPersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerService) { + super(topic, ledger, brokerService); + } + + @Override + public void onUpdate(TopicPolicies policies) { + receivedUpdates.add(policies); + } + } + + final String topic = "persistent://prop/ns-abc/testTopicPolicyInitFailure-" + UUID.randomUUID(); + ManagedLedger ledger = mock(ManagedLedger.class); + doReturn(new ManagedLedgerConfig()).when(ledger).getConfig(); + doReturn(Collections.emptyMap()).when(ledger).getProperties(); + + TopicPoliciesService policiesService = mock(TopicPoliciesService.class); + doReturn(policiesService).when(pulsar).getTopicPoliciesService(); + doReturn(CompletableFuture.completedFuture(true)).when(policiesService) + .registerListenerAsync(any(TopicName.class), any(TopicPolicyListener.class)); + doReturn(CompletableFuture.failedFuture(new RuntimeException("initial topic policy load failed"))) + .when(policiesService).getTopicPoliciesAsync(any(TopicName.class), + any(TopicPoliciesService.GetType.class)); + + RecordingPersistentTopic persistentTopic = + new RecordingPersistentTopic(topic, ledger, pulsar.getBrokerService()); + persistentTopic.initTopicPolicy().handle((ignored, ex) -> null).get(3, TimeUnit.SECONDS); + + ArgumentCaptor listenerCaptor = ArgumentCaptor.forClass(TopicPolicyListener.class); + verify(policiesService).registerListenerAsync(any(TopicName.class), listenerCaptor.capture()); + + TopicPolicies livePolicies = new TopicPolicies(); + livePolicies.setIsGlobal(false); + livePolicies.setMaxConsumerPerTopic(10); + listenerCaptor.getValue().onUpdate(livePolicies); + + assertEquals(persistentTopic.receivedUpdates, Collections.singletonList(livePolicies)); + } + @Test public void testDynamicConfigurationAutoSkipNonRecoverableData() throws Exception { pulsar.getConfiguration().setAutoSkipNonRecoverableData(false); From a6fe82334ec9b2af0624d709f63afc6adf3a5140 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 29 Jun 2026 21:46:27 +0300 Subject: [PATCH 179/201] [fix][test][branch-4.2] Adapt ConfigurationDataUtilsTest to Jackson 2.18.8 InetSocketAddress deserialization --- .../client/impl/conf/ConfigurationDataUtilsTest.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java index 9ca0c2d81b7e5..7b534e055c799 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java @@ -77,7 +77,15 @@ public void testLoadClientConfigurationData() { assertEquals("v2", confData.getAuthParamMap().get("k2")); assertEquals("0.0.0.0", confData.getDnsLookupBindAddress()); assertEquals(0, confData.getDnsLookupBindPort()); - assertEquals(dnsServerAddresses, confData.getDnsServerAddresses()); + // jackson-databind 2.18.8+/2.22+ defers DNS resolution when deserializing InetSocketAddress + // (CVE-2026-54514 fix), which changes the resolved/unresolved representation. Compare host + // and port — the values that must survive the config round-trip — instead of object equality. + List loadedDnsServerAddresses = confData.getDnsServerAddresses(); + assertEquals(loadedDnsServerAddresses.size(), dnsServerAddresses.size()); + for (int i = 0; i < dnsServerAddresses.size(); i++) { + assertEquals(loadedDnsServerAddresses.get(i).getHostString(), dnsServerAddresses.get(i).getHostString()); + assertEquals(loadedDnsServerAddresses.get(i).getPort(), dnsServerAddresses.get(i).getPort()); + } } @Test From 6f4a12297c3f91e90cb390c98ccf62aa8913feea Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 30 Jun 2026 01:32:02 +0300 Subject: [PATCH 180/201] Release 4.2.3 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- jetcd-core-shaded/pom.xml | 2 +- jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml | 2 +- jetty-upgrade/pom.xml | 2 +- jetty-upgrade/zookeeper-prometheus-metrics/pom.xml | 2 +- jetty-upgrade/zookeeper-with-patched-admin/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- microbench/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-bom/pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-cli-utils/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-dependencies-minimized/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-docs-tools/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/azure-data-explorer/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis-kpl-shaded/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-opentelemetry/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 135 files changed, 138 insertions(+), 138 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 3cbedba9f5752..3d5ea95d568b2 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.3-SNAPSHOT + 4.2.3 bouncy-castle-bc diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 52f809a039e18..079a6fb422c83 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.3-SNAPSHOT + 4.2.3 bcfips-include-test diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 52566cceae225..535108d749074 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 4.2.3-SNAPSHOT + 4.2.3 bouncy-castle-bcfips diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index ecd43412030ac..47f9e41f8f004 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.3-SNAPSHOT + 4.2.3 diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 9a591de7693bc..24d364509a0e7 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 4.2.3-SNAPSHOT + 4.2.3 jar Pulsar Build Tools @@ -43,7 +43,7 @@ - 2026-06-08T18:53:03Z + 2026-06-29T22:31:59Z 1.8 1.8 8 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index df2ed00fab9ac..f88d5e771c9bd 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.3-SNAPSHOT + 4.2.3 pulsar-io-distribution diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 200e54031b63d..034ac7ef584a8 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.3-SNAPSHOT + 4.2.3 pulsar-offloader-distribution diff --git a/distribution/pom.xml b/distribution/pom.xml index ff4ff01eff5bc..a2499426410ea 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.3-SNAPSHOT + 4.2.3 distribution diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index a6d53e7c5b9d8..4fac9db0832ea 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.3-SNAPSHOT + 4.2.3 pulsar-server-distribution diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index c59a0a3ce88f8..26119e6a75538 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 4.2.3-SNAPSHOT + 4.2.3 pulsar-shell-distribution diff --git a/docker/pom.xml b/docker/pom.xml index 844626bdcd048..323d69697ad5a 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.3-SNAPSHOT + 4.2.3 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 88717deda118e..aed7dfaaf7446 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.3-SNAPSHOT + 4.2.3 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 6f8ad31612b20..5e1b0c5752236 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 4.2.3-SNAPSHOT + 4.2.3 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 3a1ae930e5077..94cbfc399f7e4 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.3-SNAPSHOT + 4.2.3 jclouds-shaded diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml index 57e7474f14256..15ca62c46932c 100644 --- a/jetcd-core-shaded/pom.xml +++ b/jetcd-core-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 4.2.3-SNAPSHOT + 4.2.3 jetcd-core-shaded diff --git a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml index b37620f76dcec..1da7c921eeb13 100644 --- a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml +++ b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.3-SNAPSHOT + 4.2.3 pulsar-bookkeeper-prometheus-metrics-provider Apache Pulsar :: BookKeeper Stats Providers :: Prometheus diff --git a/jetty-upgrade/pom.xml b/jetty-upgrade/pom.xml index 99e552adb514a..d5d7860034989 100644 --- a/jetty-upgrade/pom.xml +++ b/jetty-upgrade/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.3-SNAPSHOT + 4.2.3 jetty-upgrade diff --git a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml index 510a81240e202..235c6cc0ecc4d 100755 --- a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml +++ b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar jetty-upgrade - 4.2.3-SNAPSHOT + 4.2.3 pulsar-zookeeper-prometheus-metrics diff --git a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml index 86a6dea63821b..179b7ad27d8db 100644 --- a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml +++ b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar jetty-upgrade - 4.2.3-SNAPSHOT + 4.2.3 zookeeper-with-patched-admin diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index b538c38b7df10..c2db1cc741c5f 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.3-SNAPSHOT + 4.2.3 managed-ledger diff --git a/microbench/pom.xml b/microbench/pom.xml index 76c1781a8ab31..c723936e4a142 100644 --- a/microbench/pom.xml +++ b/microbench/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 4.2.3-SNAPSHOT + 4.2.3 microbench diff --git a/pom.xml b/pom.xml index c80e28c04adf5..93f4769c439cf 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 4.2.3-SNAPSHOT + 4.2.3 Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -94,7 +94,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - 2026-06-08T18:52:58Z + 2026-06-29T22:31:55Z true 4.1.12.1 5.7.1 - 4.1.135.Final + 4.1.136.Final 0.0.26.Final 12.1.10 From 9000452d10b7ac3124bda792adf05fa14508b6de Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Mon, 13 Jul 2026 15:25:11 +0800 Subject: [PATCH 197/201] [improve][monitor][branch-4.2] Upgrade OpenTelemetry libraries (#26182) --- .../server/src/assemble/LICENSE.bin.txt | 61 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 10 +-- pom.xml | 6 +- pulsar-opentelemetry/pom.xml | 2 +- .../opentelemetry/OpenTelemetryService.java | 31 ++++++---- .../OpenTelemetryServiceTest.java | 17 ++++++ 6 files changed, 75 insertions(+), 52 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index b9815fd3927ed..adbfc41bbda6b 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -338,12 +338,12 @@ The Apache Software License, Version 2.0 - io.prometheus-simpleclient_tracer_otel-0.16.0.jar - io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar * Prometheus exporter - - io.prometheus-prometheus-metrics-config-1.3.10.jar - - io.prometheus-prometheus-metrics-exporter-common-1.3.10.jar - - io.prometheus-prometheus-metrics-exporter-httpserver-1.3.10.jar - - io.prometheus-prometheus-metrics-exposition-formats-no-protobuf-1.3.10.jar - - io.prometheus-prometheus-metrics-model-1.3.10.jar - - io.prometheus-prometheus-metrics-exposition-textformats-1.3.10.jar + - io.prometheus-prometheus-metrics-config-1.5.1.jar + - io.prometheus-prometheus-metrics-exporter-common-1.5.1.jar + - io.prometheus-prometheus-metrics-exporter-httpserver-1.5.1.jar + - io.prometheus-prometheus-metrics-exposition-formats-no-protobuf-1.5.1.jar + - io.prometheus-prometheus-metrics-exposition-textformats-1.5.1.jar + - io.prometheus-prometheus-metrics-model-1.5.1.jar * Jakarta Bean Validation API - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar @@ -530,31 +530,30 @@ The Apache Software License, Version 2.0 - io.reactivex.rxjava3-rxjava-3.0.1.jar * RoaringBitmap - org.roaringbitmap-RoaringBitmap-1.6.9.jar - * OpenTelemetry - - io.opentelemetry-opentelemetry-api-1.56.0.jar - - io.opentelemetry-opentelemetry-api-incubator-1.56.0-alpha.jar - - io.opentelemetry-opentelemetry-common-1.56.0.jar - - io.opentelemetry-opentelemetry-context-1.56.0.jar - - io.opentelemetry-opentelemetry-exporter-common-1.56.0.jar - - io.opentelemetry-opentelemetry-exporter-otlp-1.56.0.jar - - io.opentelemetry-opentelemetry-exporter-otlp-common-1.56.0.jar - - io.opentelemetry-opentelemetry-exporter-prometheus-1.56.0-alpha.jar - - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-common-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-logs-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-metrics-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-trace-1.56.0.jar - - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-2.21.0.jar - - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-incubator-2.21.0-alpha.jar - - io.opentelemetry.instrumentation-opentelemetry-resources-2.21.0-alpha.jar - - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-2.21.0-alpha.jar - - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-2.21.0-alpha.jar - - io.opentelemetry.semconv-opentelemetry-semconv-1.37.0.jar - - com.google.cloud.opentelemetry-detector-resources-support-0.36.0.jar - - io.opentelemetry.contrib-opentelemetry-gcp-resources-1.48.0-alpha.jar +* OpenTelemetry + - io.opentelemetry-opentelemetry-api-1.62.0.jar + - io.opentelemetry-opentelemetry-api-incubator-1.62.0-alpha.jar + - io.opentelemetry-opentelemetry-common-1.62.0.jar + - io.opentelemetry-opentelemetry-context-1.62.0.jar + - io.opentelemetry-opentelemetry-exporter-common-1.62.0.jar + - io.opentelemetry-opentelemetry-exporter-otlp-1.62.0.jar + - io.opentelemetry-opentelemetry-exporter-otlp-common-1.62.0.jar + - io.opentelemetry-opentelemetry-exporter-prometheus-1.62.0-alpha.jar + - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-common-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-logs-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-metrics-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-trace-1.62.0.jar + - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-2.28.1.jar + - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-incubator-2.28.1-alpha.jar + - io.opentelemetry.instrumentation-opentelemetry-resources-2.28.1-alpha.jar + - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-2.28.1-alpha.jar + - io.opentelemetry.semconv-opentelemetry-semconv-1.41.1.jar + - com.google.cloud.opentelemetry-detector-resources-support-0.36.0.jar + - io.opentelemetry.contrib-opentelemetry-gcp-resources-1.48.0-alpha.jar * Spotify completable-futures - com.spotify-completable-futures-0.3.6.jar * JSpecify diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 94ce08f9ecae4..cd833d3733880 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -385,11 +385,11 @@ The Apache Software License, Version 2.0 - log4j-core-2.25.4.jar - log4j-slf4j2-impl-2.25.4.jar - log4j-web-2.25.4.jar - * OpenTelemetry - - opentelemetry-api-1.56.0.jar - - opentelemetry-api-incubator-1.56.0-alpha.jar - - opentelemetry-common-1.56.0.jar - - opentelemetry-context-1.56.0.jar +* OpenTelemetry + - opentelemetry-api-1.62.0.jar + - opentelemetry-api-incubator-1.62.0-alpha.jar + - opentelemetry-common-1.62.0.jar + - opentelemetry-context-1.62.0.jar * BookKeeper - bookkeeper-common-allocator-4.17.3.jar diff --git a/pom.xml b/pom.xml index f1d5fc11b1a08..b75fc8d4f1e3d 100644 --- a/pom.xml +++ b/pom.xml @@ -308,11 +308,11 @@ flexible messaging model and an intuitive client API. the core logic is switched to java implementation of zstd in org.apache.commons:commons-compress --> 1.5.7-3 2.0.6 - 1.56.0 + 1.62.0 ${opentelemetry.version}-alpha - 2.21.0 + 2.28.1 ${opentelemetry.instrumentation.version}-alpha - 1.37.0 + 1.41.1 4.7.7 1.8 0.3.6 diff --git a/pulsar-opentelemetry/pom.xml b/pulsar-opentelemetry/pom.xml index b6b5826d3f0c5..e15be8f3bb1a5 100644 --- a/pulsar-opentelemetry/pom.xml +++ b/pulsar-opentelemetry/pom.xml @@ -61,7 +61,7 @@ io.opentelemetry.instrumentation - opentelemetry-runtime-telemetry-java17 + opentelemetry-runtime-telemetry diff --git a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java index d143b743d3497..084b0ee3737a3 100644 --- a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java +++ b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java @@ -22,7 +22,10 @@ import com.google.common.annotations.VisibleForTesting; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; -import io.opentelemetry.instrumentation.runtimemetrics.java17.RuntimeMetrics; +import io.opentelemetry.instrumentation.runtimetelemetry.RuntimeTelemetry; +import io.opentelemetry.instrumentation.runtimetelemetry.RuntimeTelemetryBuilder; +import io.opentelemetry.instrumentation.runtimetelemetry.internal.Experimental; +import io.opentelemetry.instrumentation.runtimetelemetry.internal.Internal; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; @@ -44,11 +47,12 @@ public class OpenTelemetryService implements Closeable { public static final String OTEL_SDK_DISABLED_KEY = "otel.sdk.disabled"; + static final String OTEL_EXPORTER_PROMETHEUS_HOST_KEY = "otel.exporter.prometheus.host"; static final int MAX_CARDINALITY_LIMIT = 10000; private final AtomicReference openTelemetrySdkReference = new AtomicReference<>(); - private final AtomicReference runtimeMetricsReference = new AtomicReference<>(); + private final AtomicReference runtimeTelemetryReference = new AtomicReference<>(); /** * Instantiates the OpenTelemetry SDK. All attributes are overridden by system properties or environment @@ -76,7 +80,9 @@ public OpenTelemetryService(String clusterName, // Cardinality limit includes the overflow attribute set, so we need to add 1. "otel.java.metrics.cardinality.limit", Integer.toString(MAX_CARDINALITY_LIMIT + 1), // Reduce number of allocations by using reusable data mode. - "otel.java.exporter.memory_mode", MemoryMode.REUSABLE_DATA.name() + "otel.java.exporter.memory_mode", MemoryMode.REUSABLE_DATA.name(), + // Preserve the pre-1.62.0 Prometheus exporter default of binding to all interfaces. + OTEL_EXPORTER_PROMETHEUS_HOST_KEY, "0.0.0.0" )); sdkBuilder.addResourceCustomizer( @@ -119,12 +125,13 @@ public OpenTelemetryService(String clusterName, openTelemetrySdkReference.set(sdkBuilder.build().getOpenTelemetrySdk()); // For a list of exposed metrics, see https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ - runtimeMetricsReference.set(RuntimeMetrics.builder(openTelemetrySdkReference.get()) - // disable JFR based telemetry and use only JMX telemetry - .disableAllFeatures() - // enable experimental JMX telemetry in addition - .emitExperimentalTelemetry() - .build()); + RuntimeTelemetryBuilder runtimeTelemetryBuilder = + RuntimeTelemetry.builder(openTelemetrySdkReference.get()); + // Disable JFR-based telemetry and rely on JMX-based metrics only. + Internal.setDisableAllJfrFeatures(runtimeTelemetryBuilder, true); + // Emit experimental JMX-based runtime metrics in addition to the stable ones. + Experimental.setEmitExperimentalMetrics(runtimeTelemetryBuilder, true); + runtimeTelemetryReference.set(runtimeTelemetryBuilder.build()); } public OpenTelemetry getOpenTelemetry() { @@ -133,9 +140,9 @@ public OpenTelemetry getOpenTelemetry() { @Override public void close() { - RuntimeMetrics runtimeMetrics = runtimeMetricsReference.getAndSet(null); - if (runtimeMetrics != null) { - runtimeMetrics.close(); + RuntimeTelemetry runtimeTelemetry = runtimeTelemetryReference.getAndSet(null); + if (runtimeTelemetry != null) { + runtimeTelemetry.close(); } OpenTelemetrySdk openTelemetrySdk = openTelemetrySdkReference.getAndSet(null); if (openTelemetrySdk != null) { diff --git a/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java b/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java index e3dd29cd1b23e..059152eb0c2b7 100644 --- a/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java +++ b/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import lombok.Cleanup; import org.apache.commons.lang3.StringUtils; @@ -153,6 +154,22 @@ public void testMetricCardinalityIsSet() { }); } + @Test + public void testPrometheusExporterDefaultsToAllInterfacesHost() { + var capturedHost = new AtomicReference(); + @Cleanup + var ots = OpenTelemetryService.builder() + .builderCustomizer(getBuilderCustomizer(null, + Map.of(OpenTelemetryService.OTEL_SDK_DISABLED_KEY, "false")) + .andThen(builder -> builder.addPropertiesCustomizer(config -> { + capturedHost.set(config.getString(OpenTelemetryService.OTEL_EXPORTER_PROMETHEUS_HOST_KEY)); + return Map.of(); + }))) + .clusterName("openTelemetryServicePrometheusHostTestCluster") + .build(); + assertThat(capturedHost.get()).isEqualTo("0.0.0.0"); + } + @Test public void testLongCounter() { var longCounter = meter.counterBuilder("dummyLongCounter").build(); From 07f05c14ae845ab5665260026008e235dc315f62 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Tue, 14 Jul 2026 10:41:29 +0800 Subject: [PATCH 198/201] [fix][sec][branch-4.2] Upgrade Jackson version to 2.18.9 (#26186) --- .../server/src/assemble/LICENSE.bin.txt | 22 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 22 +++++++++---------- pom.xml | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index adbfc41bbda6b..0b3903892b135 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -249,17 +249,17 @@ The Apache Software License, Version 2.0 - info.picocli-picocli-shell-jline3-4.7.7.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.18.8.jar - - com.fasterxml.jackson.core-jackson-core-2.18.8.jar - - com.fasterxml.jackson.core-jackson-databind-2.18.8.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.18.8.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.18.8.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.18.8.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.18.8.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.18.8.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.18.8.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.8.jar - - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.8.jar + - com.fasterxml.jackson.core-jackson-annotations-2.18.9.jar + - com.fasterxml.jackson.core-jackson-core-2.18.9.jar + - com.fasterxml.jackson.core-jackson-databind-2.18.9.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.18.9.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.18.9.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.18.9.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.18.9.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.18.9.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.18.9.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.9.jar + - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.9.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-3.2.4.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Fastutil -- it.unimi.dsi-fastutil-8.5.16.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index cd833d3733880..eddd1b0fad58c 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -313,17 +313,17 @@ The Apache Software License, Version 2.0 - picocli-4.7.7.jar - picocli-shell-jline3-4.7.7.jar * Jackson - - jackson-annotations-2.18.8.jar - - jackson-core-2.18.8.jar - - jackson-databind-2.18.8.jar - - jackson-dataformat-yaml-2.18.8.jar - - jackson-jaxrs-base-2.18.8.jar - - jackson-jaxrs-json-provider-2.18.8.jar - - jackson-module-jaxb-annotations-2.18.8.jar - - jackson-module-jsonSchema-2.18.8.jar - - jackson-datatype-jdk8-2.18.8.jar - - jackson-datatype-jsr310-2.18.8.jar - - jackson-module-parameter-names-2.18.8.jar + - jackson-annotations-2.18.9.jar + - jackson-core-2.18.9.jar + - jackson-databind-2.18.9.jar + - jackson-dataformat-yaml-2.18.9.jar + - jackson-jaxrs-base-2.18.9.jar + - jackson-jaxrs-json-provider-2.18.9.jar + - jackson-module-jaxb-annotations-2.18.9.jar + - jackson-module-jsonSchema-2.18.9.jar + - jackson-datatype-jdk8-2.18.9.jar + - jackson-datatype-jsr310-2.18.9.jar + - jackson-module-parameter-names-2.18.9.jar * Caffeine -- caffeine-3.2.4.jar * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar * Gson diff --git a/pom.xml b/pom.xml index b75fc8d4f1e3d..de6463dd93a94 100644 --- a/pom.xml +++ b/pom.xml @@ -211,7 +211,7 @@ flexible messaging model and an intuitive client API. 2.0.11 2.0.6 2.0.1 - 2.18.8 + 2.18.9 8.5.16 4.0.5 0.10.2 From 7d790b0f31cfd75e701d5d2580175029276b1fda Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Wed, 15 Jul 2026 15:53:49 +0800 Subject: [PATCH 199/201] [fix][sec][branch-4.2] Upgrade Hadoop to 3.5.0 (#26194) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index de6463dd93a94..3e9aaadabc8c8 100644 --- a/pom.xml +++ b/pom.xml @@ -256,7 +256,7 @@ flexible messaging model and an intuitive client API. 9.4.0 0.13.0 0.28.0 - 3.4.3 + 3.5.0 3.6.2 ${hadoop3.version} 2.6.4-hadoop3 From dd719ac5d05cafca591feca1f9a1f3523fb4e45b Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Tue, 14 Jul 2026 17:51:47 +0800 Subject: [PATCH 200/201] [fix][broker] Prevent early replay of non-strict delayed messages (#26188) (cherry picked from commit 16476b503d11b1091a4bb3e1adebbee1abd8fe02) --- .../InMemoryDelayedDeliveryTracker.java | 10 ++++- .../delayed/InMemoryDeliveryTrackerTest.java | 39 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 4f05661a28b25..089069968340e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -157,6 +157,8 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { * in a loop until the deliverAt time is reached (see issue #25996). * * In non-strict mode the timestamp is rounded down, since delivering up to tickTimeMillis early is allowed. + * Availability checks account for the full bucket width so that the original deliverAt is still within that + * allowed window. */ private long roundTimestamp(long deliverAt) { if (isDeliverAtTimeStrict()) { @@ -167,6 +169,10 @@ private long roundTimestamp(long deliverAt) { return trimLowerBit(deliverAt, timestampPrecisionBitCnt); } + private long getBucketCutoffTime() { + return isDeliverAtTimeStrict() ? getCutoffTime() : getCutoffTime() - precisionMillis + 1; + } + /** * Check that new delivery time comes after the current highest, or at * least within a single tick time interval of 1 second. @@ -185,7 +191,7 @@ private void checkAndUpdateHighest(long deliverAt) { @Override public boolean hasMessageAvailable() { boolean hasMessageAvailable = !delayedMessageMap.isEmpty() - && delayedMessageMap.firstLongKey() <= getCutoffTime(); + && delayedMessageMap.firstLongKey() <= getBucketCutoffTime(); if (!hasMessageAvailable) { updateTimer(); } @@ -199,7 +205,7 @@ public boolean hasMessageAvailable() { public NavigableSet getScheduledMessages(int maxMessages) { int n = maxMessages; NavigableSet positions = new TreeSet<>(); - long cutoffTime = getCutoffTime(); + long cutoffTime = getBucketCutoffTime(); while (n > 0 && !delayedMessageMap.isEmpty()) { long timestamp = delayedMessageMap.firstLongKey(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index 1c16ed4228f33..b3d026137ed53 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -111,6 +111,10 @@ public Object[][] provider(Method method) throws Exception { new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1, clock, true, 0) }}; + case "testNonStrictModeDoesNotReplayPositionRepeatedly" -> new Object[][]{{ + new InMemoryDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + false, 0) + }}; case "testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict" -> new Object[][]{{ new InMemoryDelayedDeliveryTracker(dispatcher, timer, 500, clock, true, 0) @@ -372,6 +376,41 @@ public void testStrictModeNeverDeliversEarlyAndKeepsTimerArmed(InMemoryDelayedDe tracker.close(); } + @Test(dataProvider = "delayedTracker") + public void testNonStrictModeDoesNotReplayPositionRepeatedly(InMemoryDelayedDeliveryTracker tracker) + throws Exception { + clockTime.set(0); + + // tickTimeMillis=1000 uses 512ms buckets. deliverAt=60400 is stored in the bucket starting at 59904. + assertTrue(tracker.addMessage(1, 1, 60400)); + + // The exact deliverAt is still more than one tick away. Before the fix, each dispatch round popped the + // same position from the prematurely eligible bucket, read the entry, and re-added the position because + // the original deliverAt was still outside the non-strict early-delivery window. + clockTime.set(59000); + int replayReads = 0; + for (int i = 0; i < 10; i++) { + Set scheduled = tracker.getScheduledMessages(100); + replayReads += scheduled.size(); + for (Position position : scheduled) { + assertEquals(position, PositionFactory.create(1, 1)); + assertTrue(tracker.addMessage(position.getLedgerId(), position.getEntryId(), 60400)); + } + } + assertEquals(replayReads, 0, "the position must not be replayed before its early-delivery window"); + + // Once the full bucket is inside the non-strict early-delivery window, the position is read exactly once + // and cannot be inserted back into the tracker with the original deliverAt. + clockTime.set(59415); + Set scheduled = tracker.getScheduledMessages(100); + replayReads += scheduled.size(); + assertEquals(scheduled, Set.of(PositionFactory.create(1, 1))); + assertFalse(tracker.addMessage(1, 1, 60400)); + assertEquals(replayReads, 1); + + tracker.close(); + } + /** * A timeout that was superseded by a newer one may still fire: HashedWheelTimer can run a task that passed * its isCancelled() check just before updateTimer() cancelled it. Such a stale trigger must not clear the From 64fc12bbb6f46eca553ae4e4ef6f4f59bd6a4f8c Mon Sep 17 00:00:00 2001 From: lushiji Date: Tue, 21 Jul 2026 13:16:07 +0800 Subject: [PATCH 201/201] [improve][broker] Upgrade bookkeeper to 4.17.4 --- .../server/src/assemble/LICENSE.bin.txt | 52 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 6 +-- pom.xml | 11 +++- 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 0b3903892b135..99838fdfb27c0 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -357,32 +357,32 @@ The Apache Software License, Version 2.0 - net.java.dev.jna-jna-jpms-5.18.1.jar - net.java.dev.jna-jna-platform-jpms-5.18.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.17.3.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.17.3.jar - - org.apache.bookkeeper-bookkeeper-proto-4.17.3.jar - - org.apache.bookkeeper-bookkeeper-server-4.17.3.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.17.3.jar - - org.apache.bookkeeper-circe-checksum-4.17.3.jar - - org.apache.bookkeeper-cpu-affinity-4.17.3.jar - - org.apache.bookkeeper-statelib-4.17.3.jar - - org.apache.bookkeeper-stream-storage-api-4.17.3.jar - - org.apache.bookkeeper-stream-storage-common-4.17.3.jar - - org.apache.bookkeeper-stream-storage-java-client-4.17.3.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.17.3.jar - - org.apache.bookkeeper-stream-storage-proto-4.17.3.jar - - org.apache.bookkeeper-stream-storage-server-4.17.3.jar - - org.apache.bookkeeper-stream-storage-service-api-4.17.3.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.17.3.jar - - org.apache.bookkeeper.http-http-server-4.17.3.jar - - org.apache.bookkeeper.http-vertx-http-server-4.17.3.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.17.3.jar - - org.apache.distributedlog-distributedlog-common-4.17.3.jar - - org.apache.distributedlog-distributedlog-core-4.17.3-tests.jar - - org.apache.distributedlog-distributedlog-core-4.17.3.jar - - org.apache.distributedlog-distributedlog-protocol-4.17.3.jar - - org.apache.bookkeeper-bookkeeper-slogger-api-4.17.3.jar - - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.17.3.jar - - org.apache.bookkeeper-native-io-4.17.3.jar + - org.apache.bookkeeper-bookkeeper-common-4.17.4.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.17.4.jar + - org.apache.bookkeeper-bookkeeper-proto-4.17.4.jar + - org.apache.bookkeeper-bookkeeper-server-4.17.4.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.17.4.jar + - org.apache.bookkeeper-circe-checksum-4.17.4.jar + - org.apache.bookkeeper-cpu-affinity-4.17.4.jar + - org.apache.bookkeeper-statelib-4.17.4.jar + - org.apache.bookkeeper-stream-storage-api-4.17.4.jar + - org.apache.bookkeeper-stream-storage-common-4.17.4.jar + - org.apache.bookkeeper-stream-storage-java-client-4.17.4.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.17.4.jar + - org.apache.bookkeeper-stream-storage-proto-4.17.4.jar + - org.apache.bookkeeper-stream-storage-server-4.17.4.jar + - org.apache.bookkeeper-stream-storage-service-api-4.17.4.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.17.4.jar + - org.apache.bookkeeper.http-http-server-4.17.4.jar + - org.apache.bookkeeper.http-vertx-http-server-4.17.4.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.17.4.jar + - org.apache.distributedlog-distributedlog-common-4.17.4.jar + - org.apache.distributedlog-distributedlog-core-4.17.4-tests.jar + - org.apache.distributedlog-distributedlog-core-4.17.4.jar + - org.apache.distributedlog-distributedlog-protocol-4.17.4.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.17.4.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.17.4.jar + - org.apache.bookkeeper-native-io-4.17.4.jar - at.yawk.lz4-lz4-java-1.10.3.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.14.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index eddd1b0fad58c..d53e280b66290 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -392,9 +392,9 @@ The Apache Software License, Version 2.0 - opentelemetry-context-1.62.0.jar * BookKeeper - - bookkeeper-common-allocator-4.17.3.jar - - cpu-affinity-4.17.3.jar - - circe-checksum-4.17.3.jar + - bookkeeper-common-allocator-4.17.4.jar + - cpu-affinity-4.17.4.jar + - circe-checksum-4.17.4.jar * AirCompressor - aircompressor-2.0.3.jar * AsyncHttpClient diff --git a/pom.xml b/pom.xml index 3e9aaadabc8c8..15a606dc4fa6e 100644 --- a/pom.xml +++ b/pom.xml @@ -182,7 +182,7 @@ flexible messaging model and an intuitive client API. 1.28.0 - 4.17.3 + 4.17.4 3.9.5 1.11.0 1.15.0 @@ -3168,6 +3168,15 @@ flexible messaging model and an intuitive client API. + + + bk-staging + bk-staging + https://repository.apache.org/content/repositories/orgapachebookkeeper-1107/ + + false + + central default