From 66a43ef77b0e3cf501fd67863d5259aa30b07e05 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Mon, 20 Jul 2026 13:17:22 +0800 Subject: [PATCH] fix(ci): wait for the kafka broker before creating topics Topic creation ran as soon as the containers were up, so when the broker had not registered in ZooKeeper yet the command failed with "Replication factor: 1 larger than available brokers: 0". The error was ignored, and because KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE is on, the topic was then auto-created with the default single partition on first produce. That is what makes t/plugin/kafka-logger.t TEST 18 flaky: it expects test3 to have 3 partitions and asserts partition ids 0, 1 and 2 are all logged, but every message goes to partition 0 when the topic silently degrades to one partition. Retry topic creation until the broker accepts it, and fail the setup instead of starting a test run against a wrong partition layout. --- ci/init-last-test-service.sh | 31 +++++++++++++++++++++++++++---- ci/init-plugin-test-service.sh | 33 ++++++++++++++++++++++++++++----- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/ci/init-last-test-service.sh b/ci/init-last-test-service.sh index 694349056f55..bc90a25a37c8 100755 --- a/ci/init-last-test-service.sh +++ b/ci/init-last-test-service.sh @@ -16,16 +16,39 @@ # limitations under the License. # +# A broker that is not registered in ZooKeeper yet rejects topic creation with +# "Replication factor: 1 larger than available brokers: 0". The failure used to be +# silent, and the topic was then auto-created with the default single partition on +# first produce, which quietly broke tests that expect a specific partition layout. +create_kafka_topic() { + local container="$1" + local zookeeper="$2" + local partitions="$3" + local topic="$4" + + for _ in $(seq 30); do + if docker exec -i "$container" /opt/bitnami/kafka/bin/kafka-topics.sh --create \ + --zookeeper "$zookeeper" --replication-factor 1 \ + --partitions "$partitions" --topic "$topic"; then + return 0 + fi + sleep 2 + done + + echo "failed to create kafka topic $topic on $container" + exit 1 +} + before() { # generating SSL certificates for Kafka sudo keytool -genkeypair -keyalg RSA -dname "CN=127.0.0.1" -alias 127.0.0.1 -keystore ./ci/pod/kafka/kafka-server/selfsigned.jks -validity 365 -keysize 2048 -storepass changeit } after() { - docker exec -i apache-apisix-kafka-server1-1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 1 --topic test2 - docker exec -i apache-apisix-kafka-server1-1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 3 --topic test3 - docker exec -i apache-apisix-kafka-server2-1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server2:2181 --replication-factor 1 --partitions 1 --topic test4 - docker exec -i apache-apisix-kafka-server1-1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 1 --topic test-consumer + create_kafka_topic apache-apisix-kafka-server1-1 zookeeper-server1:2181 1 test2 + create_kafka_topic apache-apisix-kafka-server1-1 zookeeper-server1:2181 3 test3 + create_kafka_topic apache-apisix-kafka-server2-1 zookeeper-server2:2181 1 test4 + create_kafka_topic apache-apisix-kafka-server1-1 zookeeper-server1:2181 1 test-consumer # create messages for test-consumer for i in `seq 30` do diff --git a/ci/init-plugin-test-service.sh b/ci/init-plugin-test-service.sh index 6ef72f1df65f..6cfb565df163 100755 --- a/ci/init-plugin-test-service.sh +++ b/ci/init-plugin-test-service.sh @@ -16,12 +16,35 @@ # limitations under the License. # +# A broker that is not registered in ZooKeeper yet rejects topic creation with +# "Replication factor: 1 larger than available brokers: 0". The failure used to be +# silent, and the topic was then auto-created with the default single partition on +# first produce, which quietly broke tests that expect a specific partition layout. +create_kafka_topic() { + local container="$1" + local zookeeper="$2" + local partitions="$3" + local topic="$4" + + for _ in $(seq 30); do + if docker exec -i "$container" /opt/bitnami/kafka/bin/kafka-topics.sh --create \ + --zookeeper "$zookeeper" --replication-factor 1 \ + --partitions "$partitions" --topic "$topic"; then + return 0 + fi + sleep 2 + done + + echo "failed to create kafka topic $topic on $container" + exit 1 +} + after() { - docker exec -i apache-apisix-kafka-server1-1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 1 --topic test2 - docker exec -i apache-apisix-kafka-server1-1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 3 --topic test3 - docker exec -i apache-apisix-kafka-server2-1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server2:2181 --replication-factor 1 --partitions 1 --topic test4 - docker exec -i apache-apisix-kafka-server3-scram-1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server3:2181 --replication-factor 1 --partitions 1 --topic test-scram-256 - docker exec -i apache-apisix-kafka-server3-scram-1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server3:2181 --replication-factor 1 --partitions 1 --topic test-scram-512 + create_kafka_topic apache-apisix-kafka-server1-1 zookeeper-server1:2181 1 test2 + create_kafka_topic apache-apisix-kafka-server1-1 zookeeper-server1:2181 3 test3 + create_kafka_topic apache-apisix-kafka-server2-1 zookeeper-server2:2181 1 test4 + create_kafka_topic apache-apisix-kafka-server3-scram-1 zookeeper-server3:2181 1 test-scram-256 + create_kafka_topic apache-apisix-kafka-server3-scram-1 zookeeper-server3:2181 1 test-scram-512 # Create user with SCRAM-SHA-512 docker exec apache-apisix-kafka-server3-scram-1 /opt/bitnami/kafka/bin/kafka-configs.sh \ --zookeeper zookeeper-server3:2181 \