Stabilize integration log verifiers

Author: jiangpengcheng

.ci/tests/integration/cases/crypto-function/verify.sh

@@ -56,29 +56,46 @@ function upload_string_schema() {
     sh "${input_topic}" > /dev/null
 }
 
+function wait_string_schema() {
+  for attempt in $(seq 1 30); do
+    if kubectl exec -n "${PULSAR_NAMESPACE}" "${PULSAR_RELEASE_NAME}"-pulsar-broker-0 -- \
+      bin/pulsar-admin schemas get "${input_topic}" > /dev/null 2>&1; then
+      return 0
+    fi
+    echo "Schema for ${input_topic} is not ready, retry ${attempt}/30" >&2
+    sleep 2
+  done
+  return 1
+}
+
 trap cleanup EXIT
 cleanup
 
 if ! create_topic_result=$(NAMESPACE=${PULSAR_NAMESPACE} CLUSTER=${PULSAR_RELEASE_NAME} ci::create_topic "${input_topic}" 2>&1); then
-  echo "$create_topic_result"
+  echo "$create_topic_result" >&2
   exit 1
 fi
 
 if ! upload_schema_result=$(upload_string_schema 2>&1); then
-  echo "$upload_schema_result"
+  echo "$upload_schema_result" >&2
+  exit 1
+fi
+
+if ! wait_schema_result=$(wait_string_schema 2>&1); then
+  echo "$wait_schema_result" >&2
   exit 1
 fi
 
 kubectl apply -f "${manifests_file}" > /dev/null 2>&1
 
 if ! verify_fm_result=$(ci::verify_function_mesh java-function-crypto-sample 2>&1); then
-  echo "$verify_fm_result"
+  echo "$verify_fm_result" >&2
   exit 1
 fi
 
 if verify_crypto_result=$(NAMESPACE=${PULSAR_NAMESPACE} CLUSTER=${PULSAR_RELEASE_NAME} ci::verify_crypto_function 2>&1); then
   echo "e2e-test: ok" | yq eval -
 else
-  echo "$verify_crypto_result"
+  echo "$verify_crypto_result" >&2
   exit 1
 fi

.ci/tests/integration/cases/logging-window-function/verify.sh

@@ -36,6 +36,8 @@ manifests_file="${BASE_DIR}"/.ci/tests/integration/cases/logging-window-function
 input_topic="persistent://public/default/window-function-input-topic"
 output_topic="persistent://public/default/window-function-output-topic"
 log_topic="persistent://public/default/window-function-logs"
+expected_window_log_lines=15
+expected_log_topic_messages=14
 
 function delete_topic_if_exists() {
   topic=$1
@@ -115,32 +117,32 @@ fi
 verify_log_result=0
 for attempt in $(seq 1 30); do
   verify_log_result=$(kubectl logs -l compute.functionmesh.io/name=window-function-sample --tail=-1 | grep -e "-window-log" | wc -l)
-  if [ "$verify_log_result" -eq 15 ]; then
+  if [ "$verify_log_result" -eq "${expected_window_log_lines}" ]; then
     break
   fi
   sleep 2
 done
 
-if [ "$verify_log_result" -eq 15 ]; then
+if [ "$verify_log_result" -eq "${expected_window_log_lines}" ]; then
   verify_log_topic_result=0
   for attempt in $(seq 1 20); do
     sub_name=$(echo "${RANDOM}-${attempt}" | md5sum | head -c 20; echo;)
-    verify_log_topic_result=$(timeout 8s kubectl exec -n "${PULSAR_NAMESPACE}" "${PULSAR_RELEASE_NAME}"-pulsar-broker-0 -- bin/pulsar-client consume -n 15 -s "${sub_name}" --subscription-position Earliest "${log_topic}" 2>/dev/null | grep  -e "-window-log" | wc -l)
-    if [ "$verify_log_topic_result" -eq 15 ]; then
+    verify_log_topic_result=$(timeout 8s kubectl exec -n "${PULSAR_NAMESPACE}" "${PULSAR_RELEASE_NAME}"-pulsar-broker-0 -- bin/pulsar-client consume -n "${expected_log_topic_messages}" -s "${sub_name}" --subscription-position Earliest "${log_topic}" 2>/dev/null | grep  -e "-window-log" | wc -l)
+    if [ "$verify_log_topic_result" -ge "${expected_log_topic_messages}" ]; then
       break
     fi
     sleep 2
   done
 
-  if [ "$verify_log_topic_result" -eq 15 ]; then
+  if [ "$verify_log_topic_result" -ge "${expected_log_topic_messages}" ]; then
     echo "e2e-test: ok" | yq eval -
   else
-    echo "expected 15 window log topic messages, got ${verify_log_topic_result}" >&2
+    echo "expected at least ${expected_log_topic_messages} window log topic messages, got ${verify_log_topic_result}" >&2
     kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
     exit 1
   fi
 else
-  echo "expected 15 window log lines, got ${verify_log_result}" >&2
+  echo "expected ${expected_window_log_lines} window log lines, got ${verify_log_result}" >&2
   kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
   exit 1
 fi