diff --git a/tests/README.md b/tests/README.md index 1688d6a7a..420fb75f6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -36,6 +36,87 @@ Here we test Kruize [Local monitoring APIs](/design/KruizeLocalAPI.md). For details refer this [doc](/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md) +### Authentication Test: + +Kruize 0.2 onwards supports the authentication which provides the user an option to pass authentication details in the YAML for the service they are using. + +The authentication test is a standalone shell-based test. It contains various valid and invalid scenarios for testing. + +It can be run directly from the shell test location, for example: + +`tests/config_tests/authentication_tests.sh` + +#### Scenarios +**_valid_**: a valid path to the token + +**_expired_**: an expired token value + +**_invalid_**: an invalid path to the token + +**_empty_**: a blank input in place of the token file path + +### Datasource Availability/Serviceability Test: + +Kruize supports multiple datasources such as Prometheus and Thanos Querier. During startup, Kruize validates the reachability of all configured datasources before proceeding. + +The datasource availability/serviceability test is a standalone shell-based test and is not part of the pytest-based functional/local monitoring suite. +It validates Kruize behavior when one or more datasources are reachable or unreachable. + +Kruize startup behavior follows these rules: + +* Kruize continues startup if at least one datasource is reachable. +* Kruize logs an error for each unreachable datasource. +* Kruize fails startup only when all configured datasources are unreachable. + +The test can be run using the command below: + +``` +./test_autotune.sh -c -i --testsuite=datasource_tests +``` + +#### Scenarios + +Both cluster types support multiple datasources. Scenarios vary by which datasources are in the cluster YAML: +- **OpenShift:** YAML has Prometheus + Thanos (both available in cluster) +- **Minikube/Kind:** YAML has Prometheus only (Thanos not running; multiple Prometheus datasources are also valid) + +**both-valid** (OpenShift) + +Both datasources are reachable. + +**✔ Expected:** Kruize starts successfully. + +**valid-invalid** (OpenShift) + +Datasource 1 is reachable, datasource 2 is unreachable. + +**✔ Expected:** Kruize starts successfully and logs an error for datasource 2. + +**invalid-valid** (OpenShift) + +Datasource 1 is unreachable, datasource 2 is reachable. + +**✔ Expected:** Kruize starts successfully and logs an error for datasource 1. + +**both-invalid** (OpenShift) + +Both datasources are unreachable. + +**❌ Expected:** Kruize fails to start and exits with an error. + +**valid** (Minikube/Kind) + +Datasource is reachable. + +**✔ Expected:** Kruize starts successfully. + +**invalid** (Minikube/Kind) + +Datasource is unreachable. + +**❌ Expected:** Kruize fails to start and exits with an error. + + ## Supported Clusters - Minikube, Openshift diff --git a/tests/scripts/local_monitoring_tests/authentication_tests.sh b/tests/config_tests/authentication_tests.sh similarity index 86% rename from tests/scripts/local_monitoring_tests/authentication_tests.sh rename to tests/config_tests/authentication_tests.sh index c593ca287..df872aefd 100644 --- a/tests/scripts/local_monitoring_tests/authentication_tests.sh +++ b/tests/config_tests/authentication_tests.sh @@ -17,10 +17,10 @@ # # Get the path of the test dir -LOCAL_MONITORING_TEST_DIR="${KRUIZE_REPO}/tests/scripts/local_monitoring_tests" +CONFIG_TEST_DIR="${KRUIZE_REPO}/tests/config_tests" # Source the common functions scripts -. ${LOCAL_MONITORING_TEST_DIR}/../common/common_functions.sh +. ${KRUIZE_REPO}/tests/scripts/common/common_functions.sh APP_DEPLOYMENT="kruize" @@ -33,7 +33,15 @@ tokens=( ["empty"]="" ) # Tests to validate authentication types in Kruize +# Note: Skipped on minikube/kind - the minikube YAML has no datasource authentication block, +# and Prometheus on minikube typically does not require bearer auth. These tests only apply +# to OpenShift where Prometheus enforces OAuth/bearer token authentication. function authentication_tests() { + if [ "$cluster_type" == "minikube" ] || [ "$cluster_type" == "kind" ]; then + echo "Skipping authentication_tests: datasource auth is not configured on ${cluster_type} (Prometheus does not require bearer token)." + return 0 + fi + start_time=$(get_date) FAILED_CASES=() TESTS=0 @@ -56,10 +64,10 @@ function authentication_tests() { if [ "$cluster_type" == "minikube" ] || [ "$cluster_type" == "kind" ]; then NAMESPACE="monitoring" - YAML_FILE="${LOCAL_MONITORING_TEST_DIR}/../../../manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml" + YAML_FILE="${KRUIZE_REPO}/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml" elif [ "$cluster_type" == "openshift" ]; then NAMESPACE="openshift-tuning" - YAML_FILE="${LOCAL_MONITORING_TEST_DIR}/../../../manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml" + YAML_FILE="${KRUIZE_REPO}/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml" else echo "Invalid cluster type found: ${cluster_type}" fi @@ -130,7 +138,7 @@ deploy_and_check_pod() { $kubectl_cmd wait --for=condition=Ready pod -l app=$APP_DEPLOYMENT --timeout=120s > /dev/null # Check pod logs for errors echo "Checking logs for the pod..." - POD_NAME=$($kubectl_cmd get pods | grep 'kruize' | grep -v -E 'kruize-db|kruize-ui' | awk 'NR==1{print $1}') + POD_NAME=$($kubectl_cmd get pods | grep 'kruize' | grep -v -E 'kruize-db|kruize-ui' | awk 'NR==1{print $1}') echo "$kubectl_cmd logs -f ${POD_NAME} > ${POD_LOG} 2>&1 &" $kubectl_cmd logs -f "${POD_NAME}" > "${POD_LOG}" 2>&1 & sleep 10 diff --git a/tests/scripts/local_monitoring_tests/datasource_tests.sh b/tests/config_tests/datasource_tests.sh similarity index 66% rename from tests/scripts/local_monitoring_tests/datasource_tests.sh rename to tests/config_tests/datasource_tests.sh index e80d3e532..4d0d86f76 100644 --- a/tests/scripts/local_monitoring_tests/datasource_tests.sh +++ b/tests/config_tests/datasource_tests.sh @@ -17,28 +17,40 @@ # # Get the absolute path of current directory -LOCAL_MONITORING_TEST_DIR="${KRUIZE_REPO}/tests/scripts/local_monitoring_tests" +CONFIG_TEST_DIR="${KRUIZE_REPO}/tests/config_tests" # Source the common functions scripts -. ${LOCAL_MONITORING_TEST_DIR}/../common/common_functions.sh +. ${KRUIZE_REPO}/tests/scripts/common/common_functions.sh APP_DEPLOYMENT="kruize" # Datasource serviceName overrides to simulate reachability declare -A datasource_scenarios datasource_scenarios=( + # OpenShift YAML: Prometheus + Thanos (both available in cluster) ["both-invalid"]="invalid invalid" ["both-valid"]="prometheus-k8s thanos-querier" - ["prom-valid-thanos-invalid"]="prometheus-k8s invalid-thanos" - ["prom-invalid-thanos-valid"]="invalid-prometheus thanos-querier" + ["valid-invalid"]="prometheus-k8s invalid-thanos" + ["invalid-valid"]="invalid-prometheus thanos-querier" + # Minikube/Kind YAML: Prometheus only (Thanos not running; could add more Prometheus DS) + ["invalid"]="invalid invalid" + ["valid"]="prometheus-k8s invalid" ) -datasource_scenario_order=( + +# OpenShift: test with Prometheus + Thanos (both in cluster YAML) +openshift_scenario_order=( "both-invalid" - "prom-invalid-thanos-valid" - "prom-valid-thanos-invalid" + "invalid-valid" + "valid-invalid" "both-valid" ) +# Minikube/Kind: test with Prometheus only (Thanos not in cluster; YAML has one DS) +non_openshift_scenario_order=( + "invalid" + "valid" +) + function datasource_tests() { start_time=$(get_date) FAILED_CASES=() @@ -55,10 +67,10 @@ function datasource_tests() { echo "" if [ "$cluster_type" == "minikube" ] || [ "$cluster_type" == "kind" ]; then NAMESPACE="monitoring" - YAML_FILE="${LOCAL_MONITORING_TEST_DIR}/../../../manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml" + YAML_FILE="${KRUIZE_REPO}/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml" elif [ "$cluster_type" == "openshift" ]; then NAMESPACE="openshift-tuning" - YAML_FILE="${LOCAL_MONITORING_TEST_DIR}/../../../manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml" + YAML_FILE="${KRUIZE_REPO}/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml" else echo "Invalid cluster type found: ${cluster_type}" return @@ -76,12 +88,24 @@ function datasource_tests() { kubectl_cmd="kubectl -n ${NAMESPACE}" + # Select scenarios based on cluster type: + # - OpenShift: Prometheus + Thanos in YAML, test both + # - Minikube/Kind: Prometheus only in YAML (Thanos not running) + # Both cluster types support multiple datasources + if [ "$cluster_type" == "openshift" ]; then + scenario_order=("${openshift_scenario_order[@]}") + echo "Cluster type: OpenShift - Testing with Prometheus and Thanos datasources" + else + scenario_order=("${non_openshift_scenario_order[@]}") + echo "Cluster type: ${cluster_type} - Testing with Prometheus datasource (Thanos not available)" + fi + echo "" echo "******************* Executing test suite ${FUNCNAME} ****************" echo "" - suffix=1 - for scenario in "${datasource_scenario_order[@]}"; do + suffix=1 + for scenario in "${scenario_order[@]}"; do echo "" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo " Running datasource scenario: ${scenario}" @@ -121,13 +145,15 @@ run_datasource_scenario() { suffix=$2 POD_LOG="${TEST_SUITE_DIR}/${scenario}-pod.log" - read PROM_SERVICE THANOS_SERVICE <<< "${datasource_scenarios[$scenario]}" + read DS1_SERVICE DS2_SERVICE <<< "${datasource_scenarios[$scenario]}" echo "Updating YAML:" - echo " Prometheus serviceName = ${PROM_SERVICE}" - echo " Thanos serviceName = ${THANOS_SERVICE}" + echo " Datasource 1 serviceName = ${DS1_SERVICE}" + if [ "$cluster_type" == "openshift" ]; then + echo " Datasource 2 serviceName = ${DS2_SERVICE}" + fi - update_yaml_with_datasources "${PROM_SERVICE}" "${THANOS_SERVICE}" "${suffix}" + update_yaml_with_datasources "${DS1_SERVICE}" "${DS2_SERVICE}" "${suffix}" $kubectl_cmd apply -f "$YAML_FILE" > /dev/null $kubectl_cmd rollout restart deployment kruize @@ -139,10 +165,10 @@ run_datasource_scenario() { sleep 5 $kubectl_cmd logs "$POD_NAME" > "$POD_LOG" 2>&1 - if [[ "$scenario" == "both-invalid" ]]; then - echo "inside both-invalid" + if [[ "$scenario" == "both-invalid" ]] || [[ "$scenario" == "invalid" ]]; then + echo "inside ${scenario} (expecting datasource failure)" if grep -i "No datasource could be added or are serviceable" "$POD_LOG"; then - echo "Expected failure detected (both datasources invalid)" + echo "Expected failure detected (datasource(s) invalid)" ((TESTS_PASSED++)) else echo "Expected failure NOT detected" @@ -178,28 +204,28 @@ cleanup_datasources_from_yaml() { } update_yaml_with_datasources() { - local prom_service=$1 - local thanos_service=$2 - # create unique datasource name for each scenario + local ds1_service=$1 + local ds2_service=$2 + # create unique datasource name for each scenario (YAML keys: prometheus-1, thanos-1) SUFFIX=$3 - PROM_DS_NAME="prometheus-${SUFFIX}" - THANOS_DS_NAME="thanos-${SUFFIX}" + DS1_NAME="prometheus-${SUFFIX}" + DS2_NAME="thanos-${SUFFIX}" echo "Using datasource names:" - echo " Prometheus: ${PROM_DS_NAME}" - echo " Thanos: ${THANOS_DS_NAME}" + echo " Datasource 1: ${DS1_NAME}" + echo " Datasource 2: ${DS2_NAME}" # Backup once cp "$YAML_FILE" "${YAML_FILE}.ds.bak" sed -i ' /"name": *"prometheus-1"/,/}/{ - s/"name": *"[^"]*"/"name": "'"$PROM_DS_NAME"'"/ - s/"serviceName": *"[^"]*"/"serviceName": "'"$prom_service"'"/ + s/"name": *"[^"]*"/"name": "'"$DS1_NAME"'"/ + s/"serviceName": *"[^"]*"/"serviceName": "'"$ds1_service"'"/ } /"name": *"thanos-1"/,/}/{ - s/"name": *"[^"]*"/"name": "'"$THANOS_DS_NAME"'"/ - s/"serviceName": *"[^"]*"/"serviceName": "'"$thanos_service"'"/ + s/"name": *"[^"]*"/"name": "'"$DS2_NAME"'"/ + s/"serviceName": *"[^"]*"/"serviceName": "'"$ds2_service"'"/ } ' "$YAML_FILE" diff --git a/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md b/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md index ea34c0351..8d1294317 100644 --- a/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md +++ b/tests/scripts/local_monitoring_tests/Local_monitoring_tests.md @@ -358,66 +358,3 @@ Else, you can change the workload name and namespace name in the test to match w Note: The test will fail if it's run as is if there are no matching workloads that the test looks for. This test result can be ignored in case of a non-gpu workload -### Authentication Test: - -Kruize 0.2 supports the authentication which provides the user an option to pass authentication details in the yaml for the service they are using. - -The authentication test is part of functional bucket and has a separate script similar to local_monitoring tests. It contains various valid and invalid scenarios for testing. - -It can be run as shown in the example below: - -`/test_autotune.sh -c -i -r benchmarks/ --testsuite=authentication_tests` - -#### Scenarios -**_valid_**: a valid path to the token - -**_expired_**: an expired token value - -**_invalid_**: an invalid path to the token - -**_empty_**: a blank input in place of the token file path - -### Datasource Availability/Serviceability Test: - -Kruize supports multiple datasources such as Prometheus and Thanos Querier. During startup, Kruize validates the reachability of all configured datasources before proceeding. - -The datasource availability/serviceability test is part of the functional test bucket and is implemented as a standalone shell script, similar to the authentication tests. -It validates Kruize behavior when one or more datasources are reachable or unreachable. - -Kruize startup behavior follows these rules: - -* Kruize continues startup if at least one datasource is reachable. -* Kruize logs an error for each unreachable datasource. -* Kruize fails startup only when all configured datasources are unreachable. - -The test can be run using the command below: - -``` -./test_autotune.sh -c -i -r benchmarks/ --testsuite=datasource_tests -``` - -#### Scenarios - -**both-valid** - -Both Prometheus and Thanos Querier datasources are reachable. - -**✔ Expected:** Kruize starts successfully. - -**prom-valid-thanos-invalid** - -Prometheus is reachable and Thanos Querier is unreachable. - -**✔ Expected:** Kruize starts successfully and logs an error for Thanos. - -**prom-invalid-thanos-valid** - -Prometheus is unreachable and Thanos Querier is reachable. - -**✔ Expected:** Kruize starts successfully and logs an error for Prometheus. - -**both-invalid** - -Both Prometheus and Thanos Querier datasources are unreachable. - -**❌ Expected:** Kruize fails to start and exits with an error. diff --git a/tests/test_autotune.sh b/tests/test_autotune.sh index 57053cf76..ce94fa0b8 100755 --- a/tests/test_autotune.sh +++ b/tests/test_autotune.sh @@ -25,8 +25,8 @@ SCRIPTS_DIR="${CURRENT_DIR}/scripts" # Source the test suite scripts . ${SCRIPTS_DIR}/remote_monitoring_tests/remote_monitoring_tests.sh . ${SCRIPTS_DIR}/local_monitoring_tests/local_monitoring_tests.sh -. ${SCRIPTS_DIR}/local_monitoring_tests/authentication_tests.sh -. ${SCRIPTS_DIR}/local_monitoring_tests/datasource_tests.sh +. ${CURRENT_DIR}/config_tests/authentication_tests.sh +. ${CURRENT_DIR}/config_tests/datasource_tests.sh resultsdir="${CURRENT_DIR}"