From 10693e96f8f3f1d94388f0edfe72da4e3875522e Mon Sep 17 00:00:00 2001 From: Faisal Abujabal Date: Mon, 11 May 2026 15:09:02 -0500 Subject: [PATCH 1/2] fix: E2E Tests Varios Fixes --- .cursor/rules/e2e-testing.mdc | 2 +- .github/workflows/ci.yml | 17 ++- .gitignore | 1 + Dockerfile.e2e | 16 +- Makefile | 4 +- hack/{ => e2e}/install-pykmip.sh | 0 hack/e2e/install-squid.sh | 65 +++++++++ hack/{ => e2e}/kms.sh | 0 hack/e2e/proxy.sh | 137 ++++++++++++++++++ hack/e2e/setup-e2e-testbed.sh | 106 ++++++++++++++ hack/{ => e2e}/setup-testbed-env.sh | 23 ++- hack/e2e/squid.conf | 22 +++ hack/{ => e2e}/wait-for-artifact-dir.sh | 0 test/e2e/README.md | 2 +- test/e2e/vmservice/vmservice/util.go | 12 +- .../vmservice/vmservice/viadmin/registervm.go | 15 +- .../virtualmachine/virtualmachinelcm.go | 26 ++-- .../vmservice/virtualmachine/vm_group.go | 65 +++++---- .../virtualmachine/vm_group_publishrequest.go | 4 +- .../vmservice/virtualmachine/vm_hardware.go | 49 ++++--- .../vmservice/virtualmachine/vm_longevity.go | 7 +- .../virtualmachine/vm_multipleclusters.go | 9 +- .../vmservice/virtualmachine/vm_networking.go | 7 +- .../vmservice/virtualmachine/vm_snapshot.go | 4 +- .../virtualmachine/vm_vpcnetworking.go | 7 +- 25 files changed, 508 insertions(+), 92 deletions(-) rename hack/{ => e2e}/install-pykmip.sh (100%) create mode 100755 hack/e2e/install-squid.sh rename hack/{ => e2e}/kms.sh (100%) create mode 100755 hack/e2e/proxy.sh create mode 100755 hack/e2e/setup-e2e-testbed.sh rename hack/{ => e2e}/setup-testbed-env.sh (90%) create mode 100644 hack/e2e/squid.conf rename hack/{ => e2e}/wait-for-artifact-dir.sh (100%) diff --git a/.cursor/rules/e2e-testing.mdc b/.cursor/rules/e2e-testing.mdc index 62d48548d..97e8843df 100644 --- a/.cursor/rules/e2e-testing.mdc +++ b/.cursor/rules/e2e-testing.mdc @@ -15,7 +15,7 @@ alwaysApply: false **Environment setup** (kubeconfig under `~/.kube/wcp-config`, vars exported — see README for `testbedInfo.json` or a remote URL): ```bash -source ./hack/setup-testbed-env.sh --e2e +source ./hack/e2e/setup-testbed-env.sh --e2e ``` **Makefile targets** (from repo root): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 861c985c1..5da679b64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -212,6 +212,15 @@ jobs: needs: - verify-go-modules runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - target: image-build + dir: . + - target: e2e-image-build + dir: test/e2e + name: ${{ matrix.target }} steps: - name: Check out code uses: actions/checkout@v4 @@ -219,11 +228,11 @@ jobs: uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} - go-version-file: 'go.mod' + go-version-file: '${{ matrix.dir }}/go.mod' cache: true - cache-dependency-path: '**/go.sum' - - name: Build Image - run: GOOS=linux GOARCH=amd64 make image-build + cache-dependency-path: '${{ matrix.dir }}/**/go.sum' + - name: Build ${{ matrix.target }} + run: GOOS=linux GOARCH=amd64 make ${{ matrix.target }} test: needs: diff --git a/.gitignore b/.gitignore index 67ea72ce3..a4b2bec9e 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,4 @@ infrastructure-components.yaml test/e2e/**/test_logs e2e-tests kubectl +test/e2e/**/test-results.xml diff --git a/Dockerfile.e2e b/Dockerfile.e2e index 7d2476e2e..d08847465 100644 --- a/Dockerfile.e2e +++ b/Dockerfile.e2e @@ -59,9 +59,18 @@ RUN tdnf update -y && \ glibc-devel \ go \ sudo \ + unzip \ + openssl \ && \ rm -rf /var/cache/tdnf +# Install govc - required by hack/kms.sh and hack/proxy.sh for gateway VM discovery and KMS setup. +# The default URL points to the public GitHub release. This is configurable to support overrides +# by the build system. +ARG GOVC_DOWNLOAD_URL=https://github.com/vmware/govmomi/releases/download/v0.53.0/govc_Linux_x86_64.tar.gz +RUN curl -fsSL "${GOVC_DOWNLOAD_URL}" | tar -xz -C /usr/local/bin govc && \ + chmod +x /usr/local/bin/govc + # kubectl is always staged into the build context as ./kubectl before docker build runs. # This is to support passing the kubectl binary from the build system. COPY kubectl /usr/local/bin/kubectl @@ -91,11 +100,8 @@ RUN mkdir -p /vm-operator /tmp/go-cache && \ WORKDIR /vm-operator -COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/hack/setup-testbed-env.sh ./hack/setup-testbed-env.sh -COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/hack/wait-for-artifact-dir.sh ./hack/wait-for-artifact-dir.sh -RUN ln -sf /vm-operator/hack/setup-testbed-env.sh /usr/local/bin/setup-testbed-env && \ - ln -sf /vm-operator/hack/wait-for-artifact-dir.sh /usr/local/bin/wait-for-artifact-dir && \ - chmod +x /vm-operator/hack/wait-for-artifact-dir.sh +COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/hack/e2e/ ./hack/e2e/ +RUN chmod +x /vm-operator/hack/e2e/*.sh COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/Makefile ./Makefile COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/e2e-tests ./e2e-tests diff --git a/Makefile b/Makefile index a0617c6f8..e0d57ef4b 100644 --- a/Makefile +++ b/Makefile @@ -1011,7 +1011,7 @@ test-e2e: ## Run e2e tests (auto-detect: prebuilt binary if available, else gink test-e2e-prebuilt: ## Run e2e tests using precompiled binary. Used by the E2E container image. @test -x "$(E2E_PREBUILT_BINARY)" || { echo "error: $(E2E_PREBUILT_BINARY) missing or not executable. Run: cd test/e2e/vmservice && go test -c -o ../../../e2e-tests ."; exit 1; } @echo "Running E2E tests (prebuilt $(E2E_PREBUILT_BINARY))..." - @$(eval GINKGO_ARGS := --ginkgo.v) + @$(eval GINKGO_ARGS := --ginkgo.v --ginkgo.junit-report=$(or $(E2E_ARTIFACT_FOLDER),.)/test-results.xml) @$(eval E2E_ARGS := -e2e.e2e-config="$(ROOT_DIR)test/e2e/vmservice/config/wcp.yaml" -e2e.artifactFolder=$(or $(E2E_ARTIFACT_FOLDER),test_logs)) $(if $(TEST_FOCUS),$(eval GINKGO_ARGS += --ginkgo.focus="$(TEST_FOCUS)")) $(if $(TEST_SKIP),$(eval GINKGO_ARGS += --ginkgo.skip="$(TEST_SKIP)")) @@ -1024,7 +1024,7 @@ test-e2e-prebuilt: ## Run e2e tests using precompiled binary. Used by the E2E co test-e2e-ginkgo: | $(GINKGO) test-e2e-ginkgo: ## Run e2e tests using ginkgo CLI (compile + run) @echo "Running E2E tests (ginkgo compile)..." - @$(eval GINKGO_ARGS := -v) + @$(eval GINKGO_ARGS := -v --junit-report=$(or $(E2E_ARTIFACT_FOLDER),.)/test-results.xml) @$(eval E2E_ARGS := -e2e.e2e-config="$(ROOT_DIR)test/e2e/vmservice/config/wcp.yaml" -e2e.artifactFolder=$(or $(E2E_ARTIFACT_FOLDER),test_logs)) $(if $(TEST_FOCUS),$(eval GINKGO_ARGS += --focus="$(TEST_FOCUS)")) $(if $(TEST_SKIP),$(eval GINKGO_ARGS += --skip="$(TEST_SKIP)")) diff --git a/hack/install-pykmip.sh b/hack/e2e/install-pykmip.sh similarity index 100% rename from hack/install-pykmip.sh rename to hack/e2e/install-pykmip.sh diff --git a/hack/e2e/install-squid.sh b/hack/e2e/install-squid.sh new file mode 100755 index 000000000..3fdb491d7 --- /dev/null +++ b/hack/e2e/install-squid.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# Function to ensure that the "squid" package is installed +ensure_squid_package() { + if [ -f "/etc/os-release" ]; then + source "/etc/os-release" + if [[ $ID == "ubuntu" ]]; then + if [ "$(dpkg-query -W -f='${Status}' squid 2>/dev/null | grep -c "ok installed")" -eq 0 ]; then + echo "Installing Squid package..." + dpkg --configure -a # to fix the imporoperly configured packages in the vm (as left in the appliance image) + apt-get install -y squid + else + echo "Squid package is already installed." + fi + elif [[ $ID == "photon" ]]; then + if ! rpm -q squid >/dev/null 2>&1; then + echo "Installing Squid package..." + tdnf install -y squid --nogpgcheck + else + echo "Squid package is already installed." + fi + else + echo "Unsupported operating system. Exiting..." + exit 1 + fi + else + echo "Unable to detect operating system. Exiting..." + exit 1 + fi +} + +# Function to interact with the squid service +squid_service_action() { + echo "${1}ing Squid service..." + systemctl ${1} squid +} + +usage() { + echo "./install-squid.sh [install|start|restart|stop]" + exit 1 +} + +main() { + if [ -z "$1" ]; then + usage + fi + case $1 in + "install") + ensure_squid_package + ;; + "start"|"stop"|"restart") + squid_service_action $1 + ;; + *) + usage + ;; + esac +} + +main "$@" diff --git a/hack/kms.sh b/hack/e2e/kms.sh similarity index 100% rename from hack/kms.sh rename to hack/e2e/kms.sh diff --git a/hack/e2e/proxy.sh b/hack/e2e/proxy.sh new file mode 100755 index 000000000..3f2bde837 --- /dev/null +++ b/hack/e2e/proxy.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# This script helps you to find out the external-gateway-vm's IP given the +# vCenter IP. It also installs/uninstalls an httpd proxy on the target vm exposed +# at port 3128 with a predefined configuration file. +# Set environmental variable HTTP_PROXY=${GATEWAY_VM_IP}:3128 to use it. +# This files works with an httpd.conf file that lives under the same folder. + + +SCRIPT_DIR="$(dirname "${BASH_SOURCE}")" +GOVC_INSECURE="${GOVC_INSECURE:-1}" +GOVC_USERNAME="${GOVC_USERNAME:-administrator@vsphere.local}" +GOVC_PASSWORD="${GOVC_PASSWORD:-Admin!23}" +GATEWAY_VM_USERNAME="${GATEWAY_VM_USERNAME:-root}" +GATEWAY_VM_PASSWORD=${GATEWAY_VM_PASSWORD:-'vmware'} + + +find_gateway_vm_path() { + # This below code is a hack primarily to compensate for the discrepancies that exist in the various testbed deployment + # methods + vmnames=('external-gateway' 'external-vm-gateway') + for vmname in "${vmnames[@]}"; do + vmpath=$(GOVC_URL=$vc GOVC_INSECURE=$GOVC_INSECURE GOVC_USERNAME=$GOVC_USERNAME GOVC_PASSWORD=$GOVC_PASSWORD govc find / -type m -name ${vmname}) + if [[ "${vmpath}" != "" ]]; then + echo ${vmpath} + break + fi + done +} + +# finds the gateway vm ip in a testbed given a vCenter IP +find_gateway_ip() { + vc=$1 + mgmtCidr=$2 + GATEWAY_VM_PATH=$(find_gateway_vm_path) + ips=$(GOVC_URL=$vc GOVC_INSECURE=$GOVC_INSECURE GOVC_USERNAME=$GOVC_USERNAME GOVC_PASSWORD=$GOVC_PASSWORD govc vm.ip -a "${GATEWAY_VM_PATH}") + # loop over all ips, only return the one starts with '10.' which is + # company-level routable ip + IFS=',' read -ra ADDR <<< "$ips" + for i in "${ADDR[@]}"; do + # ignore error when IP not in CIDR + echo "$i" | grepcidr "$mgmtCidr" || true + done +} + +# installs httpd proxy on the specified remote Linux machine with predefined configuration file +install() { + user=$1 + ip=$2 + password=$3 + sshpass -p "$password" scp -o StrictHostKeyChecking=no ${SCRIPT_DIR}/install-squid.sh $user@$ip:/root/install-squid.sh + sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip /root/install-squid.sh install + sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip /root/install-squid.sh stop + # Reason for the pkill is that on ubuntu squid systemctl restart is not reliable to refresh the process. + # Without this the process originally started by the default package install continues and cause failure to process the updated squid config + sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip pkill -e squid* || true # ignore not found + sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip netstat -tunlp | grep 3128 || true # ignore not found + + sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip 'cat > /etc/squid/squid.conf' < ${SCRIPT_DIR}/squid.conf + sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip 'cat /etc/squid/squid.conf' + sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip /root/install-squid.sh start + # Wait for squid to start listening on port 3128 (timeout after 60s) + echo "Waiting for squid to start listening on port 3128..." + for i in $(seq 1 60); do + if sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip netstat -tunlp 2>/dev/null | grep -q 3128; then + echo "Squid is listening on port 3128 after ${i}s" + break + fi + if [ $i -eq 60 ]; then + echo "Timeout waiting for squid to start on port 3128" + exit 1 + fi + sleep 1 + done + sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip ps -elf | grep squid + sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip netstat -tunlp | grep 3128 +} + +# stops httpd on the specified remote Linux machine +uninstall() { + user=$1 + ip=$2 + password=$3 + sshpass -p "$password" scp -o StrictHostKeyChecking=no ${SCRIPT_DIR}/install-squid.sh $user@$ip:/root/install-squid.sh + sshpass -p "$password" ssh -o StrictHostKeyChecking=no $user@$ip /root/install-squid.sh stop +} + +usage() { + echo "./proxy.sh [install|uninstall|gateway] [vCenter IP] [Management Network CIDR]" + exit 1 +} + +hasGateway() { + gatewayIp=$1 + if [[ "${gatewayIp}" == null ]]; then + echo "Gateway IP is null or could not be found. Are you sure this is a VDS testbed?" + exit 1 + fi + echo "Gateway IP is ${gatewayIp}" +} + +main() { + if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then + usage + fi + case $1 in + "install") + gatewayIp=$(find_gateway_ip $2 $3) + hasGateway $gatewayIp + install ${GATEWAY_VM_USERNAME} ${gatewayIp} ${GATEWAY_VM_PASSWORD} + # Print a message that the user can copy/paste to set up their proxy. + cat </dev/null; then + unzip -o vsphere-plugin.zip -d /tmp/vsphere-plugin + # Try /usr/local/bin (directly or via sudo), fall back to ~/.local/bin + if mv -f /tmp/vsphere-plugin/bin/* /usr/local/bin/ 2>/dev/null || \ + sudo mv -f /tmp/vsphere-plugin/bin/* /usr/local/bin/ 2>/dev/null; then + echo "✓ kubectl-vsphere installed to /usr/local/bin" + else + echo "⚠ Could not write to /usr/local/bin, installing to ~/.local/bin" + mkdir -p "${HOME}/.local/bin" + mv -f /tmp/vsphere-plugin/bin/* "${HOME}/.local/bin/" + export PATH="${HOME}/.local/bin:${PATH}" + echo "✓ kubectl-vsphere installed to ${HOME}/.local/bin" + fi + rm -rf vsphere-plugin.zip /tmp/vsphere-plugin + else + echo "⚠ Failed to download kubectl-vsphere plugin from ${WCP_IP}" + fi +else + echo "⚠ Skipping kubectl-vsphere install: WCP_IP not set" +fi + +# --------------------------------------------------------------------------- +# Gateway VM / HTTP proxy and KMS key providers +# Both require govc to discover the external-gateway VM in the testbed. +# --------------------------------------------------------------------------- +GATEWAY_IP="" +if [ -n "${VC_URL:-}" ] && command -v govc >/dev/null 2>&1; then + export GOVC_URL="${VC_ROOT_USERNAME}:${VC_ROOT_PASSWORD}@${VC_URL}" + export GOVC_INSECURE=true + + echo "Discovering gateway VM IP via govc..." + GATEWAY_IP=$(GOVC_USERNAME="${VC_ROOT_USERNAME}" GOVC_PASSWORD="${VC_ROOT_PASSWORD}" \ + "${SCRIPT_DIR}/proxy.sh" gateway "${VC_URL}" "${MGMT_CIDR}" 2>/dev/null || true) + + if [ -n "${GATEWAY_IP:-}" ] && [ "${GATEWAY_IP}" != "null" ]; then + echo "Gateway VM IP: ${GATEWAY_IP}" + + # Install squid proxy on gateway and export HTTP_PROXY. + # Tests using VerifyLoginAndRunCmdsInVDSSetup require HTTP_PROXY to be set. + echo "Installing squid proxy on gateway VM..." + GOVC_USERNAME="${VC_ROOT_USERNAME}" GOVC_PASSWORD="${VC_ROOT_PASSWORD}" \ + "${SCRIPT_DIR}/proxy.sh" install "${VC_URL}" "${MGMT_CIDR}" 2>/dev/null \ + || echo "⚠ Proxy install failed (may already be set up)" + export HTTP_PROXY="${GATEWAY_IP}:3128" + export HTTPS_PROXY="${GATEWAY_IP}:3128" + export GATEWAY_IP="${GATEWAY_IP}" + echo "✓ HTTP_PROXY set to ${HTTP_PROXY}" + + # Full KMS install: deploys pykmip on the gateway VM then registers both + # gce2e-standard (KMIP) and gce2e-native key providers with vCenter. + echo "Setting up KMS key providers (with pykmip on gateway VM)..." + GOVC_USERNAME="${VC_ROOT_USERNAME}" GOVC_PASSWORD="${VC_ROOT_PASSWORD}" \ + "${SCRIPT_DIR}/kms.sh" install "${GOVC_URL}" "${MGMT_CIDR}" 2>/dev/null \ + && echo "✓ KMS key providers configured" \ + || echo "⚠ KMS install failed (may already be configured)" + else + echo "⚠ Could not find gateway VM (may not be a VDS testbed)" + + # Without a gateway VM, we can still configure the native key provider + # directly on vCenter (no pykmip / gce2e-standard). + echo "Setting up KMS native key provider only..." + GOVC_USERNAME="${VC_ROOT_USERNAME}" GOVC_PASSWORD="${VC_ROOT_PASSWORD}" \ + "${SCRIPT_DIR}/kms.sh" setup "${GOVC_URL}" "${MGMT_CIDR}" 2>/dev/null \ + && echo "✓ KMS native key provider configured" \ + || echo "⚠ KMS setup failed" + fi +else + if ! command -v govc >/dev/null 2>&1; then + echo "⚠ Skipping gateway/proxy and KMS setup: govc not found in PATH" + fi +fi + +echo " HTTP_PROXY: ${HTTP_PROXY:-not set}" +echo " GATEWAY_IP: ${GATEWAY_IP:-not set}" +echo "=== E2E Testbed Setup Complete ===" diff --git a/hack/setup-testbed-env.sh b/hack/e2e/setup-testbed-env.sh similarity index 90% rename from hack/setup-testbed-env.sh rename to hack/e2e/setup-testbed-env.sh index c04097c95..c3fc508ce 100755 --- a/hack/setup-testbed-env.sh +++ b/hack/e2e/setup-testbed-env.sh @@ -6,14 +6,14 @@ # for running vm-operator E2E tests by setting up kubeconfig and environment variables. # # # Local file: -# source ./hack/setup-testbed-env.sh ./testbedinfo.json +# source ./hack/e2e/setup-testbed-env.sh ./testbedinfo.json # # # Remote URL: -# source ./hack/setup-testbed-env.sh https://example.com/testbed.json +# source ./hack/e2e/setup-testbed-env.sh https://example.com/testbed.json # # # Enable E2E kubeconfig setup by copying kubeconfig to ~/.kube/wcp-config # # which is used by the E2E test to access the Supervisor cluster. -# source ./hack/setup-testbed-env.sh ./testbedinfo.json --e2e +# source ./hack/e2e/setup-testbed-env.sh ./testbedinfo.json --e2e # set -u @@ -47,7 +47,8 @@ parse_args() { echo " TESTBED_SOURCE Path to local testbedinfo.json or HTTP(S) URL (required)" echo "" echo "Options:" - echo " --e2e Enable E2E kubeconfig setup (copy kubeconfig to wcp-config)" + echo " --e2e Enable E2E setup: copy kubeconfig to wcp-config, install" + echo " kubectl-vsphere, set up HTTP proxy and KMS key providers" echo " --help, -h Show this help message" echo "" echo "Examples:" @@ -263,10 +264,20 @@ fi export SUPERVISOR_CLUSTER_IP="${WCP_IP:-}" export SUPERVISOR_CLUSTER_PASSWORD="${WCP_PASSWORD:-}" +# Run E2E-specific setup when --e2e is passed: +# - install kubectl-vsphere plugin +# - set up squid proxy on gateway VM and export HTTP_PROXY +# - configure KMS key providers (gce2e-native, gce2e-standard) +if [ "$ENABLE_E2E_KUBECONFIG" = "true" ]; then + _SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # shellcheck source=hack/e2e/setup-e2e-testbed.sh + source "${_SCRIPT_DIR}/setup-e2e-testbed.sh" +fi + echo "" echo "=== Environment Setup Complete ===" echo "Data source: ${TESTBED_SOURCE}" -echo "E2E kubeconfig: $(if [ "$ENABLE_E2E_KUBECONFIG" = "true" ]; then echo "Enabled"; else echo "Disabled (use --e2e to enable)"; fi)" +echo "E2E setup: $(if [ "$ENABLE_E2E_KUBECONFIG" = "true" ]; then echo "Enabled (--e2e)"; else echo "Disabled (use --e2e to enable)"; fi)" echo "" echo "The following variables are now available:" echo " VC_URL/VCSA_IP: ${VC_URL}" @@ -274,6 +285,8 @@ echo " VC_ROOT_USERNAME/SSH_USERNAME: ${VC_ROOT_USERNAME}" echo " NETWORK: ${NETWORK}" echo " KUBECONFIG: ${KUBECONFIG:-not set}" echo " SUPERVISOR_CLUSTER_IP: ${SUPERVISOR_CLUSTER_IP:-not set}" +echo " HTTP_PROXY: ${HTTP_PROXY:-not set}" +echo " GATEWAY_IP: ${GATEWAY_IP:-not set}" echo "" echo "You can now run vm-operator E2E tests with these credentials." echo "Example: make e2e-smoke" diff --git a/hack/e2e/squid.conf b/hack/e2e/squid.conf new file mode 100644 index 000000000..2951f36e9 --- /dev/null +++ b/hack/e2e/squid.conf @@ -0,0 +1,22 @@ +# Requests denied by this directive will not be served from the cache and their +# responses will not be stored in the cache. This directive has no effect on +# other transactions and on already cached responses. Denies both serving a hit +# and storing a miss. +cache deny all + +# Allow all other access to this proxy +http_access allow all + +# The socket addresses where Squid will listen for HTTP client requests. You +# may specify multiple socket addresses. There are three forms: port alone, +# hostname with port, and IP address with port. If you specify a hostname or IP +# address, Squid binds the socket to that specific address. Squid normally +# listens to port 3128. 0.0.0.0 is here to force Squid to bind on IPv4. +http_port 0.0.0.0:3128 + +# With the IPv6 Internet being as fast or faster than IPv4 Internet for most +# networks Squid prefers to contact websites over IPv6. This option reverses +# the order of preference to make Squid contact dual-stack websites over IPv4 +# first. Squid will still perform both IPv6 and IPv4 DNS lookups before +# connecting. +dns_v4_first on diff --git a/hack/wait-for-artifact-dir.sh b/hack/e2e/wait-for-artifact-dir.sh similarity index 100% rename from hack/wait-for-artifact-dir.sh rename to hack/e2e/wait-for-artifact-dir.sh diff --git a/test/e2e/README.md b/test/e2e/README.md index 7510a64bd..b6a6ebc2a 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -49,7 +49,7 @@ The kubeconfig will be stored under the directory `~/.kube/.kubeconfig` and copied to `~/.kube/wcp-config` which the E2E test reads from. ```bash -source ./hack/setup-testbed-env.sh --e2e +source ./hack/e2e/setup-testbed-env.sh --e2e ``` 2. **Running Tests** diff --git a/test/e2e/vmservice/vmservice/util.go b/test/e2e/vmservice/vmservice/util.go index 3d3f27803..d7cf2001a 100644 --- a/test/e2e/vmservice/vmservice/util.go +++ b/test/e2e/vmservice/vmservice/util.go @@ -1212,6 +1212,7 @@ func LabelVM(ctx context.Context, config *config.E2EConfig, clusterProxy *common func DeployVMWithCloudInit( ctx context.Context, vmSvcClusterProxy *common.VMServiceClusterProxy, + e2eConfig *config.E2EConfig, clusterResources *config.Resources, ns, vmName, groupName string, pvcs []manifestbuilders.PVC) { @@ -1224,11 +1225,13 @@ func DeployVMWithCloudInit( Expect(vmSvcClusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create the Secret with cloud-config data", string(secretYaml)) linuxImageDisplayName := GetDefaultImageDisplayName(clusterResources) + linuxVMIName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &e2eConfig.Config, vmSvcClusterProxy.GetClient(), ns, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get VMI name for display name %q in namespace %q", linuxImageDisplayName, ns) vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: ns, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, SecretName: secretName, @@ -1742,6 +1745,12 @@ func SetupClusterRoleBindings(clusterProxy *common.VMServiceClusterProxy) error Resource: "cnsnodevmbatchattachments", Verbs: "get,list,watch", }, + { + Name: "gce2e-cns-unregister-volumes", + APIGroup: "cns.vmware.com", + Resource: "cnsunregistervolumes", + Verbs: "create,update,patch,delete,get,list,watch", + }, { Name: "gce2e-import-operations", APIGroup: "mobility-operator.vmware.com", @@ -1841,6 +1850,7 @@ func CleanupClusterRoleBindings(clusterProxy *common.VMServiceClusterProxy) erro customRoles := []RBACRole{ {Name: "gce2e-crds"}, {Name: "gce2e-cns-batch-attachments"}, + {Name: "gce2e-cns-unregister-volumes"}, {Name: "gce2e-import-operations"}, } diff --git a/test/e2e/vmservice/vmservice/viadmin/registervm.go b/test/e2e/vmservice/vmservice/viadmin/registervm.go index 700dc093a..cc2803cbe 100644 --- a/test/e2e/vmservice/vmservice/viadmin/registervm.go +++ b/test/e2e/vmservice/vmservice/viadmin/registervm.go @@ -75,7 +75,8 @@ func VIAdminRegisterVMSpec(ctx context.Context, inputGetter func() VIAdminRegist svClusterClientSet *kubernetes.Clientset vmServiceBackupRestoreEnabled bool incrementalRestoreEnabled bool - linuxImageDisplayName string + linuxImageDisplayName string + linuxVMIName string ) BeforeEach(func() { @@ -96,6 +97,10 @@ func VIAdminRegisterVMSpec(ctx context.Context, inputGetter func() VIAdminRegist linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(config.InfraConfig.ManagementClusterConfig.Resources) + var vmiErr error + linuxVMIName, vmiErr = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(vmiErr).NotTo(HaveOccurred(), "failed to get VMI name for display name %q in namespace %q", linuxImageDisplayName, input.WCPNamespaceName) + vmServiceBackupRestoreEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMServiceBackupRestore")) incrementalRestoreEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSIncrementalRestore")) }) @@ -240,7 +245,7 @@ func VIAdminRegisterVMSpec(ctx context.Context, inputGetter func() VIAdminRegist VMClassName: resources.VMClassName, StorageClassName: resources.StorageClassName, ResourcePolicy: resources.VMResourcePolicyName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, Bootstrap: manifestbuilders.Bootstrap{ CloudInit: &manifestbuilders.CloudInit{ RawCloudConfig: &manifestbuilders.KeySelector{ @@ -373,7 +378,7 @@ func VIAdminRegisterVMSpec(ctx context.Context, inputGetter func() VIAdminRegist VMClassName: resources.VMClassName, StorageClassName: resources.StorageClassName, ResourcePolicy: resources.VMResourcePolicyName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, Bootstrap: manifestbuilders.Bootstrap{ CloudInit: &manifestbuilders.CloudInit{ RawCloudConfig: &manifestbuilders.KeySelector{ @@ -586,7 +591,7 @@ func VIAdminRegisterVMSpec(ctx context.Context, inputGetter func() VIAdminRegist VMClassName: resources.VMClassName, StorageClassName: resources.StorageClassName, ResourcePolicy: resources.VMResourcePolicyName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, Bootstrap: manifestbuilders.Bootstrap{ CloudInit: &manifestbuilders.CloudInit{ RawCloudConfig: &manifestbuilders.KeySelector{ @@ -818,7 +823,7 @@ func VIAdminRegisterVMSpec(ctx context.Context, inputGetter func() VIAdminRegist VMClassName: resources.VMClassName, StorageClassName: resources.StorageClassName, ResourcePolicy: resources.VMResourcePolicyName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, Bootstrap: manifestbuilders.Bootstrap{ CloudInit: &manifestbuilders.CloudInit{ RawCloudConfig: &manifestbuilders.KeySelector{ diff --git a/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go b/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go index a2b5bfe97..9a6256c07 100644 --- a/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go +++ b/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go @@ -82,6 +82,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { vmResizeCPUMemoryFssEnabled bool isoSupportFSSEnabled bool linuxImageDisplayName string + linuxVMIName string linuxImageGuestID string ) @@ -104,6 +105,10 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) linuxImageGuestID = vmservice.GetDefaultImageGuestID() + var err error + linuxVMIName, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get VMI name for display name %q in namespace %q", linuxImageDisplayName, input.WCPNamespaceName) + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, clusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, specName)) DeferCleanup(cancelPodWatches) @@ -176,7 +181,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, @@ -219,7 +224,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PowerState: "PoweredOff", @@ -289,7 +294,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: vmservice.VMClassE1000, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, @@ -340,7 +345,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: vmservice.VMClassVMX22, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, @@ -381,7 +386,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, @@ -476,7 +481,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: vmClassName, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, @@ -517,7 +522,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: vmClassName, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, @@ -779,6 +784,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { var ( tagManager *tags.Manager tmpNamespaceName string + tmpNamespaceVMIName string mandatoryPolicyNameMatchAll string mandatoryPolicyNameMatchByLabel string opPolicyNameMatchByGuestID string @@ -806,12 +812,12 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { wcp.WaitForNamespaceReady(wcpClient, tmpNamespaceName) By("Deploying a VM without any explicit policies") - vmiName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) + tmpNamespaceVMIName, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) Expect(err).NotTo(HaveOccurred(), "failed to get the VMI name in namespace %q", tmpNamespaceName) vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: tmpNamespaceName, Name: vmName, - ImageName: vmiName, + ImageName: tmpNamespaceVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, @@ -945,7 +951,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { Namespace: tmpNamespaceName, Name: vmName, GuestID: linuxImageGuestID, - ImageName: linuxImageDisplayName, + ImageName: tmpNamespaceVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_group.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_group.go index f35f60051..a666ffe45 100644 --- a/test/e2e/vmservice/vmservice/virtualmachine/vm_group.go +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_group.go @@ -72,6 +72,7 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { vmMemberNames []string linuxImageDisplayName string + linuxVMIName string ) BeforeEach(func() { @@ -96,6 +97,10 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + var err error + linuxVMIName, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get VMI name for display name %q in namespace %q", linuxImageDisplayName, input.WCPNamespaceName) + vmgRootYaml = nil vmMemberNames = []string{} vmgRootName = fmt.Sprintf("%s-%s-root", specName, capiutil.RandomString(4)) @@ -185,7 +190,7 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { Namespace: input.WCPNamespaceName, Name: vmName, GroupName: vmgRootName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PowerState: "PoweredOff", @@ -313,7 +318,7 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { Namespace: input.WCPNamespaceName, Name: vm4Name, PowerState: "PoweredOn", - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, } @@ -370,7 +375,7 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { GroupName: vmgRootName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, } vm1Yaml := manifestbuilders.GetVirtualMachineYamlA5(vm1Parameters) Expect(clusterProxy.ApplyWithArgs(ctx, vm1Yaml)).To(Succeed(), "failed to update VM %q power state:\n %s", vm1Name, string(vm1Yaml)) @@ -471,7 +476,7 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { Namespace: input.WCPNamespaceName, Name: vm1Name, GroupName: vmgRootName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PowerState: "PoweredOff", @@ -485,7 +490,7 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { Namespace: input.WCPNamespaceName, Name: vm2Name, GroupName: vmgChildName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PowerState: "PoweredOff", @@ -589,8 +594,9 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { Context("Group placement with affinity and anti-affinity", func() { var ( - tmpNamespaceName string - tmpNamespaceCtx wcpframework.NamespaceContext + tmpNamespaceName string + tmpNamespaceCtx wcpframework.NamespaceContext + tmpNamespaceVMIName string createVMWithAffinityAndAntiAffinityFunc = func(vmName, affinityTier string, antiAffinityTiers []string) { GinkgoHelper() @@ -600,7 +606,7 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { Name: vmName, GroupName: vmgRootName, Labels: map[string]string{"tier": affinityTier}, - ImageName: linuxImageDisplayName, + ImageName: tmpNamespaceVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, Affinity: &vmopv1a5.AffinitySpec{ @@ -676,7 +682,7 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { By("Ensuring the Linux image is available in the temp namespace") - _, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) + tmpNamespaceVMIName, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) Expect(err).NotTo(HaveOccurred(), "failed to get VMI by display name %q in namespace %q", linuxImageDisplayName, tmpNamespaceName) By("Binding all zones to the temporary namespace") @@ -951,8 +957,9 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { ) var ( - tmpNamespaceName string - tmpNamespaceCtx wcpframework.NamespaceContext + tmpNamespaceName string + tmpNamespaceCtx wcpframework.NamespaceContext + tmpNamespaceVMIName string ) // getVMHostFromVmodlFunc retrieves the host moref value from vSphere directly. @@ -1058,24 +1065,24 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { } } - vmParameters := manifestbuilders.VirtualMachineYaml{ - Namespace: tmpNamespaceName, - Name: vmName, - GroupName: vmgRootName, - Labels: labels, - ImageName: linuxImageDisplayName, - VMClassName: clusterResources.VMClassName, - StorageClassName: clusterResources.StorageClassName, - PowerState: string(vmopv1a5.VirtualMachinePowerStateOff), - Affinity: &vmopv1a5.AffinitySpec{ - VMAffinity: affinityLabelSelector, - VMAntiAffinity: antiAffinityLabelSelector, - }, - } - vmYAML := manifestbuilders.GetVirtualMachineYamlA5(vmParameters) - e2eframework.Logf("VM YAML:\n%s", string(vmYAML)) - Expect(clusterProxy.ApplyWithArgs(ctx, vmYAML)).To(Succeed(), "failed to create vm %s:\n %s", vmName, string(vmYAML)) + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + GroupName: vmgRootName, + Labels: labels, + ImageName: tmpNamespaceVMIName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PowerState: string(vmopv1a5.VirtualMachinePowerStateOff), + Affinity: &vmopv1a5.AffinitySpec{ + VMAffinity: affinityLabelSelector, + VMAntiAffinity: antiAffinityLabelSelector, + }, } + vmYAML := manifestbuilders.GetVirtualMachineYamlA5(vmParameters) + e2eframework.Logf("VM YAML:\n%s", string(vmYAML)) + Expect(clusterProxy.ApplyWithArgs(ctx, vmYAML)).To(Succeed(), "failed to create vm %s:\n %s", vmName, string(vmYAML)) + } verifyAffinity := func(vmHosts map[string]string, affinedVms []string) { GinkgoHelper() @@ -1265,7 +1272,7 @@ func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { By("Ensuring the Linux image is available in the temp namespace") - _, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) + tmpNamespaceVMIName, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) Expect(err).NotTo(HaveOccurred(), "failed to get VMI by display name %q in namespace %q", linuxImageDisplayName, tmpNamespaceName) By("Binding all zones to the temporary namespace") diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_group_publishrequest.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_group_publishrequest.go index 0b16c18f6..ec510a96a 100644 --- a/test/e2e/vmservice/vmservice/virtualmachine/vm_group_publishrequest.go +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_group_publishrequest.go @@ -123,8 +123,8 @@ func VMGroupPublishRequestSpec(ctx context.Context, inputGetter func() VMGroupPu } createVMs := func() { - vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcClusterResources, vmSvcNamespace, vm1Name, vmChildGroupName, nil) - vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcClusterResources, vmSvcNamespace, vm0Name, vmGroupName, nil) + vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcE2EConfig, vmSvcClusterResources, vmSvcNamespace, vm1Name, vmChildGroupName, nil) + vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcE2EConfig, vmSvcClusterResources, vmSvcNamespace, vm0Name, vmGroupName, nil) vmoperator.VerifyVirtualMachineGroupLinked(ctx, vmSvcE2EConfig, vmSvcClusterProxy.GetClient(), vmSvcNamespace, vmGroupName, sets.New([]vmopv1a5.GroupMember{ {Kind: "VirtualMachine", Name: vm0Name}, {Kind: "VirtualMachineGroup", Name: vmChildGroupName}, diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_hardware.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_hardware.go index df95d78ca..c2a706608 100644 --- a/test/e2e/vmservice/vmservice/virtualmachine/vm_hardware.go +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_hardware.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "os" + "path/filepath" "slices" "strings" @@ -251,6 +252,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) isoSupportFSSEnabled bool allDisksArePVCapabilityEnabled bool linuxImageDisplayName string + linuxVMIName string ) Context("VMs with attached hardware", Ordered, func() { @@ -280,13 +282,20 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) wcpClient = input.WCPClient kubeconfigPath := input.ClusterProxy.GetKubeconfigPath() - clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) - svClusterClient = clusterProxy.GetClient() - svClientSet := clusterProxy.GetClientSet() + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterClient = clusterProxy.GetClient() + svClientSet := clusterProxy.GetClientSet() - linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, svClientSet, filepath.Join(input.ArtifactFolder, specName)) + DeferCleanup(cancelPodWatches) - isoSupportFSSEnabled = utils.IsFssEnabled(ctx, + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + + var vmiErr error + linuxVMIName, vmiErr = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, vmSvcNamespace, linuxImageDisplayName) + Expect(vmiErr).NotTo(HaveOccurred(), "failed to get VMI name for display name %q in namespace %q", linuxImageDisplayName, vmSvcNamespace) + + isoSupportFSSEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), @@ -426,7 +435,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PVCs: spec.pvcs, @@ -789,7 +798,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PowerState: string(vmPowerState), @@ -872,7 +881,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PowerState: string(vmopv1a5.VirtualMachinePowerStateOff), @@ -925,7 +934,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PowerState: string(vmopv1a5.VirtualMachinePowerStateOn), @@ -1004,7 +1013,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml := manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PVCs: pvcs, @@ -1046,7 +1055,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PVCs: pvcs, @@ -1129,7 +1138,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PVCs: pvcs, @@ -1214,7 +1223,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PVCs: pvcs, @@ -1300,7 +1309,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PVCs: pvcs, @@ -1356,7 +1365,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PVCs: pvcs, @@ -1536,7 +1545,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, GuestID: "ubuntu64Guest", StorageClassName: clusterResources.StorageClassName, @@ -1606,7 +1615,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PVCs: []manifestbuilders.PVC{}, @@ -2253,7 +2262,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) }, Spec: vmopv1a5.VirtualMachineSpec{ ClassName: clusterResources.VMClassName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, StorageClass: clusterResources.StorageClassName, Volumes: []vmopv1a5.VirtualMachineVolume{ { @@ -2307,7 +2316,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) }, Spec: vmopv1a5.VirtualMachineSpec{ ClassName: clusterResources.VMClassName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, StorageClass: clusterResources.StorageClassName, Hardware: &vmopv1a5.VirtualMachineHardwareSpec{ SCSIControllers: []vmopv1a5.SCSIControllerSpec{ @@ -2366,7 +2375,7 @@ func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) vmYaml := manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ Namespace: vmSvcNamespace, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PVCs: pvcs, diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_longevity.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_longevity.go index 26c06f995..ca792f9ee 100644 --- a/test/e2e/vmservice/vmservice/virtualmachine/vm_longevity.go +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_longevity.go @@ -62,6 +62,7 @@ func VMLongevitySpec(ctx context.Context, inputGetter func() VMLongevityInput) { imageReady metav1.Condition storageReady metav1.Condition linuxImageDisplayName string + linuxVMIName string ) BeforeEach(func() { @@ -82,6 +83,10 @@ func VMLongevitySpec(ctx context.Context, inputGetter func() VMLongevityInput) { svClusterClient = clusterProxy.GetClient() linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + var vmiErr error + linuxVMIName, vmiErr = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(vmiErr).NotTo(HaveOccurred(), "failed to get VMI name for display name %q in namespace %q", linuxImageDisplayName, input.WCPNamespaceName) + By("Create a second namespace without VMClass") if _, err := wcpClient.GetNamespace(secondNamespaceName); err != nil { @@ -144,7 +149,7 @@ func VMLongevitySpec(ctx context.Context, inputGetter func() VMLongevityInput) { vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: vmClassName, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_multipleclusters.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_multipleclusters.go index 32dd07212..f76dac347 100644 --- a/test/e2e/vmservice/vmservice/virtualmachine/vm_multipleclusters.go +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_multipleclusters.go @@ -49,6 +49,7 @@ func VMMultipleClusterSpec(ctx context.Context, inputGetter func() VMMultipleClu vmName string linuxImageDisplayName string + linuxVMIName string ) BeforeEach(func() { @@ -70,6 +71,10 @@ func VMMultipleClusterSpec(ctx context.Context, inputGetter func() VMMultipleClu linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + var err error + linuxVMIName, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get VMI name for display name %q in namespace %q", linuxImageDisplayName, input.WCPNamespaceName) + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, input.ClusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, specName)) DeferCleanup(cancelPodWatches) }) @@ -111,7 +116,7 @@ func VMMultipleClusterSpec(ctx context.Context, inputGetter func() VMMultipleClu Namespace: input.WCPNamespaceName, Name: vmName, Labels: map[string]string{"topology.kubernetes.io/zone": zoneName}, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, @@ -133,7 +138,7 @@ func VMMultipleClusterSpec(ctx context.Context, inputGetter func() VMMultipleClu vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_networking.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_networking.go index bc78d90ae..791daf606 100644 --- a/test/e2e/vmservice/vmservice/virtualmachine/vm_networking.go +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_networking.go @@ -68,6 +68,7 @@ func VMNetworkSpec(ctx context.Context, inputGetter func() VMNetworkSpecInput) { isVMMutableNetworksCapEnabled bool linuxImageDisplayName string + linuxVMIName string ) BeforeEach(func() { @@ -94,6 +95,10 @@ func VMNetworkSpec(ctx context.Context, inputGetter func() VMNetworkSpecInput) { tmpNamespaceCtx = wcpframework.NamespaceContext{} vmName = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + var err error + linuxVMIName, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get VMI name for display name %q in namespace %q", linuxImageDisplayName, input.WCPNamespaceName) + sshCommandRunner, _ := e2essh.NewSSHCommandRunner( vcenter.GetVCPNIDFromKubeconfigFile(ctx, svClusterProxy.GetKubeconfigPath()), vcenter.VCSSHPort, testbed.RootUsername, []ssh.AuthMethod{ssh.Password(testbed.RootPassword)}) @@ -134,7 +139,7 @@ func VMNetworkSpec(ctx context.Context, inputGetter func() VMNetworkSpecInput) { vmParameters := manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, Name: vmName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, VMClassName: clusterResources.VMClassName, StorageClassName: clusterResources.StorageClassName, PowerState: "PoweredOff", diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_snapshot.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_snapshot.go index ece315b88..de4b0f914 100644 --- a/test/e2e/vmservice/vmservice/virtualmachine/vm_snapshot.go +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_snapshot.go @@ -134,7 +134,7 @@ func VMSnapshotSpec(ctx context.Context, inputGetter func() VMSnapshotSpecInput) When("VM doesn't have PVC", func() { BeforeEach(func() { - vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcClusterResources, vmSvcNamespace, vmName, "", nil) + vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcE2EConfig, vmSvcClusterResources, vmSvcNamespace, vmName, "", nil) vmoperator.WaitForVirtualMachineConditionCreated(ctx, vmSvcE2EConfig, svClusterClient, vmSvcNamespace, vmName) vmoperator.WaitForVirtualMachinePowerState(ctx, vmSvcE2EConfig, svClusterClient, vmSvcNamespace, vmName, "PoweredOn") }) @@ -385,7 +385,7 @@ func VMSnapshotSpec(ctx context.Context, inputGetter func() VMSnapshotSpecInput) When("VM has PVC", func() { BeforeEach(func() { - vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcClusterResources, vmSvcNamespace, vmName, "", []manifestbuilders.PVC{ + vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcE2EConfig, vmSvcClusterResources, vmSvcNamespace, vmName, "", []manifestbuilders.PVC{ { VolumeName: "test-vol" + randomString, ClaimName: "test-claim" + randomString, diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_vpcnetworking.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_vpcnetworking.go index 17ad8398c..4c1219b12 100644 --- a/test/e2e/vmservice/vmservice/virtualmachine/vm_vpcnetworking.go +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_vpcnetworking.go @@ -110,6 +110,7 @@ func VMVPCSpec(ctx context.Context, inputGetter func() VMVPCSpecInput) { secretName string vmYaml []byte linuxImageDisplayName string + linuxVMIName string ) BeforeEach(func() { @@ -137,6 +138,10 @@ func VMVPCSpec(ctx context.Context, inputGetter func() VMVPCSpecInput) { linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) vm1Name = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) vm2Name = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + + var vmiErr error + linuxVMIName, vmiErr = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(vmiErr).NotTo(HaveOccurred(), "failed to get VMI name for display name %q in namespace %q", linuxImageDisplayName, input.WCPNamespaceName) vm1IP = "" vm2IP = "" secretName = fmt.Sprintf("%s-%s", "secret", capiutil.RandomString(4)) @@ -149,7 +154,7 @@ func VMVPCSpec(ctx context.Context, inputGetter func() VMVPCSpecInput) { v1a2vmParameters = manifestbuilders.VirtualMachineYaml{ Namespace: input.WCPNamespaceName, VMClassName: clusterResources.VMClassName, - ImageName: linuxImageDisplayName, + ImageName: linuxVMIName, StorageClassName: clusterResources.StorageClassName, ResourcePolicy: clusterResources.VMResourcePolicyName, PowerState: "PoweredOn", From 45508cbab894420e6c6383c53902187f453148c8 Mon Sep 17 00:00:00 2001 From: Faisal Abujabal Date: Tue, 12 May 2026 12:18:19 -0500 Subject: [PATCH 2/2] fix: Image cache wait and testbed setup --- hack/e2e/kms.sh | 69 ++++++++++++++----- hack/e2e/proxy.sh | 14 ++-- .../vmservice/lib/vmoperator/vmoperator.go | 31 +++++++++ test/e2e/vmservice/vmservice/util.go | 28 ++++++-- .../vmservice/vmservice/viadmin/registervm.go | 4 +- .../viadmin/virtualmachineclasses.go | 7 ++ .../virtualmachine/virtualmachinelcm.go | 1 + 7 files changed, 124 insertions(+), 30 deletions(-) diff --git a/hack/e2e/kms.sh b/hack/e2e/kms.sh index 7dd354707..0fb8cb13f 100755 --- a/hack/e2e/kms.sh +++ b/hack/e2e/kms.sh @@ -24,14 +24,17 @@ find_gateway_ip() { # vm == external-gateway # NSX: # vm == external-vm-gateway - vm=$(govc find / -type m -name external*gateway) + vm=$(govc find / -type m -name external*gateway 2>/dev/null || true) + if [ -z "$vm" ]; then + return 0 + fi - # Use grepcidr if available, otherwise fallback to grep for common management networks + # Use grepcidr if available, otherwise fallback to grep for common management networks. if command -v grepcidr >/dev/null 2>&1; then - govc vm.ip -a -v4 "$vm" | tr ',' '\n' | grepcidr "$mgmtCidr" + govc vm.ip -a -v4 "$vm" 2>/dev/null | tr ',' '\n' | grepcidr "$mgmtCidr" || true else - # Fallback: get first non-169.254.x.x IP (avoid link-local) - govc vm.ip -a -v4 "$vm" | tr ',' '\n' | grep -v "^169\.254\." | head -n1 + # Fallback: get first non-link-local 10.x IP (management network uses 10.0.0.0/8). + govc vm.ip -a -v4 "$vm" 2>/dev/null | tr ',' '\n' | grep -v "^169\.254\." | grep "^10\." | head -n1 || true fi } @@ -52,28 +55,56 @@ install() { setup "$2" || echo "KMS setup failed" } +# kms_is_green returns 0 if the named provider already exists and has +# OverallStatus == "green", 1 otherwise. Safe to call from multiple parallel +# containers because it is read-only. +kms_is_green() { + local name="$1" + local status + status=$(govc kms.ls -json "$name" 2>/dev/null \ + | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('OverallStatus',''))" \ + 2>/dev/null || true) + [ "${status}" = "green" ] +} + # See also: vCenter -> Configure -> Security -> Key Providers setup() { ip="$1" - name=gce2e-standard - if ! govc kms.ls "$name" 2> /dev/null ; then - govc kms.add -n pykmip -a "$ip" "$name" + # gce2e-standard requires a running pykmip server on the gateway VM. + # Only configure it when a gateway IP is available; skip silently otherwise. + if [ -n "${ip:-}" ]; then + name=gce2e-standard + if kms_is_green "$name"; then + echo "KMS provider ${name} already green, skipping setup" + else + if ! govc kms.ls "$name" 2> /dev/null ; then + govc kms.add -n pykmip -a "$ip" "$name" + fi + crt=$(cat "$crt_dir/pykmip-crt.pem") + key=$(cat "$crt_dir/pykmip-key.pem") + + # Note: using the same key pair for the server (pykmip) and client (vCenter) + govc kms.trust -server-cert "$crt" -client-cert "$crt" -client-key "$key" "$name" + fi + govc kms.ls "$name" + else + echo "Skipping gce2e-standard KMS setup: no gateway IP available" fi - crt=$(cat "$crt_dir/pykmip-crt.pem") - key=$(cat "$crt_dir/pykmip-key.pem") - - # Note: using the same key pair for the server (pykmip) and client (vCenter) - govc kms.trust -server-cert "$crt" -client-cert "$crt" -client-key "$key" "$name" - govc kms.ls "$name" + # gce2e-native is a vCenter-native key provider that does not need an external + # server. Configure it unconditionally so encryption tests can run even on + # testbeds that have no VDS gateway VM (e.g. NSX or minimal testbeds). name=gce2e-native - if ! govc kms.ls "$name" 2> /dev/null ; then - govc kms.add -tpm=false -N "$name" + if kms_is_green "$name"; then + echo "KMS provider ${name} already green, skipping setup" + else + if ! govc kms.ls "$name" 2> /dev/null ; then + govc kms.add -tpm=false -N "$name" + fi + # Take a backup (and throw it away), required to activate the provider. + govc kms.export -f /dev/null "$name" fi - # Take a backup (and throw it away), required to activate the provider - govc kms.export -f /dev/null "$name" - govc kms.ls "$name" } diff --git a/hack/e2e/proxy.sh b/hack/e2e/proxy.sh index 3f2bde837..5a0930888 100755 --- a/hack/e2e/proxy.sh +++ b/hack/e2e/proxy.sh @@ -15,7 +15,7 @@ set -x SCRIPT_DIR="$(dirname "${BASH_SOURCE}")" GOVC_INSECURE="${GOVC_INSECURE:-1}" GOVC_USERNAME="${GOVC_USERNAME:-administrator@vsphere.local}" -GOVC_PASSWORD="${GOVC_PASSWORD:-Admin!23}" +GOVC_PASSWORD="${GOVC_PASSWORD:-vmware}" GATEWAY_VM_USERNAME="${GATEWAY_VM_USERNAME:-root}" GATEWAY_VM_PASSWORD=${GATEWAY_VM_PASSWORD:-'vmware'} @@ -39,12 +39,16 @@ find_gateway_ip() { mgmtCidr=$2 GATEWAY_VM_PATH=$(find_gateway_vm_path) ips=$(GOVC_URL=$vc GOVC_INSECURE=$GOVC_INSECURE GOVC_USERNAME=$GOVC_USERNAME GOVC_PASSWORD=$GOVC_PASSWORD govc vm.ip -a "${GATEWAY_VM_PATH}") - # loop over all ips, only return the one starts with '10.' which is - # company-level routable ip + # loop over all ips, only return the one that is in the management CIDR IFS=',' read -ra ADDR <<< "$ips" for i in "${ADDR[@]}"; do - # ignore error when IP not in CIDR - echo "$i" | grepcidr "$mgmtCidr" || true + if command -v grepcidr >/dev/null 2>&1; then + echo "$i" | grepcidr "$mgmtCidr" || true + else + # fallback: accept any non-link-local IP that starts with a '10.' prefix + # (the management network uses 10.0.0.0/8 in standard testbeds) + echo "$i" | grep -v "^169\.254\." | grep "^10\." || true + fi done } diff --git a/test/e2e/vmservice/lib/vmoperator/vmoperator.go b/test/e2e/vmservice/lib/vmoperator/vmoperator.go index 8d369f40e..04a252d70 100644 --- a/test/e2e/vmservice/lib/vmoperator/vmoperator.go +++ b/test/e2e/vmservice/lib/vmoperator/vmoperator.go @@ -98,6 +98,37 @@ func WaitForVirtualMachineConditionCreated(ctx context.Context, config *config.E }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(BeTrue(), "Timed out waiting for VirtualMachine %s to be created", vmName) } +// WaitForVirtualMachineImageCacheReady waits for VirtualMachineConditionImageCacheReady to become True. +// This is needed for ISO-type images whose content library files must be cached on the datastore. +func WaitForVirtualMachineImageCacheReady(ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, ns, vmName string) { + By("Waiting for VirtualMachine image cache to be ready") + + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, client, ns, vmName) + if err != nil { + e2eframework.Logf("retry waiting for image cache: %v", err) + g.Expect(err).ToNot(HaveOccurred()) + return + } + + for _, cond := range vm.Status.Conditions { + if cond.Type == vmopv1a5.VirtualMachineConditionImageCacheReady { + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue), + "VirtualMachineConditionImageCacheReady is %s: %s", cond.Status, cond.Message) + return + } + } + + // Condition not yet present — keep waiting. + g.Expect(false).To(BeTrue(), + "VirtualMachineConditionImageCacheReady condition not yet present on VM %s/%s", + ns, vmName) + }, config.GetIntervals("default", "default/wait-virtual-machine-image-creation")...). + Should(Succeed(), "Timed out waiting for VirtualMachine %s/%s image cache to be ready", ns, vmName) +} + // Utility function to wait for the VM's Status.Class.Name to be updated. func WaitForVirtualMachineStatusClassUpdated(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName, className string) { Eventually(func(g Gomega) bool { diff --git a/test/e2e/vmservice/vmservice/util.go b/test/e2e/vmservice/vmservice/util.go index d7cf2001a..8316be385 100644 --- a/test/e2e/vmservice/vmservice/util.go +++ b/test/e2e/vmservice/vmservice/util.go @@ -131,7 +131,15 @@ func VerifyVMClassCreate(wcpClient wcp.WorkloadManagementAPI, createSpec wcp.VMC } func VerifyVMClassDeletion(wcpClient wcp.WorkloadManagementAPI, vmClassID string) { - Expect(wcpClient.DeleteVMClass(vmClassID)).To(Succeed()) + err := wcpClient.DeleteVMClass(vmClassID) + if err != nil { + var dcliErr wcp.DcliError + if errors.As(err, &dcliErr) && strings.Contains(dcliErr.Response(), VMClassNotFound) { + framework.Logf("VMClass %q already gone during deletion; ignoring", vmClassID) + return + } + Expect(err).ToNot(HaveOccurred(), "failed to delete VMClass %q", vmClassID) + } // Eventually VMClass should be deleted. Eventually(func(g Gomega) { @@ -910,9 +918,12 @@ func waitForBackupToComplete( // UnregisterPVCVolumes unregisters all PVCs in the provided list using CnsUnregisterVolume. // This simulates a backup/restore scenario where the VM on vCenter still has its disks, // but the Kubernetes PVC/PV objects are gone. +// An admin client is used for creating CnsUnregisterVolume objects because the regular +// supervisor-admin user lacks permission to create resources in the cns.vmware.com API group. func UnregisterPVCVolumes( ctx context.Context, svClusterClient ctrlclient.Client, + clusterProxy *common.VMServiceClusterProxy, namespace string, vmName string, pvcNames []string, @@ -920,6 +931,15 @@ func UnregisterPVCVolumes( ) { By("Unregister volumes using CnsUnregisterVolume for each PVC") + // Obtain an admin client to create CnsUnregisterVolume resources — the regular + // supervisor-admin kubeconfig does not have RBAC permission on cns.vmware.com. + adminProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + Expect(err).ToNot(HaveOccurred(), "failed to get admin cluster proxy for CnsUnregisterVolume creation") + defer adminProxy.Dispose(ctx) + + adminClient, err := adminProxy.GetAdminClient() + Expect(err).ToNot(HaveOccurred(), "failed to get admin client for CnsUnregisterVolume creation") + // Create CnsUnregisterVolume resource for each PVC for _, pvcName := range pvcNames { framework.Logf("Creating CnsUnregisterVolume for PVC: %s", pvcName) @@ -940,8 +960,8 @@ func UnregisterPVCVolumes( }, } - // Create the CnsUnregisterVolume resource - Expect(svClusterClient.Create(ctx, cnsUnregister)).To(Succeed(), "failed to create CnsUnregisterVolume for PVC %s", pvcName) + // Create the CnsUnregisterVolume resource using the admin client. + Expect(adminClient.Create(ctx, cnsUnregister)).To(Succeed(), "failed to create CnsUnregisterVolume for PVC %s", pvcName) framework.Logf("Successfully created CnsUnregisterVolume: %s for PVC: %s", unregisterName, pvcName) // Wait for the CnsUnregisterVolume status to show unregistered: true @@ -1019,7 +1039,7 @@ func DeleteVMResource( } // Unregister all PVCs using the helper function - UnregisterPVCVolumes(ctx, svClusterClient, vmNamespace, vmName, pvcNames, config) + UnregisterPVCVolumes(ctx, svClusterClient, clusterProxy, vmNamespace, vmName, pvcNames, config) By("Delete only the K8s VM (vSphere VM should remain with the paused annotation applied)") vmoperator.DeleteVirtualMachine(ctx, svClusterClient, vmNamespace, vmName) diff --git a/test/e2e/vmservice/vmservice/viadmin/registervm.go b/test/e2e/vmservice/vmservice/viadmin/registervm.go index cc2803cbe..35e8183ca 100644 --- a/test/e2e/vmservice/vmservice/viadmin/registervm.go +++ b/test/e2e/vmservice/vmservice/viadmin/registervm.go @@ -316,7 +316,7 @@ func VIAdminRegisterVMSpec(ctx context.Context, inputGetter func() VIAdminRegist } // Unregister all PVCs using the helper function - vmservice.UnregisterPVCVolumes(ctx, svClusterClient, input.WCPNamespaceName, vmName, pvcNames, config) + vmservice.UnregisterPVCVolumes(ctx, svClusterClient, clusterProxy, input.WCPNamespaceName, vmName, pvcNames, config) // reconfigBeforeRegister changes the VM's resource.yaml, backupVersion to the given value. reconfigBeforeRegister := func(value []string) { @@ -482,7 +482,7 @@ func VIAdminRegisterVMSpec(ctx context.Context, inputGetter func() VIAdminRegist } // Unregister all PVCs using the helper function - vmservice.UnregisterPVCVolumes(ctx, svClusterClient, input.WCPNamespaceName, vmName, pvcNames, config) + vmservice.UnregisterPVCVolumes(ctx, svClusterClient, clusterProxy, input.WCPNamespaceName, vmName, pvcNames, config) // reconfigBeforeRegister changes the VM's resource.yaml, backupVersion and PVC properties to the given value. reconfigBeforeRegister := func(value []string) { diff --git a/test/e2e/vmservice/vmservice/viadmin/virtualmachineclasses.go b/test/e2e/vmservice/vmservice/viadmin/virtualmachineclasses.go index 686657542..e63a4f751 100644 --- a/test/e2e/vmservice/vmservice/viadmin/virtualmachineclasses.go +++ b/test/e2e/vmservice/vmservice/viadmin/virtualmachineclasses.go @@ -6,6 +6,7 @@ package viadmin import ( "context" errpkg "errors" + "fmt" "slices" "time" @@ -13,6 +14,7 @@ import ( . "github.com/onsi/gomega" "github.com/vmware/govmomi/vim25/types" e2eframework "k8s.io/kubernetes/test/e2e/framework" + capiutil "sigs.k8s.io/cluster-api/util" "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/kubectl" "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" @@ -55,8 +57,13 @@ func VIAdminVMClassSpec(ctx context.Context, inputGetter func() VIAdminVMClassSp namespacedVMClassFSSEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSNamespacedVMClass")) vmImageRegistryEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMImageRegistry")) vmClassAsConfigDaynDateFssEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMClassAsConfigDaynDate")) + // Append a random suffix so parallel test runs on the same testbed do not + // collide on these cluster-scoped WCP VMClass resources. + runSuffix := capiutil.RandomString(4) createSpecE2eTestBestEffortSmall = vmservice.CreateSpecE2eTestBestEffortSmall() + createSpecE2eTestBestEffortSmall.ID = fmt.Sprintf("%s-%s", createSpecE2eTestBestEffortSmall.ID, runSuffix) createSpecE2eTestGuaranteedXSmall = vmservice.CreateSpecE2eTestGuaranteedXSmall() + createSpecE2eTestGuaranteedXSmall.ID = fmt.Sprintf("%s-%s", createSpecE2eTestGuaranteedXSmall.ID, runSuffix) vmservice.VerifyVMClassCreate(wcpClient, createSpecE2eTestBestEffortSmall, createSpecE2eTestBestEffortSmall) vmservice.VerifyVMClassCreate(wcpClient, createSpecE2eTestGuaranteedXSmall, createSpecE2eTestGuaranteedXSmall) diff --git a/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go b/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go index 9a6256c07..741fdeb34 100644 --- a/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go +++ b/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go @@ -676,6 +676,7 @@ func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create VM with CD-ROM:\n %s", string(vmYaml)) vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmoperator.WaitForVirtualMachineImageCacheReady(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOn") By("Verifying network provider CR has an IP assigned for the VM")