🐛fix: E2E Tests Various Fixes

.cursor/rules/e2e-testing.mdc

@@ -15,7 +15,7 @@ alwaysApply: false
 **Environment setup** (kubeconfig under `~/.kube/wcp-config`, vars exported — see README for `testbedInfo.json` or a remote URL):
 
 ```bash
-source ./hack/setup-testbed-env.sh <testbedInfo.json|URL> --e2e
+source ./hack/e2e/setup-testbed-env.sh <testbedInfo.json|URL> --e2e
 ```
 
 **Makefile targets** (from repo root):

.github/workflows/ci.yml

@@ -212,18 +212,27 @@ jobs:
     needs:
     - verify-go-modules
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - target: image-build
+          dir: .
+        - target: e2e-image-build
+          dir: test/e2e
+    name: ${{ matrix.target }}
     steps:
     - name: Check out code
       uses: actions/checkout@v4
     - name: Install Go
       uses: actions/setup-go@v5
       with:
         go-version: ${{ env.GO_VERSION }}
-        go-version-file: 'go.mod'
+        go-version-file: '${{ matrix.dir }}/go.mod'
         cache: true
-        cache-dependency-path: '**/go.sum'
-    - name: Build Image
-      run: GOOS=linux GOARCH=amd64 make image-build
+        cache-dependency-path: '${{ matrix.dir }}/**/go.sum'
+    - name: Build ${{ matrix.target }}
+      run: GOOS=linux GOARCH=amd64 make ${{ matrix.target }}
 
   test:
     needs:

.gitignore

@@ -74,3 +74,4 @@ infrastructure-components.yaml
 test/e2e/**/test_logs
 e2e-tests
 kubectl
+test/e2e/**/test-results.xml

Dockerfile.e2e

@@ -59,9 +59,18 @@ RUN tdnf update -y && \
         glibc-devel \
         go \
         sudo \
+        unzip \
+        openssl \
         && \
     rm -rf /var/cache/tdnf
 
+# Install govc - required by hack/kms.sh and hack/proxy.sh for gateway VM discovery and KMS setup.
+# The default URL points to the public GitHub release. This is configurable to support overrides
+# by the build system.
+ARG GOVC_DOWNLOAD_URL=https://github.com/vmware/govmomi/releases/download/v0.53.0/govc_Linux_x86_64.tar.gz
+RUN curl -fsSL "${GOVC_DOWNLOAD_URL}" | tar -xz -C /usr/local/bin govc && \
+    chmod +x /usr/local/bin/govc
+
 # kubectl is always staged into the build context as ./kubectl before docker build runs.
 # This is to support passing the kubectl binary from the build system.
 COPY kubectl /usr/local/bin/kubectl
@@ -91,11 +100,8 @@ RUN mkdir -p /vm-operator /tmp/go-cache && \
 
 WORKDIR /vm-operator
 
-COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/hack/setup-testbed-env.sh ./hack/setup-testbed-env.sh
-COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/hack/wait-for-artifact-dir.sh ./hack/wait-for-artifact-dir.sh
-RUN ln -sf /vm-operator/hack/setup-testbed-env.sh /usr/local/bin/setup-testbed-env && \
-    ln -sf /vm-operator/hack/wait-for-artifact-dir.sh /usr/local/bin/wait-for-artifact-dir && \
-    chmod +x /vm-operator/hack/wait-for-artifact-dir.sh
+COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/hack/e2e/ ./hack/e2e/
+RUN chmod +x /vm-operator/hack/e2e/*.sh
 
 COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/Makefile ./Makefile
 COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/e2e-tests ./e2e-tests

Makefile

@@ -1011,7 +1011,7 @@ test-e2e: ## Run e2e tests (auto-detect: prebuilt binary if available, else gink
 test-e2e-prebuilt: ## Run e2e tests using precompiled binary. Used by the E2E container image.
 	@test -x "$(E2E_PREBUILT_BINARY)" || { echo "error: $(E2E_PREBUILT_BINARY) missing or not executable. Run: cd test/e2e/vmservice && go test -c -o ../../../e2e-tests ."; exit 1; }
 	@echo "Running E2E tests (prebuilt $(E2E_PREBUILT_BINARY))..."
-	@$(eval GINKGO_ARGS := --ginkgo.v)
+	@$(eval GINKGO_ARGS := --ginkgo.v --ginkgo.junit-report=$(or $(E2E_ARTIFACT_FOLDER),.)/test-results.xml)
 	@$(eval E2E_ARGS := -e2e.e2e-config="$(ROOT_DIR)test/e2e/vmservice/config/wcp.yaml" -e2e.artifactFolder=$(or $(E2E_ARTIFACT_FOLDER),test_logs))
 	$(if $(TEST_FOCUS),$(eval GINKGO_ARGS += --ginkgo.focus="$(TEST_FOCUS)"))
 	$(if $(TEST_SKIP),$(eval GINKGO_ARGS += --ginkgo.skip="$(TEST_SKIP)"))
@@ -1024,7 +1024,7 @@ test-e2e-prebuilt: ## Run e2e tests using precompiled binary. Used by the E2E co
 test-e2e-ginkgo: | $(GINKGO)
 test-e2e-ginkgo: ## Run e2e tests using ginkgo CLI (compile + run)
 	@echo "Running E2E tests (ginkgo compile)..."
-	@$(eval GINKGO_ARGS := -v)
+	@$(eval GINKGO_ARGS := -v --junit-report=$(or $(E2E_ARTIFACT_FOLDER),.)/test-results.xml)
 	@$(eval E2E_ARGS := -e2e.e2e-config="$(ROOT_DIR)test/e2e/vmservice/config/wcp.yaml" -e2e.artifactFolder=$(or $(E2E_ARTIFACT_FOLDER),test_logs))
 	$(if $(TEST_FOCUS),$(eval GINKGO_ARGS += --focus="$(TEST_FOCUS)"))
 	$(if $(TEST_SKIP),$(eval GINKGO_ARGS += --skip="$(TEST_SKIP)"))

hack/e2e/install-squid.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -x
+
+# Function to ensure that the "squid" package is installed
+ensure_squid_package() {
+    if [ -f "/etc/os-release" ]; then
+        source "/etc/os-release"
+        if [[ $ID == "ubuntu" ]]; then
+            if [ "$(dpkg-query -W -f='${Status}' squid 2>/dev/null | grep -c "ok installed")" -eq 0 ]; then
+                echo "Installing Squid package..."
+                dpkg --configure -a # to fix the imporoperly configured packages in the vm (as left in the appliance image) 
+                apt-get install -y squid
+            else
+                echo "Squid package is already installed."
+            fi
+        elif [[ $ID == "photon" ]]; then
+            if ! rpm -q squid >/dev/null 2>&1; then
+                echo "Installing Squid package..."
+                tdnf install -y squid --nogpgcheck
+            else
+                echo "Squid package is already installed."
+            fi
+        else
+            echo "Unsupported operating system. Exiting..."
+            exit 1
+        fi
+    else
+        echo "Unable to detect operating system. Exiting..."
+        exit 1
+    fi
+}
+
+# Function to interact with the squid service
+squid_service_action() {
+    echo "${1}ing Squid service..."
+    systemctl ${1} squid
+}
+
+usage() {
+	echo "./install-squid.sh [install|start|restart|stop]"
+	exit 1
+}
+
+main() {
+	if [ -z "$1" ]; then
+		usage
+	fi
+	case $1 in
+		"install")
+            ensure_squid_package
+			;;
+		"start"|"stop"|"restart")
+			squid_service_action $1
+			;;
+		*)
+			usage
+			;;
+	esac
+}
+
+main "$@"

hack/e2e/kms.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+
+# This script installs and configures a Native and Standard (PyKMIP) Key Provider for use by vCenter.
+# To run directly (default vds password below, nsx password is generated):
+# % export GOVC_URL="Administrator@vsphere.local:${vc_pass}@${vc_host}"
+# % GATEWAY_VM_PASSWORD=vmware ./hack/kms.sh install
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -x
+
+export GOVC_URL # set in main()
+export GOVC_INSECURE=true
+GATEWAY_VM_USERNAME="${GATEWAY_VM_USERNAME:-root}"
+GATEWAY_VM_PASSWORD="${GATEWAY_VM_PASSWORD:-vmware}"
+script_dir="$(dirname "$0")"
+crt_dir="$script_dir/tools/bin"
+
+find_gateway_ip() {
+  mgmtCidr="$1"
+
+  # VDS:
+  #  vm == external-gateway
+  # NSX:
+  #  vm == external-vm-gateway
+  vm=$(govc find / -type m -name external*gateway 2>/dev/null || true)
+  if [ -z "$vm" ]; then
+    return 0
+  fi
+
+  # Use grepcidr if available, otherwise fallback to grep for common management networks.
+  if command -v grepcidr >/dev/null 2>&1; then
+    govc vm.ip -a -v4 "$vm" 2>/dev/null | tr ',' '\n' | grepcidr "$mgmtCidr" || true
+  else
+    # Fallback: get first non-link-local 10.x IP (management network uses 10.0.0.0/8).
+    govc vm.ip -a -v4 "$vm" 2>/dev/null | tr ',' '\n' | grep -v "^169\.254\." | grep "^10\." | head -n1 || true
+  fi
+}
+
+install() {
+  if [ ! -e "$crt_dir/pykmip-crt.pem" ] ; then
+    mkdir -p "$crt_dir"
+    openssl req -x509 -newkey rsa:4096 -sha256 -days 365 -nodes \
+            -subj "/C=US/ST=CA/L=PA/O=Broadcom/OU=VCF/CN=pykmip" \
+            -keyout "$crt_dir"/pykmip-key.pem -out "$crt_dir"/pykmip-crt.pem
+  fi
+
+  target="$1@$2"
+  password=$3
+
+  sshpass -p "$password" scp -o PubkeyAuthentication=no -o StrictHostKeyChecking=no "$crt_dir"/pykmip-*.pem "$script_dir"/install-pykmip.sh "$target":
+  sshpass -p "$password" ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no "$target" /bin/bash ./install-pykmip.sh
+
+  setup "$2" || echo "KMS setup failed"
+}
+
+# kms_is_green returns 0 if the named provider already exists and has
+# OverallStatus == "green", 1 otherwise.  Safe to call from multiple parallel
+# containers because it is read-only.
+kms_is_green() {
+  local name="$1"
+  local status
+  status=$(govc kms.ls -json "$name" 2>/dev/null \
+    | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('OverallStatus',''))" \
+    2>/dev/null || true)
+  [ "${status}" = "green" ]
+}
+
+# See also: vCenter -> Configure -> Security -> Key Providers
+setup() {
+  ip="$1"
+
+  # gce2e-standard requires a running pykmip server on the gateway VM.
+  # Only configure it when a gateway IP is available; skip silently otherwise.
+  if [ -n "${ip:-}" ]; then
+    name=gce2e-standard
+    if kms_is_green "$name"; then
+      echo "KMS provider ${name} already green, skipping setup"
+    else
+      if ! govc kms.ls "$name" 2> /dev/null ; then
+        govc kms.add -n pykmip -a "$ip" "$name"
+      fi
+      crt=$(cat "$crt_dir/pykmip-crt.pem")
+      key=$(cat "$crt_dir/pykmip-key.pem")
+
+      # Note: using the same key pair for the server (pykmip) and client (vCenter)
+      govc kms.trust -server-cert "$crt" -client-cert "$crt" -client-key "$key" "$name"
+    fi
+    govc kms.ls "$name"
+  else
+    echo "Skipping gce2e-standard KMS setup: no gateway IP available"
+  fi
+
+  # gce2e-native is a vCenter-native key provider that does not need an external
+  # server. Configure it unconditionally so encryption tests can run even on
+  # testbeds that have no VDS gateway VM (e.g. NSX or minimal testbeds).
+  name=gce2e-native
+  if kms_is_green "$name"; then
+    echo "KMS provider ${name} already green, skipping setup"
+  else
+    if ! govc kms.ls "$name" 2> /dev/null ; then
+      govc kms.add -tpm=false -N "$name"
+    fi
+    # Take a backup (and throw it away), required to activate the provider.
+    govc kms.export -f /dev/null "$name"
+  fi
+  govc kms.ls "$name"
+}
+
+main() {
+  if [ "$#" -ge 2 ]; then
+    GOVC_URL="$2"
+  fi
+  mgmtCidr='10.0.0.0/8'
+  if [ "$#" -ge 3 ]; then
+    mgmtCidr="$3"
+  fi
+
+  case $1 in
+    "install")
+      install "$GATEWAY_VM_USERNAME" "$(find_gateway_ip "$mgmtCidr")" "$GATEWAY_VM_PASSWORD"
+      ;;
+    "setup")
+      setup "$(find_gateway_ip "$mgmtCidr")"
+      ;;
+    *)
+      echo "unknown command: $1"
+      exit 1
+      ;;
+  esac
+}
+
+main "$@"