diff --git a/.cursor/rules/e2e-sync-with-changes.mdc b/.cursor/rules/e2e-sync-with-changes.mdc new file mode 100644 index 000000000..b2cbebb35 --- /dev/null +++ b/.cursor/rules/e2e-sync-with-changes.mdc @@ -0,0 +1,18 @@ +--- +description: Add, update, or delete E2E tests in the same change set as related product behavior +alwaysApply: true +--- + +# E2E tests stay aligned with code changes + +When changing VM Operator behavior that is observable on a WCP supervisor cluster (APIs, controllers, webhooks, defaults, validation, VM lifecycle or platform integration), **treat `test/e2e` as part of the same deliverable**: + +- **Add** E2E coverage when introducing or materially changing user-facing or cluster-observable behavior that this suite is meant to guard, or when fixing a gap a regression would have caught. +- **Update** existing specs when expectations, labels, status, or workflows under test change. +- **Delete** (or replace) scenarios and helpers when removing features or obsolete flows; drop fixtures and manifest patterns that become dead. + +Prefer extending existing suites under `test/e2e/vmservice/` (see `vmservice/virtualmachine/`), correct Ginkgo `Label(...)` usage, and shared helpers (`lib/vmoperator`, `manifestbuilders`, `infrastructure/vsphere`) per `test/e2e/README.md`. + +**Do not** land product-only changes that clearly require E2E updates without adjusting tests in the same effort (unless the team has explicitly scoped E2E as follow-up — then say so in the PR/summary). + +How to run and write tests (commands, labels, config): **`test/e2e/README.md`**. Cheatsheet when editing E2E files: **`.cursor/rules/e2e-testing.mdc`**. diff --git a/.cursor/rules/e2e-testing.mdc b/.cursor/rules/e2e-testing.mdc new file mode 100644 index 000000000..62d48548d --- /dev/null +++ b/.cursor/rules/e2e-testing.mdc @@ -0,0 +1,58 @@ +--- +description: Run and write VM Operator E2E tests (Ginkgo suite against real WCP/vSphere) +globs: "test/e2e/**/*.go" +alwaysApply: false +--- + +# E2E Tests (`test/e2e`) + +**Product + E2E together:** `.cursor/rules/e2e-sync-with-changes.mdc` (always on) — add, update, or delete E2E tests alongside related code changes. + +**Canonical documentation:** `test/e2e/README.md` — prerequisites, architecture, env vars, labels, debugging, and writing patterns. Prefer updating that README when workflows change; keep this rule as a short pointer plus the commands used most often. + +## Running tests + +**Environment setup** (kubeconfig under `~/.kube/wcp-config`, vars exported — see README for `testbedInfo.json` or a remote URL): + +```bash +source ./hack/setup-testbed-env.sh --e2e +``` + +**Makefile targets** (from repo root): + +| Goal | Command | +|------|---------| +| Full E2E | `make test-e2e` | +| Smoke | `make e2e-smoke` | +| Core | `make e2e-core` | +| Extended | `make e2e-extended` | + +**Prebuilt vs compile:** `make test-e2e` auto-detects `./e2e-tests` vs `test-e2e-ginkgo`. Prebuilt binaries need `-e2e.*` and `--ginkgo.*` in **one** argv block (no `--` separator); splitting with `--` drops Ginkgo filters. + +**Common filters** (passed through Makefile — see README for full list): + +```bash +make test-e2e TEST_FOCUS="PATTERN" +make test-e2e TEST_SKIP="a|b" +make test-e2e LABEL_FILTER="smoke && !encryption" +make test-e2e FLAKE_ATTEMPTS=3 +E2E_NAMESPACE=my-ns make e2e-smoke +make test-e2e GINKGO_ARGS="-v --trace" +``` + +Config file: `test/e2e/vmservice/config/wcp.yaml`. Env vars such as `NETWORK`, `STORAGE_CLASS`, `E2E_NAMESPACE`, `TEST_FOCUS`, etc. are documented in the README table. + +## Writing tests + +- Follow layouts and examples under `test/e2e/vmservice/` (e.g. `vmservice/virtualmachine/`). +- Use Ginkgo `Label(...)` with the project’s label scheme (`smoke`, `core-functional`, `extended-functional`, feature labels — see README). +- Prefer helpers in `lib/vmoperator`, `manifestbuilders`, and `infrastructure/vsphere` rather than ad-hoc API calls. +- Async behavior: `Eventually` / `Consistently`, not sleeps — same spirit as controller tests; details and snippets are in the README. + +### Structure and manifests + +- **Avoid inline closures** (`func(...) { ... }` assigned to a variable) inside `Describe` / `Context` unless they are clearly the smallest way to express a one-off and would not read cleaner as a few lines at the call site or a **package-level** helper in the same test package. Prefer inlining the steps in `It` / `BeforeEach`, or extending **`manifestbuilders`** / **`createPvcsFromSpec`**-style helpers so PVC and VM shapes stay consistent with the rest of the suite. +- **PVCs**: Build volume claims with `createPvcsFromSpec` (or the same `manifestbuilders.PVC` fields it fills) and **`manifestbuilders.GetPersistentVolumeClaimYaml`** when you need a **standalone** PVC document (e.g. apply the PVC before the VM so admission can resolve the claim). That keeps YAML aligned with `GetVirtualMachineYamlA5` / `v1a5singlevm.yaml.in`. +- **Apply path**: If `controller-runtime` `Create` on a PVC fails against CSI mutation webhooks (e.g. TLS timeout), use **`clusterProxy.ApplyWithArgs`** with the rendered YAML so the same path as `kubectl apply` is used. + +When adding new categories, labels, or Makefile variables, **update `test/e2e/README.md`**. diff --git a/.cursor/rules/testing-standards.mdc b/.cursor/rules/testing-standards.mdc index 0f51d9565..9ec18a94a 100644 --- a/.cursor/rules/testing-standards.mdc +++ b/.cursor/rules/testing-standards.mdc @@ -6,6 +6,9 @@ alwaysApply: false # Testing Standards +**E2E (real WCP/vSphere cluster):** Tests under `test/e2e/` follow a different layout and Makefile workflow than controller tests. Follow **e2e-sync-with-changes** to add/update/delete E2E tests with product work; use **e2e-testing** when editing `test/e2e/**/*.go`, and read **`test/e2e/README.md`** for setup, running, labels, and writing patterns. + + ## Test File Naming | Suffix | Purpose | Example | diff --git a/.cursor/rules/use-this-format-for-PR-description.mdc b/.cursor/rules/use-this-format-for-PR-description.mdc new file mode 100644 index 000000000..bd78657f3 --- /dev/null +++ b/.cursor/rules/use-this-format-for-PR-description.mdc @@ -0,0 +1,56 @@ +--- +description: +alwaysApply: true +--- + +generate md format. +format: + + +**What does this PR do, and why is it needed?** + + + + +**Which issue(s) is/are addressed by this PR?** *(optional, in `fixes #(, fixes #, ...)` format, will close the issue(s) when PR gets merged)*: + +Fixes # + + +**Are there any special notes for your reviewer**: + + + + +**Please add a release note if necessary**: + + + +```release-note + +``` diff --git a/.dockerignore b/.dockerignore index d874ad67c..eff42846f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,16 @@ *.tar +# Test binaries and build artifacts +*.test +**/*.test +# Build outputs +_output/ +# IDE and editor files +.vscode/ +.idea/ +*.swp +*.swo +# OS files +.DS_Store +Thumbs.db +# Git +.git/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39e0e2599..c43c99da0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,10 +195,7 @@ jobs: - name: Install Go uses: actions/setup-go@v5 with: - go-version: '1.26.2' # TODO (brito-rafa) For GO-2026-4865 - # GO-2026-4866, GO-2026-4869, GO-2026-4870 - # GO-2026-4946, GO-2026-4947 - # remove once internal builds use 1.26.2 + go-version: ${{ env.GO_VERSION }} go-version-file: 'go.mod' cache: true cache-dependency-path: '**/go.sum' diff --git a/.gitignore b/.gitignore index 3497af65e..235317c02 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,7 @@ infrastructure-components.yaml /local.envvars /vendor /kind-cluster-info-dump + +# E2E tests +test/e2e/**/test_logs +e2e-tests diff --git a/.golangci.yml b/.golangci.yml index cdc9de71c..90f760a46 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -382,3 +382,25 @@ linters: # Ignore different receiver names in conversion functions (e.g. src vs dst). - path: "_conversion.go" text: ".*receiver name" + + - path: "test/e2e" + linters: + - revive + - gosec + - prealloc + - wsl + - wsl_v5 + - gocritic + - gocyclo + - noinlineerr + - makezero + - modernize + - gocheckcompilerdirectives + - embeddedstructcheck + - importas + - ineffassign + - goconst + - depguard + - godoclint + # E2E tests use . imports for gomega or ginkgo. + - staticcheck diff --git a/Dockerfile.e2e b/Dockerfile.e2e new file mode 100644 index 000000000..87524419e --- /dev/null +++ b/Dockerfile.e2e @@ -0,0 +1,94 @@ +# VM Operator E2E Test Container +# Self-contained E2E testing environment + +ARG BASE_IMAGE=mirror.gcr.io/library/photon:5.0 + +# Build stage - compile tests +FROM ${BASE_IMAGE} AS builder + +# Build stage setup +WORKDIR /vm-operator + +# Copy all source code needed for compilation +COPY go.mod go.sum ./ +COPY api/ ./api/ +COPY external/ ./external/ +COPY pkg/ ./pkg/ +COPY test/e2e/ ./test/e2e/ +COPY hack/ ./hack/ +COPY Makefile ./ + +# Install Go dependencies +RUN tdnf install -y \ + build-essential \ + glibc-devel \ + go +RUN go mod download + +# Pre-compile test binaries for faster execution +RUN cd test/e2e/vmservice && go test -c -o /vm-operator/e2e-tests . + +# Runtime stage - minimal image with compiled tests +FROM ${BASE_IMAGE} AS runtime + +# Build information +ARG BUILD_BRANCH +ARG BUILD_COMMIT +ARG BUILD_NUMBER +ARG BUILD_VERSION + +# Image labels +LABEL branch="${BUILD_BRANCH}" \ + buildNumber="${BUILD_NUMBER}" \ + commit="${BUILD_COMMIT}" \ + name="VM Operator E2E Tests" \ + vendor="Broadcom" \ + version="${BUILD_VERSION}" + +# Install required packages +RUN tdnf update -y && \ + tdnf install -y \ + curl \ + jq \ + git \ + openssh \ + sshpass \ + bash \ + make \ + build-essential \ + glibc-devel \ + go \ + && \ + rm -rf /var/cache/tdnf + +ARG KUBECTL_VERSION=v1.32.0 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +RUN curl -fsSL "${KUBECTL_URL}" -o /usr/local/bin/kubectl && \ + chmod +x /usr/local/bin/kubectl + +# Create non-root user early +RUN groupadd -g 1000 vmoperator && \ + useradd -u 1000 -g vmoperator -s /bin/bash -m vmoperator + +# Create directories and set up convenience scripts +RUN mkdir -p /vm-operator /tmp/go-cache && \ + chown vmoperator:vmoperator /vm-operator /tmp/go-cache + +WORKDIR /vm-operator + +COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/hack/setup-testbed-env.sh ./hack/setup-testbed-env.sh +RUN ln -sf /vm-operator/hack/setup-testbed-env.sh /usr/local/bin/setup-testbed-env + +COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/Makefile ./Makefile +COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/e2e-tests ./e2e-tests +COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/test/e2e/vmservice/config/ ./test/e2e/vmservice/config/ +COPY --from=builder --chown=vmoperator:vmoperator /vm-operator/test/e2e/fixtures/ ./test/e2e/fixtures/ + +# Switch to non-root user for runtime +USER vmoperator + +# Set up Go environment for runtime +ENV GOCACHE="/tmp/go-cache" + +# Default command +CMD ["/bin/bash"] diff --git a/Makefile b/Makefile index f21439bf9..70f0774a6 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,11 @@ IMAGE ?= vmoperator-controller IMAGE_TAG ?= latest IMG ?= ${IMAGE}:${IMAGE_TAG} +# E2E test image configuration +E2E_BASE_IMAGE ?= mirror.gcr.io/library/photon:5.0 +E2E_IMAGE ?= vmoperator-e2e +E2E_IMG ?= ${E2E_IMAGE}:${IMAGE_TAG} + # Code coverage files COVERAGE_FILE ?= cover.out @@ -160,8 +165,8 @@ NET_OP_API_SLUG := github.com/vmware-tanzu/net-operator-api BUILD_TYPE ?= dev BUILD_NUMBER ?= 00000000 -BUILD_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) -BUILD_COMMIT ?= $(shell git rev-parse --short HEAD) +BUILD_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") +BUILD_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") # ex. 1.2.3+abcdefg+4.5.6+hijklmn ifeq (,$(strip $(PRDCT_VERSION))) @@ -937,3 +942,98 @@ verify-local-manifests: ## Verify the local manifests .PHONY: verify-wcp-manifests verify-wcp-manifests: ## Verify the WCP manifests VERIFY_MANIFESTS=true $(MAKE) deploy-wcp + +## -------------------------------------- +## E2E Tests +## -------------------------------------- + +.PHONY: e2e-image-build +e2e-image-build: GOOS=linux +e2e-image-build: ## Build E2E test container image + @echo "Building VM Operator E2E test image..." + $(CRI_BIN) build \ + -f Dockerfile.e2e \ + -t "$(E2E_IMAGE):$(IMAGE_TAG)" \ + -t "$(E2E_IMAGE):$(BUILD_NUMBER)" \ + -t "$(E2E_IMAGE):$(IMAGE_VERSION)" \ + --build-arg BUILD_BRANCH="$(BUILD_BRANCH)" \ + --build-arg BUILD_COMMIT="$(BUILD_COMMIT)" \ + --build-arg BUILD_NUMBER="$(BUILD_NUMBER)" \ + --build-arg BUILD_VERSION="$(BUILD_VERSION)" \ + --build-arg BASE_IMAGE="$(E2E_BASE_IMAGE)" \ + $(ADDITIONAL_CRI_BUILD_FLAGS) \ + . + @if [ -n "$(IMAGE_FILE)" ]; then \ + mkdir -p "$$(dirname "$(IMAGE_FILE)")"; \ + $(CRI_BIN) save "$(E2E_IMAGE):$(IMAGE_VERSION)" -o "$(IMAGE_FILE)"; \ + fi + @echo "✅ E2E image build complete: $(E2E_IMAGE):$(IMAGE_TAG)" + +.PHONY: e2e-image-push +e2e-image-push: ## Push E2E test container image + $(CRI_BIN) push $(E2E_IMG) + +.PHONY: e2e-image-remove +e2e-image-remove: ## Remove E2E test container image + @if [[ "`$(CRI_BIN) images -q $(E2E_IMG) 2>/dev/null`" != "" ]]; then \ + echo "Remove E2E test container $(E2E_IMG)"; \ + $(CRI_BIN) rmi $(E2E_IMG); \ + fi + +# E2E Test Environment Variables: +# E2E_NAMESPACE - Use specific namespace for tests (default: random) +# TEST_FOCUS - Ginkgo focus pattern to run specific tests +# TEST_SKIP - Ginkgo skip pattern to exclude tests +# LABEL_FILTER - Ginkgo label filter (e.g., "smoke", "!extended-functional") +# FLAKE_ATTEMPTS - Number of retry attempts for flaky tests +# E2E_PREBUILT_BINARY - Path to `go test -c` output (default: $(ROOT_DIR)e2e-tests) +# E2E_ARTIFACT_FOLDER - Directory for test artifacts/logs (default: test_logs) +# E2E_ARGS - Override all e2e binary arguments (e.g. from CI pipelines) + +E2E_PREBUILT_BINARY ?= $(ROOT_DIR)e2e-tests + +.PHONY: test-e2e +test-e2e: ## Run e2e tests (auto-detect: prebuilt binary if available, else ginkgo) + @if [ -x "$(E2E_PREBUILT_BINARY)" ]; then \ + $(MAKE) test-e2e-prebuilt; \ + else \ + $(MAKE) test-e2e-ginkgo; \ + fi + +.PHONY: test-e2e-prebuilt +test-e2e-prebuilt: ## Run e2e tests using precompiled binary. Used by the E2E container image. + @test -x "$(E2E_PREBUILT_BINARY)" || { echo "error: $(E2E_PREBUILT_BINARY) missing or not executable. Run: cd test/e2e/vmservice && go test -c -o ../../../e2e-tests ."; exit 1; } + @echo "Running E2E tests (prebuilt $(E2E_PREBUILT_BINARY))..." + @$(eval GINKGO_ARGS := --ginkgo.v) + @$(eval E2E_ARGS := -e2e.e2e-config="$(ROOT_DIR)test/e2e/vmservice/config/wcp.yaml" -e2e.artifactFolder=$(or $(E2E_ARTIFACT_FOLDER),test_logs)) + $(if $(TEST_FOCUS),$(eval GINKGO_ARGS += --ginkgo.focus="$(TEST_FOCUS)")) + $(if $(TEST_SKIP),$(eval GINKGO_ARGS += --ginkgo.skip="$(TEST_SKIP)")) + $(if $(LABEL_FILTER),$(eval GINKGO_ARGS += --ginkgo.label-filter="$(LABEL_FILTER)")) + $(if $(FLAKE_ATTEMPTS),$(eval GINKGO_ARGS += --ginkgo.flake-attempts=$(FLAKE_ATTEMPTS))) + $(if $(E2E_NAMESPACE),$(eval export E2E_NAMESPACE=$(E2E_NAMESPACE))) + $(E2E_PREBUILT_BINARY) $(E2E_ARGS) $(GINKGO_ARGS) + +.PHONY: test-e2e-ginkgo +test-e2e-ginkgo: | $(GINKGO) +test-e2e-ginkgo: ## Run e2e tests using ginkgo CLI (compile + run) + @echo "Running E2E tests (ginkgo compile)..." + @$(eval GINKGO_ARGS := -v) + @$(eval E2E_ARGS := -e2e.e2e-config="$(ROOT_DIR)test/e2e/vmservice/config/wcp.yaml" -e2e.artifactFolder=$(or $(E2E_ARTIFACT_FOLDER),test_logs)) + $(if $(TEST_FOCUS),$(eval GINKGO_ARGS += --focus="$(TEST_FOCUS)")) + $(if $(TEST_SKIP),$(eval GINKGO_ARGS += --skip="$(TEST_SKIP)")) + $(if $(LABEL_FILTER),$(eval GINKGO_ARGS += --label-filter="$(LABEL_FILTER)")) + $(if $(FLAKE_ATTEMPTS),$(eval GINKGO_ARGS += --flake-attempts=$(FLAKE_ATTEMPTS))) + $(if $(E2E_NAMESPACE),$(eval export E2E_NAMESPACE=$(E2E_NAMESPACE))) + $(GINKGO) $(GINKGO_ARGS) ./test/e2e/vmservice/... -- $(E2E_ARGS) + +.PHONY: e2e-smoke +e2e-smoke: ## Run e2e smoke tests + $(MAKE) test-e2e LABEL_FILTER="smoke" + +.PHONY: e2e-core +e2e-core: ## Run e2e core functional tests + $(MAKE) test-e2e LABEL_FILTER="!smoke && !extended-functional" + +.PHONY: e2e-extended +e2e-extended: ## Run e2e extended functional tests + $(MAKE) test-e2e LABEL_FILTER="extended-functional" \ No newline at end of file diff --git a/_typos.toml b/_typos.toml index 6e59d9c47..12d49e51c 100644 --- a/_typos.toml +++ b/_typos.toml @@ -1,6 +1,4 @@ # Copyright (c) 2025 Broadcom. All Rights Reserved. -# Broadcom Confidential. The term "Broadcom" refers to Broadcom Inc. -# and/or its subsidiaries. # Check common typos using the https://github.com/crate-ci/typos tool in GitHub actions # Use "typos ." in that directory or "typos " to check for typos locally. diff --git a/api/test/v1alpha5/virtualmachine_conversion_test.go b/api/test/v1alpha5/virtualmachine_conversion_test.go index 497db2b8e..be37a80c7 100644 --- a/api/test/v1alpha5/virtualmachine_conversion_test.go +++ b/api/test/v1alpha5/virtualmachine_conversion_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/gomega" "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/utils/ptr" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -91,6 +92,28 @@ func TestVirtualMachineConversion(t *testing.T) { }, }, }, + { + name: "spec.network.interfaces.ipamModes", + hub: &vmopv1.VirtualMachine{ + Spec: vmopv1.VirtualMachineSpec{ + Network: &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + IPAMModes: []corev1.IPFamily{ + corev1.IPv4Protocol, + corev1.IPv6Protocol, + }, + }, + { + Name: "eth1", + IPAMModes: []corev1.IPFamily{corev1.IPv6Protocol}, + }, + }, + }, + }, + }, + }, { name: "spec.advanced new fields", hub: &vmopv1.VirtualMachine{ @@ -110,13 +133,17 @@ func TestVirtualMachineConversion(t *testing.T) { }, }, { - name: "spec.network.interfaces new fields", + name: "spec.network.interfaces advanced nic fields", hub: &vmopv1.VirtualMachine{ Spec: vmopv1.VirtualMachineSpec{ Network: &vmopv1.VirtualMachineNetworkSpec{ Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ { - Name: "eth0", + Name: "eth0", + IPAMModes: []corev1.IPFamily{ + corev1.IPv4Protocol, + corev1.IPv6Protocol, + }, Type: vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, VNUMANodeID: ptr.To(int32(1)), VMXNet3: &vmopv1.VirtualMachineNetworkInterfaceVMXNet3Spec{ diff --git a/api/test/v1alpha5/virtualmachineservice_conversion_test.go b/api/test/v1alpha5/virtualmachineservice_conversion_test.go new file mode 100644 index 000000000..862a7984c --- /dev/null +++ b/api/test/v1alpha5/virtualmachineservice_conversion_test.go @@ -0,0 +1,45 @@ +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha5_test + +import ( + "testing" + + . "github.com/onsi/gomega" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" +) + +func TestVirtualMachineServiceConversion(t *testing.T) { + t.Run("hub-spoke-hub dual-stack fields", func(t *testing.T) { + policy := corev1.IPFamilyPolicyPreferDualStack + hub := &vmopv1.VirtualMachineService{ + Spec: vmopv1.VirtualMachineServiceSpec{ + Type: vmopv1.VirtualMachineServiceTypeLoadBalancer, + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}, + IPFamilyPolicy: &policy, + Ports: []vmopv1.VirtualMachineServicePort{{Name: "http", Protocol: "TCP", Port: 80, TargetPort: 8080}}, + Selector: map[string]string{"app": "test"}, + LoadBalancerIP: "", + LoadBalancerSourceRanges: nil, + ClusterIP: "", + ExternalName: "", + }, + } + + g := NewWithT(t) + after := &vmopv1.VirtualMachineService{} + spoke := &vmopv1a5.VirtualMachineService{} + + g.Expect(spoke.ConvertFrom(hub)).To(Succeed()) + g.Expect(spoke.ConvertTo(after)).To(Succeed()) + g.Expect(apiequality.Semantic.DeepEqual(hub, after)).To(BeTrue(), cmp.Diff(hub, after)) + }) +} diff --git a/api/v1alpha1/virtualmachineservice_conversion.go b/api/v1alpha1/virtualmachineservice_conversion.go index b9adf7120..dd0cc542a 100644 --- a/api/v1alpha1/virtualmachineservice_conversion.go +++ b/api/v1alpha1/virtualmachineservice_conversion.go @@ -5,31 +5,61 @@ package v1alpha1 import ( + apiconversion "k8s.io/apimachinery/pkg/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" - "github.com/vmware-tanzu/vm-operator/api/v1alpha6" + "github.com/vmware-tanzu/vm-operator/api/utilconversion" + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" ) +func Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha1_VirtualMachineServiceSpec( + in *vmopv1.VirtualMachineServiceSpec, out *VirtualMachineServiceSpec, s apiconversion.Scope) error { + return autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha1_VirtualMachineServiceSpec(in, out, s) +} + +func restoreV1alpha6VirtualMachineServiceIPFamilies(dst, restored *vmopv1.VirtualMachineService) { + dst.Spec.IPFamilies = restored.Spec.IPFamilies + dst.Spec.IPFamilyPolicy = restored.Spec.IPFamilyPolicy +} + // ConvertTo converts this VirtualMachineService to the Hub version. func (src *VirtualMachineService) ConvertTo(dstRaw ctrlconversion.Hub) error { - dst := dstRaw.(*v1alpha6.VirtualMachineService) - return Convert_v1alpha1_VirtualMachineService_To_v1alpha6_VirtualMachineService(src, dst, nil) + dst := dstRaw.(*vmopv1.VirtualMachineService) + if err := Convert_v1alpha1_VirtualMachineService_To_v1alpha6_VirtualMachineService(src, dst, nil); err != nil { + return err + } + + restored := &vmopv1.VirtualMachineService{} + ok, err := utilconversion.UnmarshalData(src, restored) + if err != nil { + return err + } + if !ok { + return nil + } + + restoreV1alpha6VirtualMachineServiceIPFamilies(dst, restored) + dst.Status = restored.Status + return nil } // ConvertFrom converts the hub version to this VirtualMachineService. func (dst *VirtualMachineService) ConvertFrom(srcRaw ctrlconversion.Hub) error { - src := srcRaw.(*v1alpha6.VirtualMachineService) - return Convert_v1alpha6_VirtualMachineService_To_v1alpha1_VirtualMachineService(src, dst, nil) + src := srcRaw.(*vmopv1.VirtualMachineService) + if err := Convert_v1alpha6_VirtualMachineService_To_v1alpha1_VirtualMachineService(src, dst, nil); err != nil { + return err + } + return utilconversion.MarshalData(src, dst) } // ConvertTo converts this VirtualMachineServiceList to the Hub version. func (src *VirtualMachineServiceList) ConvertTo(dstRaw ctrlconversion.Hub) error { - dst := dstRaw.(*v1alpha6.VirtualMachineServiceList) + dst := dstRaw.(*vmopv1.VirtualMachineServiceList) return Convert_v1alpha1_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServiceList(src, dst, nil) } // ConvertFrom converts the hub version to this VirtualMachineServiceList. func (dst *VirtualMachineServiceList) ConvertFrom(srcRaw ctrlconversion.Hub) error { - src := srcRaw.(*v1alpha6.VirtualMachineServiceList) + src := srcRaw.(*vmopv1.VirtualMachineServiceList) return Convert_v1alpha6_VirtualMachineServiceList_To_v1alpha1_VirtualMachineServiceList(src, dst, nil) } diff --git a/api/v1alpha1/zz_generated.conversion.go b/api/v1alpha1/zz_generated.conversion.go index 9f5f88f08..c99b4e20b 100644 --- a/api/v1alpha1/zz_generated.conversion.go +++ b/api/v1alpha1/zz_generated.conversion.go @@ -427,11 +427,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha6.VirtualMachineServiceSpec)(nil), (*VirtualMachineServiceSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha1_VirtualMachineServiceSpec(a.(*v1alpha6.VirtualMachineServiceSpec), b.(*VirtualMachineServiceSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*VirtualMachineServiceStatus)(nil), (*v1alpha6.VirtualMachineServiceStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_VirtualMachineServiceStatus_To_v1alpha6_VirtualMachineServiceStatus(a.(*VirtualMachineServiceStatus), b.(*v1alpha6.VirtualMachineServiceStatus), scope) }); err != nil { @@ -567,6 +562,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha6.VirtualMachineServiceSpec)(nil), (*VirtualMachineServiceSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha1_VirtualMachineServiceSpec(a.(*v1alpha6.VirtualMachineServiceSpec), b.(*VirtualMachineServiceSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha6.VirtualMachineSetResourcePolicySpec)(nil), (*VirtualMachineSetResourcePolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha6_VirtualMachineSetResourcePolicySpec_To_v1alpha1_VirtualMachineSetResourcePolicySpec(a.(*v1alpha6.VirtualMachineSetResourcePolicySpec), b.(*VirtualMachineSetResourcePolicySpec), scope) }); err != nil { @@ -1828,7 +1828,17 @@ func Convert_v1alpha6_VirtualMachineService_To_v1alpha1_VirtualMachineService(in func autoConvert_v1alpha1_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServiceList(in *VirtualMachineServiceList, out *v1alpha6.VirtualMachineServiceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1alpha6.VirtualMachineService)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1alpha6.VirtualMachineService, len(*in)) + for i := range *in { + if err := Convert_v1alpha1_VirtualMachineService_To_v1alpha6_VirtualMachineService(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -1839,7 +1849,17 @@ func Convert_v1alpha1_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServic func autoConvert_v1alpha6_VirtualMachineServiceList_To_v1alpha1_VirtualMachineServiceList(in *v1alpha6.VirtualMachineServiceList, out *VirtualMachineServiceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]VirtualMachineService)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VirtualMachineService, len(*in)) + for i := range *in { + if err := Convert_v1alpha6_VirtualMachineService_To_v1alpha1_VirtualMachineService(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -1898,14 +1918,11 @@ func autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha1_VirtualMachineSe out.LoadBalancerSourceRanges = *(*[]string)(unsafe.Pointer(&in.LoadBalancerSourceRanges)) out.ClusterIP = in.ClusterIP out.ExternalName = in.ExternalName + // WARNING: in.IPFamilies requires manual conversion: does not exist in peer-type + // WARNING: in.IPFamilyPolicy requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha1_VirtualMachineServiceSpec is an autogenerated conversion function. -func Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha1_VirtualMachineServiceSpec(in *v1alpha6.VirtualMachineServiceSpec, out *VirtualMachineServiceSpec, s conversion.Scope) error { - return autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha1_VirtualMachineServiceSpec(in, out, s) -} - func autoConvert_v1alpha1_VirtualMachineServiceStatus_To_v1alpha6_VirtualMachineServiceStatus(in *VirtualMachineServiceStatus, out *v1alpha6.VirtualMachineServiceStatus, s conversion.Scope) error { if err := Convert_v1alpha1_LoadBalancerStatus_To_v1alpha6_LoadBalancerStatus(&in.LoadBalancer, &out.LoadBalancer, s); err != nil { return err diff --git a/api/v1alpha2/virtualmachine_conversion.go b/api/v1alpha2/virtualmachine_conversion.go index e2f2adfcc..4536bb524 100644 --- a/api/v1alpha2/virtualmachine_conversion.go +++ b/api/v1alpha2/virtualmachine_conversion.go @@ -7,6 +7,7 @@ package v1alpha2 import ( "slices" + corev1 "k8s.io/api/core/v1" apiconversion "k8s.io/apimachinery/pkg/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -96,11 +97,11 @@ func restore_v1alpha6_VirtualMachineAdvancedProps(dst, src *vmopv1.VirtualMachin dst.Spec.Advanced.ExtraConfig = adv.ExtraConfig } -func restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, src *vmopv1.VirtualMachine) { +func restore_v1alpha6_VirtualMachineNetworkInterfaces(dst, src *vmopv1.VirtualMachine) { if src.Spec.Network == nil || len(src.Spec.Network.Interfaces) == 0 { return } - if dst.Spec.Network == nil { + if dst.Spec.Network == nil || len(dst.Spec.Network.Interfaces) == 0 { return } srcByName := make(map[string]*vmopv1.VirtualMachineNetworkInterfaceSpec, len(src.Spec.Network.Interfaces)) @@ -113,6 +114,7 @@ func restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, src *vmop continue } dstIface := &dst.Spec.Network.Interfaces[i] + dstIface.IPAMModes = append([]corev1.IPFamily(nil), srcIface.IPAMModes...) dstIface.Type = srcIface.Type dstIface.VNUMANodeID = srcIface.VNUMANodeID dstIface.VMXNet3 = srcIface.VMXNet3 @@ -531,7 +533,7 @@ func (src *VirtualMachine) ConvertTo(dstRaw ctrlconversion.Hub) error { restore_v1alpha6_VirtualMachineVolumeAttributesClassName(dst, restored) restore_v1alpha6_VirtualMachineNetworkVLANs(dst, restored) restore_v1alpha6_VirtualMachineAdvancedProps(dst, restored) - restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, restored) + restore_v1alpha6_VirtualMachineNetworkInterfaces(dst, restored) // END RESTORE diff --git a/api/v1alpha2/virtualmachineservice_conversion.go b/api/v1alpha2/virtualmachineservice_conversion.go index dc0217d43..d706929ac 100644 --- a/api/v1alpha2/virtualmachineservice_conversion.go +++ b/api/v1alpha2/virtualmachineservice_conversion.go @@ -5,21 +5,51 @@ package v1alpha2 import ( + apiconversion "k8s.io/apimachinery/pkg/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" + "github.com/vmware-tanzu/vm-operator/api/utilconversion" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" ) +func Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha2_VirtualMachineServiceSpec( + in *vmopv1.VirtualMachineServiceSpec, out *VirtualMachineServiceSpec, s apiconversion.Scope) error { + return autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha2_VirtualMachineServiceSpec(in, out, s) +} + +func restoreV1alpha6VirtualMachineServiceIPFamilies(dst, restored *vmopv1.VirtualMachineService) { + dst.Spec.IPFamilies = restored.Spec.IPFamilies + dst.Spec.IPFamilyPolicy = restored.Spec.IPFamilyPolicy +} + // ConvertTo converts this VirtualMachineService to the Hub version. func (src *VirtualMachineService) ConvertTo(dstRaw ctrlconversion.Hub) error { dst := dstRaw.(*vmopv1.VirtualMachineService) - return Convert_v1alpha2_VirtualMachineService_To_v1alpha6_VirtualMachineService(src, dst, nil) + if err := Convert_v1alpha2_VirtualMachineService_To_v1alpha6_VirtualMachineService(src, dst, nil); err != nil { + return err + } + + restored := &vmopv1.VirtualMachineService{} + ok, err := utilconversion.UnmarshalData(src, restored) + if err != nil { + return err + } + if !ok { + return nil + } + + restoreV1alpha6VirtualMachineServiceIPFamilies(dst, restored) + dst.Status = restored.Status + return nil } // ConvertFrom converts the hub version to this VirtualMachineService. func (dst *VirtualMachineService) ConvertFrom(srcRaw ctrlconversion.Hub) error { src := srcRaw.(*vmopv1.VirtualMachineService) - return Convert_v1alpha6_VirtualMachineService_To_v1alpha2_VirtualMachineService(src, dst, nil) + if err := Convert_v1alpha6_VirtualMachineService_To_v1alpha2_VirtualMachineService(src, dst, nil); err != nil { + return err + } + return utilconversion.MarshalData(src, dst) } // ConvertTo converts this VirtualMachineServiceList to the Hub version. diff --git a/api/v1alpha2/zz_generated.conversion.go b/api/v1alpha2/zz_generated.conversion.go index 60c2007e6..0d558d18d 100644 --- a/api/v1alpha2/zz_generated.conversion.go +++ b/api/v1alpha2/zz_generated.conversion.go @@ -861,11 +861,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha6.VirtualMachineServiceSpec)(nil), (*VirtualMachineServiceSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha2_VirtualMachineServiceSpec(a.(*v1alpha6.VirtualMachineServiceSpec), b.(*VirtualMachineServiceSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*VirtualMachineServiceStatus)(nil), (*v1alpha6.VirtualMachineServiceStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_VirtualMachineServiceStatus_To_v1alpha6_VirtualMachineServiceStatus(a.(*VirtualMachineServiceStatus), b.(*v1alpha6.VirtualMachineServiceStatus), scope) }); err != nil { @@ -1061,6 +1056,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha6.VirtualMachineServiceSpec)(nil), (*VirtualMachineServiceSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha2_VirtualMachineServiceSpec(a.(*v1alpha6.VirtualMachineServiceSpec), b.(*VirtualMachineServiceSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha6.VirtualMachineSpec)(nil), (*VirtualMachineSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha6_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec(a.(*v1alpha6.VirtualMachineSpec), b.(*VirtualMachineSpec), scope) }); err != nil { @@ -2966,6 +2966,7 @@ func autoConvert_v1alpha6_VirtualMachineNetworkInterfaceSpec_To_v1alpha2_Virtual // WARNING: in.VNUMANodeID requires manual conversion: does not exist in peer-type // WARNING: in.VMXNet3 requires manual conversion: does not exist in peer-type // WARNING: in.AdvancedProperties requires manual conversion: does not exist in peer-type + // WARNING: in.IPAMModes requires manual conversion: does not exist in peer-type return nil } @@ -3498,7 +3499,17 @@ func Convert_v1alpha6_VirtualMachineService_To_v1alpha2_VirtualMachineService(in func autoConvert_v1alpha2_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServiceList(in *VirtualMachineServiceList, out *v1alpha6.VirtualMachineServiceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1alpha6.VirtualMachineService)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1alpha6.VirtualMachineService, len(*in)) + for i := range *in { + if err := Convert_v1alpha2_VirtualMachineService_To_v1alpha6_VirtualMachineService(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -3509,7 +3520,17 @@ func Convert_v1alpha2_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServic func autoConvert_v1alpha6_VirtualMachineServiceList_To_v1alpha2_VirtualMachineServiceList(in *v1alpha6.VirtualMachineServiceList, out *VirtualMachineServiceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]VirtualMachineService)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VirtualMachineService, len(*in)) + for i := range *in { + if err := Convert_v1alpha6_VirtualMachineService_To_v1alpha2_VirtualMachineService(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -3568,14 +3589,11 @@ func autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha2_VirtualMachineSe out.LoadBalancerSourceRanges = *(*[]string)(unsafe.Pointer(&in.LoadBalancerSourceRanges)) out.ClusterIP = in.ClusterIP out.ExternalName = in.ExternalName + // WARNING: in.IPFamilies requires manual conversion: does not exist in peer-type + // WARNING: in.IPFamilyPolicy requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha2_VirtualMachineServiceSpec is an autogenerated conversion function. -func Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha2_VirtualMachineServiceSpec(in *v1alpha6.VirtualMachineServiceSpec, out *VirtualMachineServiceSpec, s conversion.Scope) error { - return autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha2_VirtualMachineServiceSpec(in, out, s) -} - func autoConvert_v1alpha2_VirtualMachineServiceStatus_To_v1alpha6_VirtualMachineServiceStatus(in *VirtualMachineServiceStatus, out *v1alpha6.VirtualMachineServiceStatus, s conversion.Scope) error { if err := Convert_v1alpha2_LoadBalancerStatus_To_v1alpha6_LoadBalancerStatus(&in.LoadBalancer, &out.LoadBalancer, s); err != nil { return err diff --git a/api/v1alpha3/virtualmachine_conversion.go b/api/v1alpha3/virtualmachine_conversion.go index 4045e8638..23f9cada8 100644 --- a/api/v1alpha3/virtualmachine_conversion.go +++ b/api/v1alpha3/virtualmachine_conversion.go @@ -7,6 +7,7 @@ package v1alpha3 import ( "slices" + corev1 "k8s.io/api/core/v1" apiconversion "k8s.io/apimachinery/pkg/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -155,11 +156,11 @@ func restore_v1alpha6_VirtualMachineAdvancedProps(dst, src *vmopv1.VirtualMachin dst.Spec.Advanced.ExtraConfig = adv.ExtraConfig } -func restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, src *vmopv1.VirtualMachine) { +func restore_v1alpha6_VirtualMachineNetworkInterfaces(dst, src *vmopv1.VirtualMachine) { if src.Spec.Network == nil || len(src.Spec.Network.Interfaces) == 0 { return } - if dst.Spec.Network == nil { + if dst.Spec.Network == nil || len(dst.Spec.Network.Interfaces) == 0 { return } srcByName := make(map[string]*vmopv1.VirtualMachineNetworkInterfaceSpec, len(src.Spec.Network.Interfaces)) @@ -172,6 +173,7 @@ func restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, src *vmop continue } dstIface := &dst.Spec.Network.Interfaces[i] + dstIface.IPAMModes = append([]corev1.IPFamily(nil), srcIface.IPAMModes...) dstIface.Type = srcIface.Type dstIface.VNUMANodeID = srcIface.VNUMANodeID dstIface.VMXNet3 = srcIface.VMXNet3 @@ -466,11 +468,10 @@ func (src *VirtualMachine) ConvertTo(dstRaw ctrlconversion.Hub) error { restore_v1alpha6_VirtualMachinePolicies(dst, restored) restore_v1alpha6_VirtualMachineCryptoVTPM(dst, restored) restore_v1alpha6_VirtualMachineAffinity(dst, restored) - restore_v1alpha6_VirtualMachineCryptoVTPM(dst, restored) restore_v1alpha6_VirtualMachineVolumeAttributesClassName(dst, restored) restore_v1alpha6_VirtualMachineNetworkVLANs(dst, restored) restore_v1alpha6_VirtualMachineAdvancedProps(dst, restored) - restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, restored) + restore_v1alpha6_VirtualMachineNetworkInterfaces(dst, restored) // END RESTORE diff --git a/api/v1alpha3/virtualmachineservice_conversion.go b/api/v1alpha3/virtualmachineservice_conversion.go index 2912fbfa0..19692cf5a 100644 --- a/api/v1alpha3/virtualmachineservice_conversion.go +++ b/api/v1alpha3/virtualmachineservice_conversion.go @@ -5,21 +5,51 @@ package v1alpha3 import ( + apiconversion "k8s.io/apimachinery/pkg/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" + "github.com/vmware-tanzu/vm-operator/api/utilconversion" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" ) +func Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha3_VirtualMachineServiceSpec( + in *vmopv1.VirtualMachineServiceSpec, out *VirtualMachineServiceSpec, s apiconversion.Scope) error { + return autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha3_VirtualMachineServiceSpec(in, out, s) +} + +func restoreV1alpha6VirtualMachineServiceIPFamilies(dst, restored *vmopv1.VirtualMachineService) { + dst.Spec.IPFamilies = restored.Spec.IPFamilies + dst.Spec.IPFamilyPolicy = restored.Spec.IPFamilyPolicy +} + // ConvertTo converts this VirtualMachineService to the Hub version. func (src *VirtualMachineService) ConvertTo(dstRaw ctrlconversion.Hub) error { dst := dstRaw.(*vmopv1.VirtualMachineService) - return Convert_v1alpha3_VirtualMachineService_To_v1alpha6_VirtualMachineService(src, dst, nil) + if err := Convert_v1alpha3_VirtualMachineService_To_v1alpha6_VirtualMachineService(src, dst, nil); err != nil { + return err + } + + restored := &vmopv1.VirtualMachineService{} + ok, err := utilconversion.UnmarshalData(src, restored) + if err != nil { + return err + } + if !ok { + return nil + } + + restoreV1alpha6VirtualMachineServiceIPFamilies(dst, restored) + dst.Status = restored.Status + return nil } // ConvertFrom converts the hub version to this VirtualMachineService. func (dst *VirtualMachineService) ConvertFrom(srcRaw ctrlconversion.Hub) error { src := srcRaw.(*vmopv1.VirtualMachineService) - return Convert_v1alpha6_VirtualMachineService_To_v1alpha3_VirtualMachineService(src, dst, nil) + if err := Convert_v1alpha6_VirtualMachineService_To_v1alpha3_VirtualMachineService(src, dst, nil); err != nil { + return err + } + return utilconversion.MarshalData(src, dst) } // ConvertTo converts this VirtualMachineServiceList to the Hub version. diff --git a/api/v1alpha3/zz_generated.conversion.go b/api/v1alpha3/zz_generated.conversion.go index e7dded3b9..0a1b4a695 100644 --- a/api/v1alpha3/zz_generated.conversion.go +++ b/api/v1alpha3/zz_generated.conversion.go @@ -1018,11 +1018,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha6.VirtualMachineServiceSpec)(nil), (*VirtualMachineServiceSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha3_VirtualMachineServiceSpec(a.(*v1alpha6.VirtualMachineServiceSpec), b.(*VirtualMachineServiceSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*VirtualMachineServiceStatus)(nil), (*v1alpha6.VirtualMachineServiceStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_VirtualMachineServiceStatus_To_v1alpha6_VirtualMachineServiceStatus(a.(*VirtualMachineServiceStatus), b.(*v1alpha6.VirtualMachineServiceStatus), scope) }); err != nil { @@ -1253,6 +1248,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha6.VirtualMachineServiceSpec)(nil), (*VirtualMachineServiceSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha3_VirtualMachineServiceSpec(a.(*v1alpha6.VirtualMachineServiceSpec), b.(*VirtualMachineServiceSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha6.VirtualMachineSpec)(nil), (*VirtualMachineSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha6_VirtualMachineSpec_To_v1alpha3_VirtualMachineSpec(a.(*v1alpha6.VirtualMachineSpec), b.(*VirtualMachineSpec), scope) }); err != nil { @@ -3433,6 +3433,7 @@ func autoConvert_v1alpha6_VirtualMachineNetworkInterfaceSpec_To_v1alpha3_Virtual // WARNING: in.VNUMANodeID requires manual conversion: does not exist in peer-type // WARNING: in.VMXNet3 requires manual conversion: does not exist in peer-type // WARNING: in.AdvancedProperties requires manual conversion: does not exist in peer-type + // WARNING: in.IPAMModes requires manual conversion: does not exist in peer-type return nil } @@ -4082,7 +4083,17 @@ func Convert_v1alpha6_VirtualMachineService_To_v1alpha3_VirtualMachineService(in func autoConvert_v1alpha3_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServiceList(in *VirtualMachineServiceList, out *v1alpha6.VirtualMachineServiceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1alpha6.VirtualMachineService)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1alpha6.VirtualMachineService, len(*in)) + for i := range *in { + if err := Convert_v1alpha3_VirtualMachineService_To_v1alpha6_VirtualMachineService(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -4093,7 +4104,17 @@ func Convert_v1alpha3_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServic func autoConvert_v1alpha6_VirtualMachineServiceList_To_v1alpha3_VirtualMachineServiceList(in *v1alpha6.VirtualMachineServiceList, out *VirtualMachineServiceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]VirtualMachineService)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VirtualMachineService, len(*in)) + for i := range *in { + if err := Convert_v1alpha6_VirtualMachineService_To_v1alpha3_VirtualMachineService(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -4152,14 +4173,11 @@ func autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha3_VirtualMachineSe out.LoadBalancerSourceRanges = *(*[]string)(unsafe.Pointer(&in.LoadBalancerSourceRanges)) out.ClusterIP = in.ClusterIP out.ExternalName = in.ExternalName + // WARNING: in.IPFamilies requires manual conversion: does not exist in peer-type + // WARNING: in.IPFamilyPolicy requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha3_VirtualMachineServiceSpec is an autogenerated conversion function. -func Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha3_VirtualMachineServiceSpec(in *v1alpha6.VirtualMachineServiceSpec, out *VirtualMachineServiceSpec, s conversion.Scope) error { - return autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha3_VirtualMachineServiceSpec(in, out, s) -} - func autoConvert_v1alpha3_VirtualMachineServiceStatus_To_v1alpha6_VirtualMachineServiceStatus(in *VirtualMachineServiceStatus, out *v1alpha6.VirtualMachineServiceStatus, s conversion.Scope) error { if err := Convert_v1alpha3_LoadBalancerStatus_To_v1alpha6_LoadBalancerStatus(&in.LoadBalancer, &out.LoadBalancer, s); err != nil { return err diff --git a/api/v1alpha4/virtualmachine_conversion.go b/api/v1alpha4/virtualmachine_conversion.go index 09af7bd2a..a870daaca 100644 --- a/api/v1alpha4/virtualmachine_conversion.go +++ b/api/v1alpha4/virtualmachine_conversion.go @@ -7,6 +7,7 @@ package v1alpha4 import ( "slices" + corev1 "k8s.io/api/core/v1" apiconversion "k8s.io/apimachinery/pkg/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -98,11 +99,11 @@ func restore_v1alpha6_VirtualMachineAdvancedProps(dst, src *vmopv1.VirtualMachin dst.Spec.Advanced.ExtraConfig = adv.ExtraConfig } -func restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, src *vmopv1.VirtualMachine) { +func restore_v1alpha6_VirtualMachineNetworkInterfaces(dst, src *vmopv1.VirtualMachine) { if src.Spec.Network == nil || len(src.Spec.Network.Interfaces) == 0 { return } - if dst.Spec.Network == nil { + if dst.Spec.Network == nil || len(dst.Spec.Network.Interfaces) == 0 { return } srcByName := make(map[string]*vmopv1.VirtualMachineNetworkInterfaceSpec, len(src.Spec.Network.Interfaces)) @@ -115,6 +116,7 @@ func restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, src *vmop continue } dstIface := &dst.Spec.Network.Interfaces[i] + dstIface.IPAMModes = append([]corev1.IPFamily(nil), srcIface.IPAMModes...) dstIface.Type = srcIface.Type dstIface.VNUMANodeID = srcIface.VNUMANodeID dstIface.VMXNet3 = srcIface.VMXNet3 @@ -422,7 +424,7 @@ func (src *VirtualMachine) ConvertTo(dstRaw ctrlconversion.Hub) error { restore_v1alpha6_VirtualMachineVolumeAttributesClassName(dst, restored) restore_v1alpha6_VirtualMachineNetworkVLANs(dst, restored) restore_v1alpha6_VirtualMachineAdvancedProps(dst, restored) - restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, restored) + restore_v1alpha6_VirtualMachineNetworkInterfaces(dst, restored) // END RESTORE diff --git a/api/v1alpha4/virtualmachineclassinstance_conversion.go b/api/v1alpha4/virtualmachineclassinstance_conversion.go new file mode 100644 index 000000000..c51542459 --- /dev/null +++ b/api/v1alpha4/virtualmachineclassinstance_conversion.go @@ -0,0 +1,23 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha4 + +import ( + ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" +) + +// ConvertTo converts this VirtualMachineClassInstance to the Hub version. +func (src *VirtualMachineClassInstance) ConvertTo(dstRaw ctrlconversion.Hub) error { + dst := dstRaw.(*vmopv1.VirtualMachineClassInstance) + return Convert_v1alpha4_VirtualMachineClassInstance_To_v1alpha6_VirtualMachineClassInstance(src, dst, nil) +} + +// ConvertFrom converts the hub version to this VirtualMachineClassInstance. +func (dst *VirtualMachineClassInstance) ConvertFrom(srcRaw ctrlconversion.Hub) error { + src := srcRaw.(*vmopv1.VirtualMachineClassInstance) + return Convert_v1alpha6_VirtualMachineClassInstance_To_v1alpha4_VirtualMachineClassInstance(src, dst, nil) +} diff --git a/api/v1alpha4/virtualmachineservice_conversion.go b/api/v1alpha4/virtualmachineservice_conversion.go index a6282c741..f8fcae3a7 100644 --- a/api/v1alpha4/virtualmachineservice_conversion.go +++ b/api/v1alpha4/virtualmachineservice_conversion.go @@ -5,19 +5,49 @@ package v1alpha4 import ( + apiconversion "k8s.io/apimachinery/pkg/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" + "github.com/vmware-tanzu/vm-operator/api/utilconversion" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" ) +func Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha4_VirtualMachineServiceSpec( + in *vmopv1.VirtualMachineServiceSpec, out *VirtualMachineServiceSpec, s apiconversion.Scope) error { + return autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha4_VirtualMachineServiceSpec(in, out, s) +} + +func restoreV1alpha6VirtualMachineServiceIPFamilies(dst, restored *vmopv1.VirtualMachineService) { + dst.Spec.IPFamilies = restored.Spec.IPFamilies + dst.Spec.IPFamilyPolicy = restored.Spec.IPFamilyPolicy +} + // ConvertTo converts this VirtualMachineService to the Hub version. func (src *VirtualMachineService) ConvertTo(dstRaw ctrlconversion.Hub) error { dst := dstRaw.(*vmopv1.VirtualMachineService) - return Convert_v1alpha4_VirtualMachineService_To_v1alpha6_VirtualMachineService(src, dst, nil) + if err := Convert_v1alpha4_VirtualMachineService_To_v1alpha6_VirtualMachineService(src, dst, nil); err != nil { + return err + } + + restored := &vmopv1.VirtualMachineService{} + ok, err := utilconversion.UnmarshalData(src, restored) + if err != nil { + return err + } + if !ok { + return nil + } + + restoreV1alpha6VirtualMachineServiceIPFamilies(dst, restored) + dst.Status = restored.Status + return nil } // ConvertFrom converts the hub version to this VirtualMachineService. func (dst *VirtualMachineService) ConvertFrom(srcRaw ctrlconversion.Hub) error { src := srcRaw.(*vmopv1.VirtualMachineService) - return Convert_v1alpha6_VirtualMachineService_To_v1alpha4_VirtualMachineService(src, dst, nil) + if err := Convert_v1alpha6_VirtualMachineService_To_v1alpha4_VirtualMachineService(src, dst, nil); err != nil { + return err + } + return utilconversion.MarshalData(src, dst) } diff --git a/api/v1alpha4/zz_generated.conversion.go b/api/v1alpha4/zz_generated.conversion.go index 4fba47448..cee816e1a 100644 --- a/api/v1alpha4/zz_generated.conversion.go +++ b/api/v1alpha4/zz_generated.conversion.go @@ -1058,11 +1058,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha6.VirtualMachineServiceSpec)(nil), (*VirtualMachineServiceSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha4_VirtualMachineServiceSpec(a.(*v1alpha6.VirtualMachineServiceSpec), b.(*VirtualMachineServiceSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*VirtualMachineServiceStatus)(nil), (*v1alpha6.VirtualMachineServiceStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_VirtualMachineServiceStatus_To_v1alpha6_VirtualMachineServiceStatus(a.(*VirtualMachineServiceStatus), b.(*v1alpha6.VirtualMachineServiceStatus), scope) }); err != nil { @@ -1318,6 +1313,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha6.VirtualMachineServiceSpec)(nil), (*VirtualMachineServiceSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha4_VirtualMachineServiceSpec(a.(*v1alpha6.VirtualMachineServiceSpec), b.(*VirtualMachineServiceSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha6.VirtualMachineSnapshotReference)(nil), (*common.LocalObjectRef)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha6_VirtualMachineSnapshotReference_To_common_LocalObjectRef(a.(*v1alpha6.VirtualMachineSnapshotReference), b.(*common.LocalObjectRef), scope) }); err != nil { @@ -3599,6 +3599,7 @@ func autoConvert_v1alpha6_VirtualMachineNetworkInterfaceSpec_To_v1alpha4_Virtual // WARNING: in.VNUMANodeID requires manual conversion: does not exist in peer-type // WARNING: in.VMXNet3 requires manual conversion: does not exist in peer-type // WARNING: in.AdvancedProperties requires manual conversion: does not exist in peer-type + // WARNING: in.IPAMModes requires manual conversion: does not exist in peer-type return nil } @@ -4248,7 +4249,17 @@ func Convert_v1alpha6_VirtualMachineService_To_v1alpha4_VirtualMachineService(in func autoConvert_v1alpha4_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServiceList(in *VirtualMachineServiceList, out *v1alpha6.VirtualMachineServiceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1alpha6.VirtualMachineService)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1alpha6.VirtualMachineService, len(*in)) + for i := range *in { + if err := Convert_v1alpha4_VirtualMachineService_To_v1alpha6_VirtualMachineService(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -4259,7 +4270,17 @@ func Convert_v1alpha4_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServic func autoConvert_v1alpha6_VirtualMachineServiceList_To_v1alpha4_VirtualMachineServiceList(in *v1alpha6.VirtualMachineServiceList, out *VirtualMachineServiceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]VirtualMachineService)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VirtualMachineService, len(*in)) + for i := range *in { + if err := Convert_v1alpha6_VirtualMachineService_To_v1alpha4_VirtualMachineService(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -4318,14 +4339,11 @@ func autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha4_VirtualMachineSe out.LoadBalancerSourceRanges = *(*[]string)(unsafe.Pointer(&in.LoadBalancerSourceRanges)) out.ClusterIP = in.ClusterIP out.ExternalName = in.ExternalName + // WARNING: in.IPFamilies requires manual conversion: does not exist in peer-type + // WARNING: in.IPFamilyPolicy requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha4_VirtualMachineServiceSpec is an autogenerated conversion function. -func Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha4_VirtualMachineServiceSpec(in *v1alpha6.VirtualMachineServiceSpec, out *VirtualMachineServiceSpec, s conversion.Scope) error { - return autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha4_VirtualMachineServiceSpec(in, out, s) -} - func autoConvert_v1alpha4_VirtualMachineServiceStatus_To_v1alpha6_VirtualMachineServiceStatus(in *VirtualMachineServiceStatus, out *v1alpha6.VirtualMachineServiceStatus, s conversion.Scope) error { if err := Convert_v1alpha4_LoadBalancerStatus_To_v1alpha6_LoadBalancerStatus(&in.LoadBalancer, &out.LoadBalancer, s); err != nil { return err diff --git a/api/v1alpha5/virtualmachine_conversion.go b/api/v1alpha5/virtualmachine_conversion.go index 7163378bb..c8c7b237a 100644 --- a/api/v1alpha5/virtualmachine_conversion.go +++ b/api/v1alpha5/virtualmachine_conversion.go @@ -5,6 +5,7 @@ package v1alpha5 import ( + corev1 "k8s.io/api/core/v1" apiconversion "k8s.io/apimachinery/pkg/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -60,11 +61,11 @@ func restore_v1alpha6_VirtualMachineAdvancedProps(dst, src *vmopv1.VirtualMachin dst.Spec.Advanced.ExtraConfig = adv.ExtraConfig } -func restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, src *vmopv1.VirtualMachine) { +func restore_v1alpha6_VirtualMachineNetworkInterfaces(dst, src *vmopv1.VirtualMachine) { if src.Spec.Network == nil || len(src.Spec.Network.Interfaces) == 0 { return } - if dst.Spec.Network == nil { + if dst.Spec.Network == nil || len(dst.Spec.Network.Interfaces) == 0 { return } srcByName := make(map[string]*vmopv1.VirtualMachineNetworkInterfaceSpec, len(src.Spec.Network.Interfaces)) @@ -77,6 +78,7 @@ func restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, src *vmop continue } dstIface := &dst.Spec.Network.Interfaces[i] + dstIface.IPAMModes = append([]corev1.IPFamily(nil), srcIface.IPAMModes...) dstIface.Type = srcIface.Type dstIface.VNUMANodeID = srcIface.VNUMANodeID dstIface.VMXNet3 = srcIface.VMXNet3 @@ -143,7 +145,7 @@ func (src *VirtualMachine) ConvertTo(dstRaw ctrlconversion.Hub) error { restore_v1alpha6_VirtualMachineVolumeAttributesClassName(dst, restored) restore_v1alpha6_VirtualMachineNetworkVLANs(dst, restored) restore_v1alpha6_VirtualMachineAdvancedProps(dst, restored) - restore_v1alpha6_VirtualMachineNetworkInterfaceAdvancedProps(dst, restored) + restore_v1alpha6_VirtualMachineNetworkInterfaces(dst, restored) // END RESTORE diff --git a/api/v1alpha5/virtualmachineclassinstance_conversion.go b/api/v1alpha5/virtualmachineclassinstance_conversion.go new file mode 100644 index 000000000..1a1024af9 --- /dev/null +++ b/api/v1alpha5/virtualmachineclassinstance_conversion.go @@ -0,0 +1,35 @@ +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha5 + +import ( + ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" +) + +// ConvertTo converts this VirtualMachineClassInstance to the Hub version. +func (src *VirtualMachineClassInstance) ConvertTo(dstRaw ctrlconversion.Hub) error { + dst := dstRaw.(*vmopv1.VirtualMachineClassInstance) + return Convert_v1alpha5_VirtualMachineClassInstance_To_v1alpha6_VirtualMachineClassInstance(src, dst, nil) +} + +// ConvertFrom converts the hub version to this VirtualMachineClassInstance. +func (dst *VirtualMachineClassInstance) ConvertFrom(srcRaw ctrlconversion.Hub) error { + src := srcRaw.(*vmopv1.VirtualMachineClassInstance) + return Convert_v1alpha6_VirtualMachineClassInstance_To_v1alpha5_VirtualMachineClassInstance(src, dst, nil) +} + +// ConvertTo converts this VirtualMachineClassInstanceList to the Hub version. +func (src *VirtualMachineClassInstanceList) ConvertTo(dstRaw ctrlconversion.Hub) error { + dst := dstRaw.(*vmopv1.VirtualMachineClassInstanceList) + return Convert_v1alpha5_VirtualMachineClassInstanceList_To_v1alpha6_VirtualMachineClassInstanceList(src, dst, nil) +} + +// ConvertFrom converts the hub version to this VirtualMachineClassInstanceList. +func (dst *VirtualMachineClassInstanceList) ConvertFrom(srcRaw ctrlconversion.Hub) error { + src := srcRaw.(*vmopv1.VirtualMachineClassInstanceList) + return Convert_v1alpha6_VirtualMachineClassInstanceList_To_v1alpha5_VirtualMachineClassInstanceList(src, dst, nil) +} diff --git a/api/v1alpha5/virtualmachinegroup_publishrequest_conversion.go b/api/v1alpha5/virtualmachinegroup_publishrequest_conversion.go new file mode 100644 index 000000000..8957aab36 --- /dev/null +++ b/api/v1alpha5/virtualmachinegroup_publishrequest_conversion.go @@ -0,0 +1,35 @@ +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha5 + +import ( + ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" +) + +// ConvertTo converts this VirtualMachineGroupPublishRequest to the Hub version. +func (src *VirtualMachineGroupPublishRequest) ConvertTo(dstRaw ctrlconversion.Hub) error { + dst := dstRaw.(*vmopv1.VirtualMachineGroupPublishRequest) + return Convert_v1alpha5_VirtualMachineGroupPublishRequest_To_v1alpha6_VirtualMachineGroupPublishRequest(src, dst, nil) +} + +// ConvertFrom converts the hub version to this VirtualMachineGroupPublishRequest. +func (dst *VirtualMachineGroupPublishRequest) ConvertFrom(srcRaw ctrlconversion.Hub) error { + src := srcRaw.(*vmopv1.VirtualMachineGroupPublishRequest) + return Convert_v1alpha6_VirtualMachineGroupPublishRequest_To_v1alpha5_VirtualMachineGroupPublishRequest(src, dst, nil) +} + +// ConvertTo converts this VirtualMachineGroupPublishRequestList to the Hub version. +func (src *VirtualMachineGroupPublishRequestList) ConvertTo(dstRaw ctrlconversion.Hub) error { + dst := dstRaw.(*vmopv1.VirtualMachineGroupPublishRequestList) + return Convert_v1alpha5_VirtualMachineGroupPublishRequestList_To_v1alpha6_VirtualMachineGroupPublishRequestList(src, dst, nil) +} + +// ConvertFrom converts the hub version to this VirtualMachineGroupPublishRequestList. +func (dst *VirtualMachineGroupPublishRequestList) ConvertFrom(srcRaw ctrlconversion.Hub) error { + src := srcRaw.(*vmopv1.VirtualMachineGroupPublishRequestList) + return Convert_v1alpha6_VirtualMachineGroupPublishRequestList_To_v1alpha5_VirtualMachineGroupPublishRequestList(src, dst, nil) +} diff --git a/api/v1alpha5/virtualmachineservice_conversion.go b/api/v1alpha5/virtualmachineservice_conversion.go index 9007d0a23..ce75142ed 100644 --- a/api/v1alpha5/virtualmachineservice_conversion.go +++ b/api/v1alpha5/virtualmachineservice_conversion.go @@ -5,21 +5,51 @@ package v1alpha5 import ( + apiconversion "k8s.io/apimachinery/pkg/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" + "github.com/vmware-tanzu/vm-operator/api/utilconversion" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" ) +func Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha5_VirtualMachineServiceSpec( + in *vmopv1.VirtualMachineServiceSpec, out *VirtualMachineServiceSpec, s apiconversion.Scope) error { + return autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha5_VirtualMachineServiceSpec(in, out, s) +} + +func restoreV1alpha6VirtualMachineServiceIPFamilies(dst, restored *vmopv1.VirtualMachineService) { + dst.Spec.IPFamilies = restored.Spec.IPFamilies + dst.Spec.IPFamilyPolicy = restored.Spec.IPFamilyPolicy +} + // ConvertTo converts this VirtualMachineService to the Hub version. func (src *VirtualMachineService) ConvertTo(dstRaw ctrlconversion.Hub) error { dst := dstRaw.(*vmopv1.VirtualMachineService) - return Convert_v1alpha5_VirtualMachineService_To_v1alpha6_VirtualMachineService(src, dst, nil) + if err := Convert_v1alpha5_VirtualMachineService_To_v1alpha6_VirtualMachineService(src, dst, nil); err != nil { + return err + } + + restored := &vmopv1.VirtualMachineService{} + ok, err := utilconversion.UnmarshalData(src, restored) + if err != nil { + return err + } + if !ok { + return nil + } + + restoreV1alpha6VirtualMachineServiceIPFamilies(dst, restored) + dst.Status = restored.Status + return nil } // ConvertFrom converts the hub version to this VirtualMachineService. func (dst *VirtualMachineService) ConvertFrom(srcRaw ctrlconversion.Hub) error { src := srcRaw.(*vmopv1.VirtualMachineService) - return Convert_v1alpha6_VirtualMachineService_To_v1alpha5_VirtualMachineService(src, dst, nil) + if err := Convert_v1alpha6_VirtualMachineService_To_v1alpha5_VirtualMachineService(src, dst, nil); err != nil { + return err + } + return utilconversion.MarshalData(src, dst) } // ConvertTo converts this VirtualMachineServiceList to the Hub version. diff --git a/api/v1alpha5/virtualmachinesnapshot_conversion.go b/api/v1alpha5/virtualmachinesnapshot_conversion.go new file mode 100644 index 000000000..728be93ad --- /dev/null +++ b/api/v1alpha5/virtualmachinesnapshot_conversion.go @@ -0,0 +1,35 @@ +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha5 + +import ( + ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" +) + +// ConvertTo converts this VirtualMachineSnapshot to the Hub version. +func (src *VirtualMachineSnapshot) ConvertTo(dstRaw ctrlconversion.Hub) error { + dst := dstRaw.(*vmopv1.VirtualMachineSnapshot) + return Convert_v1alpha5_VirtualMachineSnapshot_To_v1alpha6_VirtualMachineSnapshot(src, dst, nil) +} + +// ConvertFrom converts the hub version to this VirtualMachineSnapshot. +func (dst *VirtualMachineSnapshot) ConvertFrom(srcRaw ctrlconversion.Hub) error { + src := srcRaw.(*vmopv1.VirtualMachineSnapshot) + return Convert_v1alpha6_VirtualMachineSnapshot_To_v1alpha5_VirtualMachineSnapshot(src, dst, nil) +} + +// ConvertTo converts this VirtualMachineSnapshotList to the Hub version. +func (src *VirtualMachineSnapshotList) ConvertTo(dstRaw ctrlconversion.Hub) error { + dst := dstRaw.(*vmopv1.VirtualMachineSnapshotList) + return Convert_v1alpha5_VirtualMachineSnapshotList_To_v1alpha6_VirtualMachineSnapshotList(src, dst, nil) +} + +// ConvertFrom converts the hub version to this VirtualMachineSnapshotList. +func (dst *VirtualMachineSnapshotList) ConvertFrom(srcRaw ctrlconversion.Hub) error { + src := srcRaw.(*vmopv1.VirtualMachineSnapshotList) + return Convert_v1alpha6_VirtualMachineSnapshotList_To_v1alpha5_VirtualMachineSnapshotList(src, dst, nil) +} diff --git a/api/v1alpha5/zz_generated.conversion.go b/api/v1alpha5/zz_generated.conversion.go index b54db9fd0..e26efe985 100644 --- a/api/v1alpha5/zz_generated.conversion.go +++ b/api/v1alpha5/zz_generated.conversion.go @@ -1343,11 +1343,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha6.VirtualMachineServiceSpec)(nil), (*VirtualMachineServiceSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha5_VirtualMachineServiceSpec(a.(*v1alpha6.VirtualMachineServiceSpec), b.(*VirtualMachineServiceSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*VirtualMachineServiceStatus)(nil), (*v1alpha6.VirtualMachineServiceStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha5_VirtualMachineServiceStatus_To_v1alpha6_VirtualMachineServiceStatus(a.(*VirtualMachineServiceStatus), b.(*v1alpha6.VirtualMachineServiceStatus), scope) }); err != nil { @@ -1638,6 +1633,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha6.VirtualMachineServiceSpec)(nil), (*VirtualMachineServiceSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha5_VirtualMachineServiceSpec(a.(*v1alpha6.VirtualMachineServiceSpec), b.(*VirtualMachineServiceSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha6.VirtualMachineSpec)(nil), (*VirtualMachineSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha6_VirtualMachineSpec_To_v1alpha5_VirtualMachineSpec(a.(*v1alpha6.VirtualMachineSpec), b.(*VirtualMachineSpec), scope) }); err != nil { @@ -4411,6 +4411,7 @@ func autoConvert_v1alpha6_VirtualMachineNetworkInterfaceSpec_To_v1alpha5_Virtual // WARNING: in.VNUMANodeID requires manual conversion: does not exist in peer-type // WARNING: in.VMXNet3 requires manual conversion: does not exist in peer-type // WARNING: in.AdvancedProperties requires manual conversion: does not exist in peer-type + // WARNING: in.IPAMModes requires manual conversion: does not exist in peer-type return nil } @@ -5066,7 +5067,17 @@ func Convert_v1alpha6_VirtualMachineService_To_v1alpha5_VirtualMachineService(in func autoConvert_v1alpha5_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServiceList(in *VirtualMachineServiceList, out *v1alpha6.VirtualMachineServiceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1alpha6.VirtualMachineService)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1alpha6.VirtualMachineService, len(*in)) + for i := range *in { + if err := Convert_v1alpha5_VirtualMachineService_To_v1alpha6_VirtualMachineService(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -5077,7 +5088,17 @@ func Convert_v1alpha5_VirtualMachineServiceList_To_v1alpha6_VirtualMachineServic func autoConvert_v1alpha6_VirtualMachineServiceList_To_v1alpha5_VirtualMachineServiceList(in *v1alpha6.VirtualMachineServiceList, out *VirtualMachineServiceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]VirtualMachineService)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VirtualMachineService, len(*in)) + for i := range *in { + if err := Convert_v1alpha6_VirtualMachineService_To_v1alpha5_VirtualMachineService(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -5136,14 +5157,11 @@ func autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha5_VirtualMachineSe out.LoadBalancerSourceRanges = *(*[]string)(unsafe.Pointer(&in.LoadBalancerSourceRanges)) out.ClusterIP = in.ClusterIP out.ExternalName = in.ExternalName + // WARNING: in.IPFamilies requires manual conversion: does not exist in peer-type + // WARNING: in.IPFamilyPolicy requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha5_VirtualMachineServiceSpec is an autogenerated conversion function. -func Convert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha5_VirtualMachineServiceSpec(in *v1alpha6.VirtualMachineServiceSpec, out *VirtualMachineServiceSpec, s conversion.Scope) error { - return autoConvert_v1alpha6_VirtualMachineServiceSpec_To_v1alpha5_VirtualMachineServiceSpec(in, out, s) -} - func autoConvert_v1alpha5_VirtualMachineServiceStatus_To_v1alpha6_VirtualMachineServiceStatus(in *VirtualMachineServiceStatus, out *v1alpha6.VirtualMachineServiceStatus, s conversion.Scope) error { if err := Convert_v1alpha5_LoadBalancerStatus_To_v1alpha6_LoadBalancerStatus(&in.LoadBalancer, &out.LoadBalancer, s); err != nil { return err diff --git a/api/v1alpha6/virtualmachine_network_advanced_types.go b/api/v1alpha6/virtualmachine_network_advanced_types.go index d4d760443..0984d55f7 100644 --- a/api/v1alpha6/virtualmachine_network_advanced_types.go +++ b/api/v1alpha6/virtualmachine_network_advanced_types.go @@ -22,6 +22,9 @@ const ( // TxContextThreadingMode specifies the transmit context threading mode for a // VMXNet3 interface. // This is a "weak enum": constants are well-known values; the field accepts any string for forward compatibility. +// +// +kubebuilder:validation:MaxLength=50 +// +kubebuilder:validation:XValidation:rule="self.matches(\"^(PerDevice|PerVM|PerQueue|[1-9])$\")",message="must be PerDevice, PerVM, PerQueue, or a number (1-9)" type TxContextThreadingMode string const ( @@ -40,6 +43,8 @@ const ( // CoalescingScheme specifies the interrupt coalescing scheme for a VMXNet3 // interface. // This is a "weak enum": constants are well-known values; the field accepts any string for forward compatibility. +// +// +kubebuilder:validation:XValidation:rule="self == 'Disabled' || self == 'Adapt' || self == 'Static' || self == 'RateBasedCoalescing' || size(self) < 128",message="must be Disabled, Adapt, Static, RateBasedCoalescing, or any other string shorter than 128 characters" type CoalescingScheme string const ( @@ -54,7 +59,7 @@ const ( CoalescingSchemeAdapt CoalescingScheme = "Adapt" // CoalescingSchemeStatic queues a fixed number of packets before triggering - // an interrupt. CoalescingParams sets the Tx,Rx packet count (range 1-64, + // an interrupt. CoalescingParams sets the Tx,Rx packet queue limit (range 1-64, // default "64"). CoalescingSchemeStatic CoalescingScheme = "Static" @@ -66,6 +71,9 @@ const ( // PNICQueueFeature names one physical NIC queue offload feature for VMXNet3 // pnicFeatures. // This is a "weak enum": constants are well-known values; the field accepts any string for forward compatibility. +// +// +kubebuilder:validation:MaxLength=50 +// +kubebuilder:validation:XValidation:rule="self.matches(\"^(LargeReceiveOffload|ReceiveSideScaling|^[1-9][0-9]*)$\")",message="must be LargeReceiveOffload, ReceiveSideScaling, or a non-empty decimal digits string for VMX pNICFeatures bitmask values (powers of two such as 1, 2, 4, 8)" type PNICQueueFeature string const ( @@ -86,6 +94,8 @@ const ( // These fields are only valid when the interface Type is VMXNet3. The CRD // admission webhook rejects this struct when Type is set to an incompatible // value. +// +// +kubebuilder:validation:XValidation:rule="!has(self.coalescingParams) || size(self.coalescingParams) < 128",message="coalescingParams must have length < 128" type VirtualMachineNetworkInterfaceVMXNet3Spec struct { // +optional @@ -106,6 +116,8 @@ type VirtualMachineNetworkInterfaceVMXNet3Spec struct { // PerQueue gives 2-8 TX threads per vNIC queue (scheduler-determined); // recommended for 100G workloads combined with pnicFeatures including ReceiveSideScaling. // Visible in esxtop as NetWorld-Dev--Tx threads. + // Accepts known enum values (PerDevice, PerVM, PerQueue) or single digit integers (1-9) + // for direct VMX values. CtxPerDev *TxContextThreadingMode `json:"ctxPerDev,omitempty" vmx:"ethernet%d.ctxPerDev"` // +optional @@ -125,6 +137,7 @@ type VirtualMachineNetworkInterfaceVMXNet3Spec struct { UDPRSSEnabled *bool `json:"udpRSSEnabled,omitempty" vmx:"ethernet%d.udpRSS"` // +optional + // +kubebuilder:validation:MaxItems=16 // +listType=set // @@ -133,6 +146,8 @@ type VirtualMachineNetworkInterfaceVMXNet3Spec struct { // NIC RSS hardware queues. Typically set to ["ReceiveSideScaling"] alongside // ctxPerDev=PerQueue for maximum 100G throughput. Omitted or empty means no // extra pNIC queue features beyond defaults. + // Accepts known enum values (LargeReceiveOffload, ReceiveSideScaling) or integer strings + // representing powers of 2 (1,2,4,8,...) for direct VMX bitmask values. PNICFeatures []PNICQueueFeature `json:"pnicFeatures,omitempty" vmx:"ethernet%d.pnicfeatures"` // +optional @@ -140,14 +155,17 @@ type VirtualMachineNetworkInterfaceVMXNet3Spec struct { // CoalescingScheme sets the interrupt coalescing scheme for this interface. // Use CoalescingSchemeDisabled for latency-sensitive (LS=High) non-DPDK // workloads to minimise interrupt latency. + // Accepts known enum values (Disabled, Adapt, Static, RateBasedCoalescing) or any + // string with length < 128 for forward compatibility. CoalescingScheme *CoalescingScheme `json:"coalescingScheme,omitempty" vmx:"ethernet%d.coalescingScheme"` // +optional // CoalescingParams sets the coalescing parameter when coalescingScheme is // RateBasedCoalescing or Static. The format depends on the scheme: - // - RateBasedCoalescing: single integer string, interrupts/sec (100-100000, e.g. "4000") - // - Static: single integer string, packet queue limit (1-64, e.g. "64") + // - RateBasedCoalescing: integer string for interrupts/sec (e.g. "4000") + // - Static: integer string for packet queue limit (e.g. "64") // Ignored when coalescingScheme is Disabled or Adapt. + // Must be length < 128 and valid 32-bit unsigned integer for RateBasedCoalescing. CoalescingParams *string `json:"coalescingParams,omitempty" vmx:"ethernet%d.coalescingParams"` } diff --git a/api/v1alpha6/virtualmachine_network_interface_validation_test.go b/api/v1alpha6/virtualmachine_network_interface_validation_test.go deleted file mode 100644 index 972f724f3..000000000 --- a/api/v1alpha6/virtualmachine_network_interface_validation_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// © Broadcom. All Rights Reserved. -// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha6 - -import "testing" - -// vmxNet3TuningCEL holds the same boolean logic as the XValidation CEL on -// VirtualMachineNetworkInterfaceSpec (post-mutation object: type must be -// VMXNet3 whenever vmxnet3 is set). -func vmxNet3TuningCEL(iface VirtualMachineNetworkInterfaceSpec) bool { - if iface.VMXNet3 == nil { - return true - } - return iface.Type == VirtualMachineNetworkInterfaceTypeVMXNet3 -} - -func TestVMXNet3TuningCELMatchesXValidationRule(t *testing.T) { - t.Parallel() - vmx := &VirtualMachineNetworkInterfaceVMXNet3Spec{} - cases := []struct { - name string - iface VirtualMachineNetworkInterfaceSpec - ok bool - }{ - { - name: "no vmxnet3 block", - iface: VirtualMachineNetworkInterfaceSpec{ - Name: "eth0", - Type: VirtualMachineNetworkInterfaceTypeSRIOV, - }, - ok: true, - }, - { - name: "vmxnet3 with VMXNet3 type", - iface: VirtualMachineNetworkInterfaceSpec{ - Name: "eth0", - Type: VirtualMachineNetworkInterfaceTypeVMXNet3, - VMXNet3: vmx, - }, - ok: true, - }, - { - name: "vmxnet3 with SRIOV type", - iface: VirtualMachineNetworkInterfaceSpec{ - Name: "eth0", - Type: VirtualMachineNetworkInterfaceTypeSRIOV, - VMXNet3: vmx, - }, - ok: false, - }, - { - name: "vmxnet3 with empty type", - iface: VirtualMachineNetworkInterfaceSpec{ - Name: "eth0", - VMXNet3: vmx, - }, - ok: false, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - if got := vmxNet3TuningCEL(tc.iface); got != tc.ok { - t.Fatalf("vmxNet3TuningCEL(...) = %v, want %v", got, tc.ok) - } - }) - } -} diff --git a/api/v1alpha6/virtualmachine_network_types.go b/api/v1alpha6/virtualmachine_network_types.go index 892d1533a..3e67a3d8b 100644 --- a/api/v1alpha6/virtualmachine_network_types.go +++ b/api/v1alpha6/virtualmachine_network_types.go @@ -5,6 +5,7 @@ package v1alpha6 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" vmopv1common "github.com/vmware-tanzu/vm-operator/api/v1alpha6/common" @@ -26,6 +27,7 @@ type VirtualMachineNetworkRouteSpec struct { } // +kubebuilder:validation:XValidation:rule="!has(self.vmxnet3) || self.type == 'VMXNet3'",message="vmxnet3 tuning fields require interface type VMXNet3" +// +kubebuilder:validation:XValidation:rule="!has(self.ipamModes) || self.ipamModes.all(m, m == 'IPv4' || m == 'IPv6')",message="each ipamModes entry must be IPv4 or IPv6" // VirtualMachineNetworkInterfaceSpec describes the desired state of a VM's // network interface. @@ -223,6 +225,20 @@ type VirtualMachineNetworkInterfaceSpec struct { // The admission webhook rejects keys that duplicate a first-class field // above (e.g. "ctxPerDev" is rejected because VMXNet3.CtxPerDev exists). AdvancedProperties []vmopv1common.KeyValuePair `json:"advancedProperties,omitempty"` + + // +optional + // +listType=set + // +kubebuilder:validation:MaxItems=2 + + // IPAMModes requests which IP address families (IPv4 and/or IPv6) the network + // provider allocates for this interface. Allowed values are IPv4 and IPv6. + // Each family appears at most once; duplicate values are rejected by the API + // server; order does not change meaning—[IPv4, IPv6] and [IPv6, IPv4] are the + // same dual-stack request. + // + // When unset, the provider's default applies for which families are allocated. + // When set, the controller forwards the requested families to the provider for this interface. + IPAMModes []corev1.IPFamily `json:"ipamModes,omitempty"` } // VirtualMachineNetworkVLANSpec describes a VLAN sub-interface configuration. diff --git a/api/v1alpha6/virtualmachineclassinstance_conversion.go b/api/v1alpha6/virtualmachineclassinstance_conversion.go new file mode 100644 index 000000000..4fa33a803 --- /dev/null +++ b/api/v1alpha6/virtualmachineclassinstance_conversion.go @@ -0,0 +1,11 @@ +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha6 + +// Hub marks VirtualMachineClassInstance as a conversion hub. +func (*VirtualMachineClassInstance) Hub() {} + +// Hub marks VirtualMachineClassInstanceList as a conversion hub. +func (*VirtualMachineClassInstanceList) Hub() {} diff --git a/api/v1alpha6/virtualmachinegroup_publishrequest_conversion.go b/api/v1alpha6/virtualmachinegroup_publishrequest_conversion.go new file mode 100644 index 000000000..713636b6d --- /dev/null +++ b/api/v1alpha6/virtualmachinegroup_publishrequest_conversion.go @@ -0,0 +1,11 @@ +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha6 + +// Hub marks VirtualMachineGroupPublishRequest as a conversion hub. +func (*VirtualMachineGroupPublishRequest) Hub() {} + +// Hub marks VirtualMachineGroupPublishRequestList as a conversion hub. +func (*VirtualMachineGroupPublishRequestList) Hub() {} diff --git a/api/v1alpha6/virtualmachineservice_types.go b/api/v1alpha6/virtualmachineservice_types.go index 324ae7e31..681ec93ba 100644 --- a/api/v1alpha6/virtualmachineservice_types.go +++ b/api/v1alpha6/virtualmachineservice_types.go @@ -5,6 +5,7 @@ package v1alpha6 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -78,6 +79,10 @@ type LoadBalancerIngress struct { Hostname string `json:"hostname,omitempty"` } +// +kubebuilder:validation:XValidation:rule="self.type != 'ExternalName' || ((!has(self.ipFamilies) || size(self.ipFamilies) == 0) && !has(self.ipFamilyPolicy))",message="ipFamilies and ipFamilyPolicy may not be set when type is ExternalName" +// +kubebuilder:validation:XValidation:rule="!has(self.ipFamilies) || size(self.ipFamilies) < 2 || self.ipFamilies[0] != self.ipFamilies[1]",message="ipFamilies must not contain duplicate entries" +// +kubebuilder:validation:XValidation:rule="!has(self.ipFamilies) || self.ipFamilies.all(f, f == 'IPv4' || f == 'IPv6')",message="each ipFamilies entry must be IPv4 or IPv6" + // VirtualMachineServiceSpec defines the desired state of VirtualMachineService. type VirtualMachineServiceSpec struct { // Type specifies a desired VirtualMachineServiceType for this @@ -138,6 +143,40 @@ type VirtualMachineServiceSpec struct { // Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) // and requires Type to be ExternalName. ExternalName string `json:"externalName,omitempty"` + + // +listType=atomic + // +kubebuilder:validation:MaxItems=2 + // +optional + + // IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + // VirtualMachineService. Together with ipFamilyPolicy, it guides how the controller + // configures the Kubernetes Service. Cluster-level constraints (for example requesting IPv6 + // when no IPv6 Service range exists) usually surface as errors reconciling the child Service, + // not as rejection of the VirtualMachineService object at create time. + // This field is conditionally mutable: it allows + // for adding or removing a secondary IP family, but it does not allow + // changing the primary IP family of the VirtualMachineService. Valid values are "IPv4" + // and "IPv6". This field applies to types ClusterIP (including headless) and LoadBalancer. + // This field will be wiped when updating a VirtualMachineService to type ExternalName. + // + // This field may hold a maximum of two entries (dual-stack families, in + // either order). These families must correspond to the values of the + // clusterIPs field, if specified. Both clusterIPs and ipFamilies are + // governed by the ipFamilyPolicy field. + IPFamilies []corev1.IPFamily `json:"ipFamilies,omitempty"` + + // +optional + // +kubebuilder:validation:Enum=SingleStack;PreferDualStack;RequireDualStack + + // IPFamilyPolicy represents the dual-stack-ness requested or required by + // this VirtualMachineService. If there is no value provided, then this field will be set + // to SingleStack. VirtualMachineServices can be "SingleStack" (a single IP family), + // "PreferDualStack" (two IP families on dual-stack configured clusters or + // a single IP family on single-stack clusters), or "RequireDualStack" + // (two IP families on dual-stack configured clusters, otherwise fail). The + // ipFamilies and clusterIPs fields depend on the value of this field. This + // field will be wiped when updating a VirtualMachineService to type ExternalName. + IPFamilyPolicy *corev1.IPFamilyPolicy `json:"ipFamilyPolicy,omitempty"` } // VirtualMachineServiceStatus defines the observed state of diff --git a/api/v1alpha6/virtualmachinesnapshot_conversion.go b/api/v1alpha6/virtualmachinesnapshot_conversion.go new file mode 100644 index 000000000..878ee1331 --- /dev/null +++ b/api/v1alpha6/virtualmachinesnapshot_conversion.go @@ -0,0 +1,11 @@ +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha6 + +// Hub marks VirtualMachineSnapshot as a conversion hub. +func (*VirtualMachineSnapshot) Hub() {} + +// Hub marks VirtualMachineSnapshotList as a conversion hub. +func (*VirtualMachineSnapshotList) Hub() {} diff --git a/api/v1alpha6/zz_generated.deepcopy.go b/api/v1alpha6/zz_generated.deepcopy.go index 660295bed..4c761c550 100644 --- a/api/v1alpha6/zz_generated.deepcopy.go +++ b/api/v1alpha6/zz_generated.deepcopy.go @@ -13,6 +13,7 @@ import ( "github.com/vmware-tanzu/vm-operator/api/v1alpha6/cloudinit" "github.com/vmware-tanzu/vm-operator/api/v1alpha6/common" "github.com/vmware-tanzu/vm-operator/api/v1alpha6/sysprep" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -2559,6 +2560,11 @@ func (in *VirtualMachineNetworkInterfaceSpec) DeepCopyInto(out *VirtualMachineNe *out = make([]common.KeyValuePair, len(*in)) copy(*out, *in) } + if in.IPAMModes != nil { + in, out := &in.IPAMModes, &out.IPAMModes + *out = make([]corev1.IPFamily, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineNetworkInterfaceSpec. @@ -3249,6 +3255,16 @@ func (in *VirtualMachineServiceSpec) DeepCopyInto(out *VirtualMachineServiceSpec *out = make([]string, len(*in)) copy(*out, *in) } + if in.IPFamilies != nil { + in, out := &in.IPFamilies, &out.IPFamilies + *out = make([]corev1.IPFamily, len(*in)) + copy(*out, *in) + } + if in.IPFamilyPolicy != nil { + in, out := &in.IPFamilyPolicy, &out.IPFamilyPolicy + *out = new(corev1.IPFamilyPolicy) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineServiceSpec. diff --git a/cmd/web-console-validator/main.go b/cmd/web-console-validator/main.go index 3b44d74d9..15ffac50a 100644 --- a/cmd/web-console-validator/main.go +++ b/cmd/web-console-validator/main.go @@ -25,8 +25,9 @@ import ( ) var ( - defaultServerPort = 9868 - defaultServerPath = "/validate" + defaultServerPort = 9868 + defaultServerPath = "/validate" + defaultServerBindAddress = "" ) func init() { @@ -36,6 +37,9 @@ func init() { if v, err := strconv.Atoi(os.Getenv("SERVER_PORT")); err == nil { defaultServerPort = v } + if v := os.Getenv("SERVER_BIND_ADDRESS"); v != "" { + defaultServerBindAddress = v + } } func main() { @@ -57,6 +61,11 @@ func main() { defaultServerPath, "The pattern path to handle the web-console validation requests.", ) + serverBindAddress := flag.String( + "server-bind-address", + defaultServerBindAddress, + "The IP address to bind to.", + ) flag.Parse() @@ -84,8 +93,9 @@ func main() { os.Exit(1) } + addr := *serverBindAddress + ":" + strconv.Itoa(*serverPort) server, err := webconsolevalidation.NewServer( - ":"+strconv.Itoa(*serverPort), + addr, *serverPath, client, ) @@ -94,7 +104,7 @@ func main() { os.Exit(1) } - logger.Info("Starting the web-console validation server", "port", *serverPort, "path", *serverPath) + logger.Info("Starting the web-console validation server", "addr", addr, "path", *serverPath) if err := server.Run(); err != nil && !errors.Is(err, http.ErrServerClosed) { logger.Error(err, "Failed to run the web-console validation server") os.Exit(1) diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachinereplicasets.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachinereplicasets.yaml index 967c429fd..9b94d84ea 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachinereplicasets.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachinereplicasets.yaml @@ -10051,6 +10051,24 @@ spec: inside the guest, ex. dvd, cdrom, sda, etc. pattern: ^\w\w+$ type: string + ipamModes: + description: |- + IPAMModes requests which IP address families (IPv4 and/or IPv6) the network + provider allocates for this interface. Allowed values are IPv4 and IPv6. + Each family appears at most once; duplicate values are rejected by the API + server; order does not change meaning—[IPv4, IPv6] and [IPv6, IPv4] are the + same dual-stack request. + + When unset, the provider's default applies for which families are allocated. + When set, the controller forwards the requested families to the provider for this interface. + items: + description: |- + IPFamily represents the IP Family (IPv4 or IPv6). This type is used + to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + maxItems: 2 + type: array + x-kubernetes-list-type: set macAddr: description: |- MACAddr is the optional MAC address of this interface. @@ -10204,16 +10222,26 @@ spec: description: |- CoalescingParams sets the coalescing parameter when coalescingScheme is RateBasedCoalescing or Static. The format depends on the scheme: - - RateBasedCoalescing: single integer string, interrupts/sec (100-100000, e.g. "4000") - - Static: single integer string, packet queue limit (1-64, e.g. "64") + - RateBasedCoalescing: integer string for interrupts/sec (e.g. "4000") + - Static: integer string for packet queue limit (e.g. "64") Ignored when coalescingScheme is Disabled or Adapt. + Must be length < 128 and valid 32-bit unsigned integer for RateBasedCoalescing. type: string coalescingScheme: description: |- CoalescingScheme sets the interrupt coalescing scheme for this interface. Use CoalescingSchemeDisabled for latency-sensitive (LS=High) non-DPDK workloads to minimise interrupt latency. + Accepts known enum values (Disabled, Adapt, Static, RateBasedCoalescing) or any + string with length < 128 for forward compatibility. type: string + x-kubernetes-validations: + - message: must be Disabled, Adapt, Static, + RateBasedCoalescing, or any other string + shorter than 128 characters + rule: self == 'Disabled' || self == 'Adapt' + || self == 'Static' || self == 'RateBasedCoalescing' + || size(self) < 128 ctxPerDev: description: |- CtxPerDev sets the TX context threading mode for this interface. @@ -10222,7 +10250,14 @@ spec: PerQueue gives 2-8 TX threads per vNIC queue (scheduler-determined); recommended for 100G workloads combined with pnicFeatures including ReceiveSideScaling. Visible in esxtop as NetWorld-Dev--Tx threads. + Accepts known enum values (PerDevice, PerVM, PerQueue) or single digit integers (1-9) + for direct VMX values. + maxLength: 50 type: string + x-kubernetes-validations: + - message: must be PerDevice, PerVM, PerQueue, + or a number (1-9) + rule: self.matches("^(PerDevice|PerVM|PerQueue|[1-9])$") pnicFeatures: description: |- PNICFeatures lists physical NIC queue offload features to enable. The @@ -10230,12 +10265,22 @@ spec: NIC RSS hardware queues. Typically set to ["ReceiveSideScaling"] alongside ctxPerDev=PerQueue for maximum 100G throughput. Omitted or empty means no extra pNIC queue features beyond defaults. + Accepts known enum values (LargeReceiveOffload, ReceiveSideScaling) or integer strings + representing powers of 2 (1,2,4,8,...) for direct VMX bitmask values. items: description: |- PNICQueueFeature names one physical NIC queue offload feature for VMXNet3 pnicFeatures. This is a "weak enum": constants are well-known values; the field accepts any string for forward compatibility. + maxLength: 50 type: string + x-kubernetes-validations: + - message: must be LargeReceiveOffload, ReceiveSideScaling, + or a non-empty decimal digits string for + VMX pNICFeatures bitmask values (powers + of two such as 1, 2, 4, 8) + rule: self.matches("^(LargeReceiveOffload|ReceiveSideScaling|^[1-9][0-9]*)$") + maxItems: 16 type: array x-kubernetes-list-type: set rssOffloadEnabled: @@ -10263,6 +10308,10 @@ spec: full VM memory reservation, and VMXNet3 v7 guest driver. type: boolean type: object + x-kubernetes-validations: + - message: coalescingParams must have length < 128 + rule: '!has(self.coalescingParams) || size(self.coalescingParams) + < 128' required: - name type: object @@ -10270,6 +10319,9 @@ spec: - message: vmxnet3 tuning fields require interface type VMXNet3 rule: '!has(self.vmxnet3) || self.type == ''VMXNet3''' + - message: each ipamModes entry must be IPv4 or IPv6 + rule: '!has(self.ipamModes) || self.ipamModes.all(m, + m == ''IPv4'' || m == ''IPv6'')' maxItems: 10 type: array x-kubernetes-list-map-keys: diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml index bbaec7c83..d6e6944cf 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml @@ -15785,6 +15785,24 @@ spec: inside the guest, ex. dvd, cdrom, sda, etc. pattern: ^\w\w+$ type: string + ipamModes: + description: |- + IPAMModes requests which IP address families (IPv4 and/or IPv6) the network + provider allocates for this interface. Allowed values are IPv4 and IPv6. + Each family appears at most once; duplicate values are rejected by the API + server; order does not change meaning—[IPv4, IPv6] and [IPv6, IPv4] are the + same dual-stack request. + + When unset, the provider's default applies for which families are allocated. + When set, the controller forwards the requested families to the provider for this interface. + items: + description: |- + IPFamily represents the IP Family (IPv4 or IPv6). This type is used + to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + maxItems: 2 + type: array + x-kubernetes-list-type: set macAddr: description: |- MACAddr is the optional MAC address of this interface. @@ -15938,16 +15956,25 @@ spec: description: |- CoalescingParams sets the coalescing parameter when coalescingScheme is RateBasedCoalescing or Static. The format depends on the scheme: - - RateBasedCoalescing: single integer string, interrupts/sec (100-100000, e.g. "4000") - - Static: single integer string, packet queue limit (1-64, e.g. "64") + - RateBasedCoalescing: integer string for interrupts/sec (e.g. "4000") + - Static: integer string for packet queue limit (e.g. "64") Ignored when coalescingScheme is Disabled or Adapt. + Must be length < 128 and valid 32-bit unsigned integer for RateBasedCoalescing. type: string coalescingScheme: description: |- CoalescingScheme sets the interrupt coalescing scheme for this interface. Use CoalescingSchemeDisabled for latency-sensitive (LS=High) non-DPDK workloads to minimise interrupt latency. + Accepts known enum values (Disabled, Adapt, Static, RateBasedCoalescing) or any + string with length < 128 for forward compatibility. type: string + x-kubernetes-validations: + - message: must be Disabled, Adapt, Static, RateBasedCoalescing, + or any other string shorter than 128 characters + rule: self == 'Disabled' || self == 'Adapt' || self + == 'Static' || self == 'RateBasedCoalescing' || + size(self) < 128 ctxPerDev: description: |- CtxPerDev sets the TX context threading mode for this interface. @@ -15956,7 +15983,14 @@ spec: PerQueue gives 2-8 TX threads per vNIC queue (scheduler-determined); recommended for 100G workloads combined with pnicFeatures including ReceiveSideScaling. Visible in esxtop as NetWorld-Dev--Tx threads. + Accepts known enum values (PerDevice, PerVM, PerQueue) or single digit integers (1-9) + for direct VMX values. + maxLength: 50 type: string + x-kubernetes-validations: + - message: must be PerDevice, PerVM, PerQueue, or a + number (1-9) + rule: self.matches("^(PerDevice|PerVM|PerQueue|[1-9])$") pnicFeatures: description: |- PNICFeatures lists physical NIC queue offload features to enable. The @@ -15964,12 +15998,22 @@ spec: NIC RSS hardware queues. Typically set to ["ReceiveSideScaling"] alongside ctxPerDev=PerQueue for maximum 100G throughput. Omitted or empty means no extra pNIC queue features beyond defaults. + Accepts known enum values (LargeReceiveOffload, ReceiveSideScaling) or integer strings + representing powers of 2 (1,2,4,8,...) for direct VMX bitmask values. items: description: |- PNICQueueFeature names one physical NIC queue offload feature for VMXNet3 pnicFeatures. This is a "weak enum": constants are well-known values; the field accepts any string for forward compatibility. + maxLength: 50 type: string + x-kubernetes-validations: + - message: must be LargeReceiveOffload, ReceiveSideScaling, + or a non-empty decimal digits string for VMX pNICFeatures + bitmask values (powers of two such as 1, 2, 4, + 8) + rule: self.matches("^(LargeReceiveOffload|ReceiveSideScaling|^[1-9][0-9]*)$") + maxItems: 16 type: array x-kubernetes-list-type: set rssOffloadEnabled: @@ -15997,12 +16041,19 @@ spec: full VM memory reservation, and VMXNet3 v7 guest driver. type: boolean type: object + x-kubernetes-validations: + - message: coalescingParams must have length < 128 + rule: '!has(self.coalescingParams) || size(self.coalescingParams) + < 128' required: - name type: object x-kubernetes-validations: - message: vmxnet3 tuning fields require interface type VMXNet3 rule: '!has(self.vmxnet3) || self.type == ''VMXNet3''' + - message: each ipamModes entry must be IPv4 or IPv6 + rule: '!has(self.ipamModes) || self.ipamModes.all(m, m == + ''IPv4'' || m == ''IPv6'')' maxItems: 10 type: array x-kubernetes-list-map-keys: diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml index c9e30d81c..d1ff9c564 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml @@ -922,6 +922,46 @@ spec: Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName. type: string + ipFamilies: + description: |- + IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + VirtualMachineService. Together with ipFamilyPolicy, it guides how the controller + configures the Kubernetes Service. Cluster-level constraints (for example requesting IPv6 + when no IPv6 Service range exists) usually surface as errors reconciling the child Service, + not as rejection of the VirtualMachineService object at create time. + This field is conditionally mutable: it allows + for adding or removing a secondary IP family, but it does not allow + changing the primary IP family of the VirtualMachineService. Valid values are "IPv4" + and "IPv6". This field applies to types ClusterIP (including headless) and LoadBalancer. + This field will be wiped when updating a VirtualMachineService to type ExternalName. + + This field may hold a maximum of two entries (dual-stack families, in + either order). These families must correspond to the values of the + clusterIPs field, if specified. Both clusterIPs and ipFamilies are + governed by the ipFamilyPolicy field. + items: + description: |- + IPFamily represents the IP Family (IPv4 or IPv6). This type is used + to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + maxItems: 2 + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this VirtualMachineService. If there is no value provided, then this field will be set + to SingleStack. VirtualMachineServices can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a VirtualMachineService to type ExternalName. + enum: + - SingleStack + - PreferDualStack + - RequireDualStack + type: string loadBalancerIP: description: |- LoadBalancer will get created with the IP specified in this field. @@ -1000,6 +1040,16 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: ipFamilies and ipFamilyPolicy may not be set when type is ExternalName + rule: self.type != 'ExternalName' || ((!has(self.ipFamilies) || size(self.ipFamilies) + == 0) && !has(self.ipFamilyPolicy)) + - message: ipFamilies must not contain duplicate entries + rule: '!has(self.ipFamilies) || size(self.ipFamilies) < 2 || self.ipFamilies[0] + != self.ipFamilies[1]' + - message: each ipFamilies entry must be IPv4 or IPv6 + rule: '!has(self.ipFamilies) || self.ipFamilies.all(f, f == ''IPv4'' + || f == ''IPv6'')' status: description: |- VirtualMachineServiceStatus defines the observed state of diff --git a/config/crd/external-crds/netoperator.vmware.com_networkinterfaces.yaml b/config/crd/external-crds/netoperator.vmware.com_networkinterfaces.yaml index 2d59b1f7e..320b20ee8 100644 --- a/config/crd/external-crds/netoperator.vmware.com_networkinterfaces.yaml +++ b/config/crd/external-crds/netoperator.vmware.com_networkinterfaces.yaml @@ -48,6 +48,24 @@ spec: If this field is omitted, then it is up to the underlying network provider to surface any information in status.externalID. type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy specifies the IP family policy for this network interface. + Values: IPv4Only, IPv6Only, DualStack. + When set to IPv4Only, only an IPv4 address will be allocated. + When set to IPv6Only, only an IPv6 address will be allocated. + When set to DualStack, both IPv4 and IPv6 addresses will be allocated. + If not specified, the allocation is determined by the IP families available in the + IPPools referenced by the backing Network: if both IPv4 and IPv6 pools are present, + one address per IP family will be allocated (equivalent to DualStack); if only a + single IP family is available, only one address of that family will be allocated. + Users can discover the supported IP families by inspecting the SupportedIPFamilies + field on the Network object. + enum: + - IPv4Only + - IPv6Only + - DualStack + type: string networkName: description: NetworkName refers to a NetworkObject in the same namespace. type: string @@ -146,7 +164,7 @@ spec: type: string ipAssignmentMode: description: |- - IPAssignmentMode indicates how IP addresses are assigned to this interface. + IPAssignmentMode indicates how IPv4 addresses are assigned to this interface. When unset: - If IP is assigned, it is assumed to be NetworkInterfaceIPAssignmentModeStaticPool. - If IP is unassigned, it is assumed to be NetworkInterfaceIPAssignmentModeDHCP. @@ -170,8 +188,19 @@ spec: description: IPFamily specifies the IP family (IPv4 vs IPv6) the IP belongs to. type: string + prefix: + description: |- + Prefix is the prefix length for the IP address (e.g. 24 for a /24 IPv4 network, + 64 for a /64 IPv6 network). If set, this field takes precedence over SubnetMask + for both IPv4 and IPv6 addresses. + format: int32 + maximum: 128 + minimum: 0 + type: integer subnetMask: - description: SubnetMask setting. + description: |- + SubnetMask setting. + Deprecated: Use Prefix instead. If Prefix is set, SubnetMask is ignored. type: string required: - gateway @@ -180,6 +209,17 @@ spec: - subnetMask type: object type: array + ipv6AssignmentMode: + description: |- + IPv6AssignmentMode indicates how IPv6 addresses are assigned to this interface. + This field is independent of IPAssignmentMode, allowing different assignment modes for IPv4 + and IPv6 (e.g., IPv4 uses DHCP while IPv6 uses static pool). + When unset, defaults to IPAssignmentModeNone (IPv6 disabled) for backward compatibility with existing IPv4-only + deployments. + When set to NetworkInterfaceIPAssignmentModeStaticPool, indicates IPv6 is assigned from a static pool. + When set to NetworkInterfaceIPAssignmentModeDHCP, indicates IPv6 should be obtained via DHCPv6. + When set to NetworkInterfaceIPAssignmentModeNone, indicates no IPv6 assignment should be performed. + type: string macAddress: description: MacAddress setting for the network interface. type: string diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index b68d61844..c613f1116 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -13,6 +13,7 @@ spec: - name: manager args: - "--metrics-addr=:8443" + - "--profiler-address=127.0.0.1:8073" ports: - containerPort: 8443 name: metrics-server diff --git a/controllers/contentlibrary/utils/controller_builder.go b/controllers/contentlibrary/utils/controller_builder.go index fd3872980..576c635c1 100644 --- a/controllers/contentlibrary/utils/controller_builder.go +++ b/controllers/contentlibrary/utils/controller_builder.go @@ -61,7 +61,7 @@ func AddToManager( ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledItemTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, controlledItemTypeName, ) diff --git a/controllers/contentlibrary/utils/controller_builder_v1a2.go b/controllers/contentlibrary/utils/controller_builder_v1a2.go index 65773cc9f..dbc6b8713 100644 --- a/controllers/contentlibrary/utils/controller_builder_v1a2.go +++ b/controllers/contentlibrary/utils/controller_builder_v1a2.go @@ -55,7 +55,7 @@ func AddToManagerV1A2( ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledItemTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, controlledItemTypeName, ) diff --git a/controllers/infra/capability/configmap/configmap_capability_controller.go b/controllers/infra/capability/configmap/configmap_capability_controller.go index c6c3638ce..edb39b44e 100644 --- a/controllers/infra/capability/configmap/configmap_capability_controller.go +++ b/controllers/infra/capability/configmap/configmap_capability_controller.go @@ -53,7 +53,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err mgr.GetClient(), cache, ctrl.Log.WithName("controllers").WithName(controllerName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ) // This controller is also run on the non-leaders (webhooks) pods too diff --git a/controllers/infra/capability/crd/crd_capability_controller.go b/controllers/infra/capability/crd/crd_capability_controller.go index a66784642..986350e53 100644 --- a/controllers/infra/capability/crd/crd_capability_controller.go +++ b/controllers/infra/capability/crd/crd_capability_controller.go @@ -38,7 +38,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controllerName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ) return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/infra/configmap/infra_configmap_controller.go b/controllers/infra/configmap/infra_configmap_controller.go index 6ae42499f..dc399bde1 100644 --- a/controllers/infra/configmap/infra_configmap_controller.go +++ b/controllers/infra/configmap/infra_configmap_controller.go @@ -41,7 +41,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controllerName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.Namespace, ctx.VMProvider, ) diff --git a/controllers/infra/node/infra_node_controller.go b/controllers/infra/node/infra_node_controller.go index 567a13686..f721bcb98 100644 --- a/controllers/infra/node/infra_node_controller.go +++ b/controllers/infra/node/infra_node_controller.go @@ -41,7 +41,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controllerName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, ) diff --git a/controllers/infra/secret/infra_secret_controller.go b/controllers/infra/secret/infra_secret_controller.go index 67f367576..787994fc8 100644 --- a/controllers/infra/secret/infra_secret_controller.go +++ b/controllers/infra/secret/infra_secret_controller.go @@ -59,7 +59,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, cache, ctrl.Log.WithName("controllers").WithName(controllerName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, vcCredsKey, ) diff --git a/controllers/infra/validatingwebhookconfiguration/validatingwebhookconfiguration_controller.go b/controllers/infra/validatingwebhookconfiguration/validatingwebhookconfiguration_controller.go index f9fb33aeb..c7a831efc 100644 --- a/controllers/infra/validatingwebhookconfiguration/validatingwebhookconfiguration_controller.go +++ b/controllers/infra/validatingwebhookconfiguration/validatingwebhookconfiguration_controller.go @@ -46,7 +46,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ) c, err := controller.New(controllerNameShort, mgr, controller.Options{ diff --git a/controllers/infra/zone/zone_controller.go b/controllers/infra/zone/zone_controller.go index 18e2fab4f..9d452d057 100644 --- a/controllers/infra/zone/zone_controller.go +++ b/controllers/infra/zone/zone_controller.go @@ -48,7 +48,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ) return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/storage/storageclass/storageclass_controller.go b/controllers/storage/storageclass/storageclass_controller.go index 1d03a2523..a74ab7b8f 100644 --- a/controllers/storage/storageclass/storageclass_controller.go +++ b/controllers/storage/storageclass/storageclass_controller.go @@ -41,7 +41,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong))) + record.New(mgr.GetEventRecorder(controllerNameLong))) return ctrl.NewControllerManagedBy(mgr). For(controlledType). diff --git a/controllers/storage/storagepolicy/storagepolicy_controller.go b/controllers/storage/storagepolicy/storagepolicy_controller.go index a24afb507..a424355c0 100644 --- a/controllers/storage/storagepolicy/storagepolicy_controller.go +++ b/controllers/storage/storagepolicy/storagepolicy_controller.go @@ -39,7 +39,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider) return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/storage/storagepolicy/storagepolicy_controller_test.go b/controllers/storage/storagepolicy/storagepolicy_controller_test.go index 54e0263e9..fce084cac 100644 --- a/controllers/storage/storagepolicy/storagepolicy_controller_test.go +++ b/controllers/storage/storagepolicy/storagepolicy_controller_test.go @@ -14,7 +14,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - apirecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -123,7 +123,7 @@ var _ = Describe("Reconcile", func() { ctx, client, log.Log.WithName("storagepolicy"), - record.New(apirecord.NewFakeRecorder(100)), + record.New(events.NewFakeRecorder(100)), provider) }) diff --git a/controllers/storage/storagepolicyquota/storagepolicyquota_controller.go b/controllers/storage/storagepolicyquota/storagepolicyquota_controller.go index fc5b54180..9b13968ca 100644 --- a/controllers/storage/storagepolicyquota/storagepolicyquota_controller.go +++ b/controllers/storage/storagepolicyquota/storagepolicyquota_controller.go @@ -45,7 +45,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.Namespace, ) diff --git a/controllers/storage/volumeattributesclass/volumeattributesclass_controller.go b/controllers/storage/volumeattributesclass/volumeattributesclass_controller.go index 8af625c69..fcb04342f 100644 --- a/controllers/storage/volumeattributesclass/volumeattributesclass_controller.go +++ b/controllers/storage/volumeattributesclass/volumeattributesclass_controller.go @@ -41,7 +41,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong))) + record.New(mgr.GetEventRecorder(controllerNameLong))) return ctrl.NewControllerManagedBy(mgr). For(controlledType). diff --git a/controllers/virtualmachine/storagepolicyusage/storagepolicyusage_controller.go b/controllers/virtualmachine/storagepolicyusage/storagepolicyusage_controller.go index 03f798975..1c7a2d627 100644 --- a/controllers/virtualmachine/storagepolicyusage/storagepolicyusage_controller.go +++ b/controllers/virtualmachine/storagepolicyusage/storagepolicyusage_controller.go @@ -56,7 +56,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controllerName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ) c, err := controller.New(controllerName, mgr, controller.Options{ diff --git a/controllers/virtualmachine/virtualmachine/virtualmachine_controller.go b/controllers/virtualmachine/virtualmachine/virtualmachine_controller.go index 6a0ca734e..cf2d8001a 100644 --- a/controllers/virtualmachine/virtualmachine/virtualmachine_controller.go +++ b/controllers/virtualmachine/virtualmachine/virtualmachine_controller.go @@ -81,7 +81,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, proberManager) diff --git a/controllers/virtualmachine/volume/volume_controller.go b/controllers/virtualmachine/volume/volume_controller.go index 02cc49bf2..6b9b75a2c 100644 --- a/controllers/virtualmachine/volume/volume_controller.go +++ b/controllers/virtualmachine/volume/volume_controller.go @@ -67,7 +67,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName("volume"), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, ) diff --git a/controllers/virtualmachine/volumebatch/volumebatch_controller.go b/controllers/virtualmachine/volumebatch/volumebatch_controller.go index 5a9c38efc..4c05bc9fa 100644 --- a/controllers/virtualmachine/volumebatch/volumebatch_controller.go +++ b/controllers/virtualmachine/volumebatch/volumebatch_controller.go @@ -107,7 +107,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName("volumebatch"), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, ) diff --git a/controllers/virtualmachineclass/virtualmachineclass_controller.go b/controllers/virtualmachineclass/virtualmachineclass_controller.go index c89462c25..8ce0e560b 100644 --- a/controllers/virtualmachineclass/virtualmachineclass_controller.go +++ b/controllers/virtualmachineclass/virtualmachineclass_controller.go @@ -45,7 +45,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ) return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/virtualmachinegroup/virtualmachinegroup_controller.go b/controllers/virtualmachinegroup/virtualmachinegroup_controller.go index 584fc0a3a..8c85ecf75 100644 --- a/controllers/virtualmachinegroup/virtualmachinegroup_controller.go +++ b/controllers/virtualmachinegroup/virtualmachinegroup_controller.go @@ -60,7 +60,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err mgr.GetClient(), mgr.GetAPIReader(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, ) diff --git a/controllers/virtualmachinegrouppublishrequest/virtualmachinegrouppublishrequest_controller.go b/controllers/virtualmachinegrouppublishrequest/virtualmachinegrouppublishrequest_controller.go index 0f962e596..1dc53ffe8 100644 --- a/controllers/virtualmachinegrouppublishrequest/virtualmachinegrouppublishrequest_controller.go +++ b/controllers/virtualmachinegrouppublishrequest/virtualmachinegrouppublishrequest_controller.go @@ -56,7 +56,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err mgr.GetClient(), mgr.GetAPIReader(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, ) return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/virtualmachineimagecache/virtualmachineimagecache_controller.go b/controllers/virtualmachineimagecache/virtualmachineimagecache_controller.go index f10f69091..60044eb01 100644 --- a/controllers/virtualmachineimagecache/virtualmachineimagecache_controller.go +++ b/controllers/virtualmachineimagecache/virtualmachineimagecache_controller.go @@ -70,7 +70,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err Context: ctx, Client: mgr.GetClient(), Logger: ctx.Logger.WithName("controllers").WithName(controlledTypeName), - Recorder: record.New(mgr.GetEventRecorderFor(controllerNameLong)), + Recorder: record.New(mgr.GetEventRecorder(controllerNameLong)), VMProvider: ctx.VMProvider, newCLSProvdrFn: newContentLibraryProviderOrDefault(ctx), diff --git a/controllers/virtualmachineimagecache/virtualmachineimagecache_controller_test.go b/controllers/virtualmachineimagecache/virtualmachineimagecache_controller_test.go index 584fc65b1..248b46008 100644 --- a/controllers/virtualmachineimagecache/virtualmachineimagecache_controller_test.go +++ b/controllers/virtualmachineimagecache/virtualmachineimagecache_controller_test.go @@ -348,6 +348,8 @@ var _ = Describe( }) BeforeEach(func() { + provider.Lock() + defer provider.Unlock() provider.VSphereClientFn = func(ctx context.Context) (*vsclient.Client, error) { return vsclient.NewClient(ctx, vcSimCtx.VCClientConfig) } diff --git a/controllers/virtualmachinepublishrequest/virtualmachinepublishrequest_controller.go b/controllers/virtualmachinepublishrequest/virtualmachinepublishrequest_controller.go index 0f8e53e4b..6f759e113 100644 --- a/controllers/virtualmachinepublishrequest/virtualmachinepublishrequest_controller.go +++ b/controllers/virtualmachinepublishrequest/virtualmachinepublishrequest_controller.go @@ -91,7 +91,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err mgr.GetClient(), mgr.GetAPIReader(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, ) diff --git a/controllers/virtualmachinereplicaset/virtualmachinereplicaset_controller.go b/controllers/virtualmachinereplicaset/virtualmachinereplicaset_controller.go index 9348038b0..7cf8e1038 100644 --- a/controllers/virtualmachinereplicaset/virtualmachinereplicaset_controller.go +++ b/controllers/virtualmachinereplicaset/virtualmachinereplicaset_controller.go @@ -85,7 +85,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong))) + record.New(mgr.GetEventRecorder(controllerNameLong))) return ctrl.NewControllerManagedBy(mgr). For(controlledType). diff --git a/controllers/virtualmachineservice/providers/loadbalancer_provider.go b/controllers/virtualmachineservice/providers/loadbalancer_provider.go index 1ea791cc8..f156ffeb1 100644 --- a/controllers/virtualmachineservice/providers/loadbalancer_provider.go +++ b/controllers/virtualmachineservice/providers/loadbalancer_provider.go @@ -113,7 +113,7 @@ func (nl *NsxtLoadbalancerProvider) GetServiceLabels(ctx context.Context, vmServ // When externalTrafficPolicy is set to Local, skip kube-proxy for the // target Service - if etp := vmService.Annotations[utils.AnnotationServiceExternalTrafficPolicyKey]; corev1.ServiceExternalTrafficPolicyType(etp) == corev1.ServiceExternalTrafficPolicyTypeLocal { + if etp := vmService.Annotations[utils.AnnotationServiceExternalTrafficPolicyKey]; corev1.ServiceExternalTrafficPolicy(etp) == corev1.ServiceExternalTrafficPolicyTypeLocal { res[LabelServiceProxyName] = NSXTServiceProxy } @@ -127,7 +127,7 @@ func (nl *NsxtLoadbalancerProvider) GetToBeRemovedServiceLabels(ctx context.Cont // When there is no externalTrafficPolicy configured or it's not Local, // remove the service-proxy label - if etp := vmService.Annotations[utils.AnnotationServiceExternalTrafficPolicyKey]; corev1.ServiceExternalTrafficPolicyType(etp) != corev1.ServiceExternalTrafficPolicyTypeLocal { + if etp := vmService.Annotations[utils.AnnotationServiceExternalTrafficPolicyKey]; corev1.ServiceExternalTrafficPolicy(etp) != corev1.ServiceExternalTrafficPolicyTypeLocal { res[LabelServiceProxyName] = NSXTServiceProxy } diff --git a/controllers/virtualmachineservice/virtualmachineservice_controller.go b/controllers/virtualmachineservice/virtualmachineservice_controller.go index 94dba73ad..85589d508 100644 --- a/controllers/virtualmachineservice/virtualmachineservice_controller.go +++ b/controllers/virtualmachineservice/virtualmachineservice_controller.go @@ -70,7 +70,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), lbProvider, ) @@ -271,6 +271,10 @@ func (r *ReconcileVirtualMachineService) reconcileVMService(ctx *pkgctx.VirtualM return err } + ctx.Logger.V(5).Info("Service spec.ipFamilies", + "service", client.ObjectKeyFromObject(service).String(), + "serviceSpecIPFamilies", service.Spec.IPFamilies) + err = r.createOrUpdateEndpoints(ctx, service) if err != nil { ctx.Logger.Error(err, "Failed to update VirtualMachineService Endpoints") @@ -409,6 +413,25 @@ func (r *ReconcileVirtualMachineService) createOrUpdateService(ctx *pkgctx.Virtu service.Spec.ExternalName = vmService.Spec.ExternalName service.Spec.LoadBalancerIP = vmService.Spec.LoadBalancerIP service.Spec.LoadBalancerSourceRanges = vmService.Spec.LoadBalancerSourceRanges + + // Set IPFamilies and IPFamilyPolicy for dual-stack support + // These fields only apply to ClusterIP and LoadBalancer types + // They are wiped when type is ExternalName + if vmService.Spec.Type != vmopv1.VirtualMachineServiceTypeExternalName { + // Only overwrite when the VirtualMachineService specifies ipFamilies so we do not clear + // values defaulted or stored by the apiserver when the CR omits them. + if len(vmService.Spec.IPFamilies) > 0 { + service.Spec.IPFamilies = vmService.Spec.IPFamilies + } + if vmService.Spec.IPFamilyPolicy != nil { + service.Spec.IPFamilyPolicy = vmService.Spec.IPFamilyPolicy + } + } else { + // Clear IPFamilies and IPFamilyPolicy for ExternalName services + service.Spec.IPFamilies = nil + service.Spec.IPFamilyPolicy = nil + } + if service.Spec.Type == corev1.ServiceTypeLoadBalancer { service.Spec.AllocateLoadBalancerNodePorts = ptr.To(false) } else { @@ -451,7 +474,7 @@ func (r *ReconcileVirtualMachineService) createOrUpdateService(ctx *pkgctx.Virtu if externalTrafficPolicy, ok := service.Annotations[utils.AnnotationServiceExternalTrafficPolicyKey]; ok { // Note that this annotation is only set (and makes sense) from the GC cloud provider. - trafficPolicy := corev1.ServiceExternalTrafficPolicyType(externalTrafficPolicy) + trafficPolicy := corev1.ServiceExternalTrafficPolicy(externalTrafficPolicy) switch trafficPolicy { case corev1.ServiceExternalTrafficPolicyTypeLocal, corev1.ServiceExternalTrafficPolicyTypeCluster: service.Spec.ExternalTrafficPolicy = trafficPolicy @@ -613,7 +636,15 @@ func (r *ReconcileVirtualMachineService) generateSubsetsForService( return nil, err } - var subsets = make([]corev1.EndpointSubset, 0, len(vmList.Items)) + // Include only VM addresses whose IP family is listed in Service spec.ipFamilies. + allowedFamilies := determineAllowedIPFamilies(service) + + type addressInfo struct { + addr corev1.EndpointAddress + ready bool + ports []corev1.EndpointPort + } + var addressInfos []addressInfo var vmInSubsetsMap map[types.UID]struct{} for i := range vmList.Items { @@ -625,15 +656,17 @@ func (r *ReconcileVirtualMachineService) generateSubsetsForService( continue } - var vmIP string + var vmIPs []string if vm.Status.Network != nil { - vmIP = vm.Status.Network.PrimaryIP4 - if vmIP == "" { - vmIP = vm.Status.Network.PrimaryIP6 + if vm.Status.Network.PrimaryIP4 != "" && allowedFamilies[corev1.IPv4Protocol] { + vmIPs = append(vmIPs, vm.Status.Network.PrimaryIP4) + } + if vm.Status.Network.PrimaryIP6 != "" && allowedFamilies[corev1.IPv6Protocol] { + vmIPs = append(vmIPs, vm.Status.Network.PrimaryIP6) } } - if vmIP == "" { + if len(vmIPs) == 0 { // The EndpointAddress must have a valid IP so we cannot include this VM in the // NotReadyAddresses. // TODO: When we more fully support multiple NICs, we'll need someway to select which IP. @@ -663,30 +696,8 @@ func (r *ReconcileVirtualMachineService) generateSubsetsForService( } } - epa := corev1.EndpointAddress{ - IP: vmIP, - TargetRef: &corev1.ObjectReference{ - APIVersion: vm.APIVersion, - Kind: vm.Kind, - Namespace: vm.Namespace, - Name: vm.Name, - UID: vm.UID, - // NOTE: This currently isn't set to limit downstream reconcile churn in things - // watching these Endpoints but isn't ideal. We should be smarter and only update - // this when something relevant to the service, e.g. the VM's IP, changes. - // ResourceVersion: vm.ResourceVersion, - }, - } - - // Populate the EP subset for this VM. We create one subset for each VM, and then our - // caller will repack the subsets that have identical ports. - subset := corev1.EndpointSubset{} - if ready { - subset.Addresses = []corev1.EndpointAddress{epa} - } else { - subset.NotReadyAddresses = []corev1.EndpointAddress{epa} - } - + // Build ports list once for reuse across all IPs + var ports []corev1.EndpointPort // TODO: Headless support for _, servicePort := range service.Spec.Ports { portName := servicePort.Name @@ -702,7 +713,7 @@ func (r *ReconcileVirtualMachineService) generateSubsetsForService( continue } - subset.Ports = append(subset.Ports, + ports = append(ports, corev1.EndpointPort{ Name: portName, Port: int32(portNum), //nolint:gosec // disable G115 @@ -710,12 +721,58 @@ func (r *ReconcileVirtualMachineService) generateSubsetsForService( }) } - subsets = append(subsets, subset) + for _, ip := range vmIPs { + epa := corev1.EndpointAddress{ + IP: ip, + TargetRef: &corev1.ObjectReference{ + APIVersion: vm.APIVersion, + Kind: vm.Kind, + Namespace: vm.Namespace, + Name: vm.Name, + UID: vm.UID, + // NOTE: This currently isn't set to limit downstream reconcile churn in things + // watching these Endpoints but isn't ideal. We should be smarter and only update + // this when something relevant to the service, e.g. the VM's IP, changes. + // ResourceVersion: vm.ResourceVersion, + }, + } + + addressInfos = append(addressInfos, addressInfo{ + addr: epa, + ready: ready, + ports: ports, + }) + } + } + + subsets := make([]corev1.EndpointSubset, 0, len(addressInfos)) + for _, addrInfo := range addressInfos { + var readyAddrs, notReadyAddrs []corev1.EndpointAddress + if addrInfo.ready { + readyAddrs = append(readyAddrs, addrInfo.addr) + } else { + notReadyAddrs = append(notReadyAddrs, addrInfo.addr) + } + subsets = append(subsets, corev1.EndpointSubset{ + Addresses: readyAddrs, + NotReadyAddresses: notReadyAddrs, + Ports: addrInfo.ports, + }) } return subsets, nil } +// determineAllowedIPFamilies returns which IP families may appear on Endpoints for this Service. +// Only spec.ipFamilies is used (from the Service object after createOrUpdateService / apiserver merge). +func determineAllowedIPFamilies(service *corev1.Service) map[corev1.IPFamily]bool { + allowedFamilies := make(map[corev1.IPFamily]bool) + for _, family := range service.Spec.IPFamilies { + allowedFamilies[family] = true + } + return allowedFamilies +} + // updateVMService syncs the VirtualMachineService Status from the Service status. // //nolint:unparam diff --git a/controllers/virtualmachineservice/virtualmachineservice_controller_intg_test.go b/controllers/virtualmachineservice/virtualmachineservice_controller_intg_test.go index 39eef3e5d..3c4dbc99a 100644 --- a/controllers/virtualmachineservice/virtualmachineservice_controller_intg_test.go +++ b/controllers/virtualmachineservice/virtualmachineservice_controller_intg_test.go @@ -9,6 +9,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -16,6 +17,7 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/conditions" "github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels" + "github.com/vmware-tanzu/vm-operator/pkg/util/ptr" "github.com/vmware-tanzu/vm-operator/test/builder" ) @@ -138,18 +140,17 @@ func intgTestsReconcile() { }) By("VirtualMachineService finalizer should get set") - Eventually(func() []string { + Eventually(func(g Gomega) { vmService := &vmopv1.VirtualMachineService{} - if err := ctx.Client.Get(ctx, objKey, vmService); err == nil { - return vmService.GetFinalizers() - } - return nil - }).Should(ContainElement(finalizerName)) + + g.Expect(ctx.Client.Get(ctx, objKey, vmService)).To(Succeed()) + g.Expect(vmService.GetFinalizers()).To(ContainElement(finalizerName)) + }).Should(Succeed()) By("Service should be created", func() { service := &corev1.Service{} - Eventually(func() error { - return ctx.Client.Get(ctx, objKey, service) + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, objKey, service)).To(Succeed()) }).Should(Succeed()) // Service should have label and annotations replicated on vmService create @@ -163,8 +164,8 @@ func intgTestsReconcile() { By("Endpoints should be created", func() { endpoints := &corev1.Endpoints{} - Eventually(func() error { - return ctx.Client.Get(ctx, objKey, endpoints) + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) }).Should(Succeed()) Expect(endpoints.Labels).To(HaveKeyWithValue(dummyLabelKey, dummyLabelVal)) @@ -214,13 +215,13 @@ func intgTestsReconcile() { By("Ready VM should be added to Endpoints", func() { endpoints := &corev1.Endpoints{} - Eventually(func() bool { - if err := ctx.Client.Get(ctx, objKey, endpoints); err == nil && len(endpoints.Subsets) == 1 { - subset := endpoints.Subsets[0] - return len(subset.Addresses) != 0 && len(subset.NotReadyAddresses) != 0 - } - return false - }).Should(BeTrue()) + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + g.Expect(endpoints.Subsets).To(HaveLen(1)) + subset := endpoints.Subsets[0] + g.Expect(subset.Addresses).ToNot(BeEmpty()) + g.Expect(subset.NotReadyAddresses).ToNot(BeEmpty()) + }).Should(Succeed()) subsets := endpoints.Subsets Expect(subsets).To(HaveLen(1)) @@ -255,25 +256,21 @@ func intgTestsReconcile() { Expect(ctx.Client.Update(ctx, notReadyVM)).To(Succeed()) Expect(ctx.Client.Delete(ctx, notReadyVM)).To(Succeed()) - Eventually(func() bool { - endpoints := &corev1.Endpoints{} - if err := ctx.Client.Get(ctx, objKey, endpoints); err == nil { - return len(endpoints.Subsets) == 1 && len(endpoints.Subsets[0].NotReadyAddresses) == 0 - } - return false - }).Should(BeTrue(), "not ready VM should be removed from Endpoints") + endpoints := &corev1.Endpoints{} + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + g.Expect(endpoints.Subsets).To(HaveLen(1)) + g.Expect(endpoints.Subsets[0].NotReadyAddresses).To(BeEmpty()) + }).Should(Succeed(), "not ready VM should be removed from Endpoints") readyVM.Finalizers = append(readyVM.Finalizers, "dummy.test.finalizer") Expect(ctx.Client.Update(ctx, readyVM)).To(Succeed()) Expect(ctx.Client.Delete(ctx, readyVM)).To(Succeed()) - Eventually(func() bool { - endpoints := &corev1.Endpoints{} - if err := ctx.Client.Get(ctx, objKey, endpoints); err == nil { - return len(endpoints.Subsets) == 0 - } - return false - }).Should(BeTrue()) + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + g.Expect(endpoints.Subsets).To(BeEmpty()) + }).Should(Succeed()) Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(notReadyVM), notReadyVM)).To(Succeed()) notReadyVM.Finalizers = nil @@ -287,13 +284,120 @@ func intgTestsReconcile() { By("Delete VirtualMachineService and finalizer should be removed", func() { Expect(ctx.Client.Delete(ctx, vmService)).To(Succeed()) - Eventually(func() []string { - vmService := &vmopv1.VirtualMachineService{} - if err := ctx.Client.Get(ctx, objKey, vmService); err == nil { - return vmService.GetFinalizers() + Eventually(func(g Gomega) { + got := &vmopv1.VirtualMachineService{} + err := ctx.Client.Get(ctx, objKey, got) + if apierrors.IsNotFound(err) { + return } - return nil - }).ShouldNot(ContainElement(finalizerName)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(got.GetFinalizers()).NotTo(ContainElement(finalizerName)) + }).Should(Succeed()) + }) + }) + }) + + Context("Endpoints reflect Service IP family selection", func() { + var vmForIPFamilyTest *vmopv1.VirtualMachine + + AfterEach(func() { + if vmForIPFamilyTest != nil { + Expect(client.IgnoreNotFound(ctx.Client.Delete(ctx, vmForIPFamilyTest))).To(Succeed()) + vmForIPFamilyTest = nil + } + }) + + Context("Dual-stack VM with SingleStack IPv4 service", func() { + BeforeEach(func() { + vmServiceName = "test-vm-service-dual-stack" + }) + + It("includes only IPv4 when VM has both primary IPv4 and IPv6", func() { + vmForIPFamilyTest = createVMWithNetwork(ctx, "dual-stack-vm", vmLabels, "192.168.1.100", "2001:db8::100") + vmService := createVMService(ctx, vmServiceName, selector, vmServicePort, + []corev1.IPFamily{corev1.IPv4Protocol}, nil) + + objKey := client.ObjectKeyFromObject(vmService) + endpoints := &corev1.Endpoints{} + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + g.Expect(endpoints.Subsets).ToNot(BeEmpty()) + }).Should(Succeed()) + + allIPs := collectEndpointIPs(endpoints) + Expect(allIPs).To(ContainElement("192.168.1.100")) + Expect(allIPs).NotTo(ContainElement("2001:db8::100")) + }) + }) + + Context("IPv6-only VM with SingleStack IPv4 service", func() { + BeforeEach(func() { + vmServiceName = "test-vm-service-ipv6-only" + }) + + It("has empty Endpoints because IPv4 service cannot route to IPv6-only VM", func() { + vmForIPFamilyTest = createVMWithNetwork(ctx, "ipv6-vm", vmLabels, "", "2001:db8::200") + vmService := createVMService(ctx, vmServiceName, selector, vmServicePort, + []corev1.IPFamily{corev1.IPv4Protocol}, ptr.To(corev1.IPFamilyPolicySingleStack)) + + objKey := client.ObjectKeyFromObject(vmService) + endpoints := &corev1.Endpoints{} + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + }).Should(Succeed()) + + Expect(endpoints.Subsets).To(BeEmpty()) + }) + }) + + Context("PreferDualStack with ipFamilies defaulted by apiserver", func() { + BeforeEach(func() { + vmServiceName = "test-vm-service-prefer-empty-ipf" + }) + + // This should succeed since envtest is single stack (explicit dual ipFamilies rejected). + It("creates child Service and IPv4 Endpoints", func() { + vmForIPFamilyTest = createVMWithNetwork(ctx, "vm-prefer-empty-ipf", vmLabels, "192.168.6.1", "2001:db8::601") + vmService := createVMService(ctx, vmServiceName, selector, vmServicePort, + nil, ptr.To(corev1.IPFamilyPolicyPreferDualStack)) + + svcKey := client.ObjectKeyFromObject(vmService) + svc := &corev1.Service{} + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, svcKey, svc)).To(Succeed()) + }).Should(Succeed()) + + endpoints := &corev1.Endpoints{} + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, svcKey, endpoints)).To(Succeed()) + g.Expect(endpoints.Subsets).ToNot(BeEmpty()) + }).Should(Succeed()) + + Expect(collectEndpointIPs(endpoints)).To(ContainElement("192.168.6.1")) + }) + }) + + // This should fail since envtest is single stack. + + // Expects a cluster where dual-stack Services are rejected (typical envtest). + // Running against a dual-stack-capable cluster may require skipping this spec or + // gating with an environment variable. + Context("RequireDualStack with empty ipFamilies", func() { + BeforeEach(func() { + vmServiceName = "test-vm-service-require-empty-ipf" + }) + + It("does not create a child Service", func() { + vmForIPFamilyTest = createVMWithNetwork(ctx, "vm-require-empty-ipf", vmLabels, "192.168.7.1", "2001:db8::701") + vmService := createVMService(ctx, vmServiceName, selector, vmServicePort, + nil, ptr.To(corev1.IPFamilyPolicyRequireDualStack)) + + svcKey := client.ObjectKeyFromObject(vmService) + svc := &corev1.Service{} + Consistently(func(g Gomega) { + err := ctx.Client.Get(ctx, svcKey, svc) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }).Should(Succeed()) }) }) }) @@ -332,9 +436,10 @@ func intgTestsReconcile() { Namespace: ctx.Namespace, }, Spec: vmopv1.VirtualMachineServiceSpec{ - Type: vmopv1.VirtualMachineServiceTypeLoadBalancer, - Ports: []vmopv1.VirtualMachineServicePort{vmServicePort}, - Selector: selector, + Type: vmopv1.VirtualMachineServiceTypeLoadBalancer, + Ports: []vmopv1.VirtualMachineServicePort{vmServicePort}, + Selector: selector, + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, }, } @@ -346,12 +451,10 @@ func intgTestsReconcile() { By("Ready VM should be added to Endpoints", func() { endpoints := &corev1.Endpoints{} - Eventually(func() bool { - if err := ctx.Client.Get(ctx, objKey, endpoints); err == nil { - return len(endpoints.Subsets) != 0 - } - return false - }).Should(BeTrue()) + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + g.Expect(endpoints.Subsets).ToNot(BeEmpty()) + }).Should(Succeed()) subsets := endpoints.Subsets Expect(subsets).To(HaveLen(1)) @@ -375,15 +478,69 @@ func intgTestsReconcile() { }) By("VM should be removed from Endpoints", func() { - Eventually(func() bool { - endpoints := &corev1.Endpoints{} - if err := ctx.Client.Get(ctx, objKey, endpoints); err == nil { - return len(endpoints.Subsets) == 0 - } - return false - }).Should(BeTrue()) + endpoints := &corev1.Endpoints{} + Eventually(func(g Gomega) { + g.Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + g.Expect(endpoints.Subsets).To(BeEmpty()) + }).Should(Succeed()) }) }) }) }) } + +func createVMWithNetwork(ctx *builder.IntegrationTestContext, name string, labels map[string]string, ip4, ip6 string) *vmopv1.VirtualMachine { + vm := &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ctx.Namespace, + Labels: labels, + }, + Spec: vmopv1.VirtualMachineSpec{ + PowerState: vmopv1.VirtualMachinePowerStateOn, + }, + } + Expect(ctx.Client.Create(ctx, vm)).To(Succeed()) + + vm.Status.Network = &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP4: ip4, + PrimaryIP6: ip6, + } + Expect(ctx.Client.Status().Update(ctx, vm)).To(Succeed()) + return vm +} + +func createVMService( + ctx *builder.IntegrationTestContext, + name string, + selector map[string]string, + port vmopv1.VirtualMachineServicePort, + ipFamilies []corev1.IPFamily, + policy *corev1.IPFamilyPolicy, +) *vmopv1.VirtualMachineService { + vmService := &vmopv1.VirtualMachineService{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ctx.Namespace, + }, + Spec: vmopv1.VirtualMachineServiceSpec{ + Type: vmopv1.VirtualMachineServiceTypeClusterIP, + Ports: []vmopv1.VirtualMachineServicePort{port}, + Selector: selector, + IPFamilies: ipFamilies, + IPFamilyPolicy: policy, + }, + } + Expect(ctx.Client.Create(ctx, vmService)).To(Succeed()) + return vmService +} + +func collectEndpointIPs(endpoints *corev1.Endpoints) []string { + var ips []string + for _, subset := range endpoints.Subsets { + for _, addr := range subset.Addresses { + ips = append(ips, addr.IP) + } + } + return ips +} diff --git a/controllers/virtualmachineservice/virtualmachineservice_controller_unit_test.go b/controllers/virtualmachineservice/virtualmachineservice_controller_unit_test.go index 29d332b1e..887a79a1e 100644 --- a/controllers/virtualmachineservice/virtualmachineservice_controller_unit_test.go +++ b/controllers/virtualmachineservice/virtualmachineservice_controller_unit_test.go @@ -194,6 +194,39 @@ func unitTestsReconcile() { Expect(*service.Spec.AllocateLoadBalancerNodePorts).To(BeFalse()) }) + Context("IPFamilies and IPFamilyPolicy", func() { + It("Copies IPFamilies to Service spec", func() { + vmService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, service)).To(Succeed()) + Expect(service.Spec.IPFamilies).To(HaveLen(2)) + Expect(service.Spec.IPFamilies).To(ContainElements(corev1.IPv4Protocol, corev1.IPv6Protocol)) + }) + + It("Copies IPFamilyPolicy to Service spec", func() { + policy := corev1.IPFamilyPolicyPreferDualStack + vmService.Spec.IPFamilyPolicy = &policy + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, service)).To(Succeed()) + Expect(service.Spec.IPFamilyPolicy).ToNot(BeNil()) + Expect(*service.Spec.IPFamilyPolicy).To(Equal(corev1.IPFamilyPolicyPreferDualStack)) + }) + + It("Clears IPFamilies and IPFamilyPolicy for ExternalName services", func() { + vmService.Spec.Type = vmopv1.VirtualMachineServiceTypeExternalName + vmService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol} + policy := corev1.IPFamilyPolicySingleStack + vmService.Spec.IPFamilyPolicy = &policy + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, service)).To(Succeed()) + Expect(service.Spec.IPFamilies).To(BeEmpty()) + Expect(service.Spec.IPFamilyPolicy).To(BeNil()) + }) + }) + Context("With Expected Spec.Ports", func() { BeforeEach(func() { vmService.Spec.Ports = []vmopv1.VirtualMachineServicePort{ @@ -441,6 +474,10 @@ func unitTestsReconcile() { vmService.Spec.Ports = []vmopv1.VirtualMachineServicePort{ vmServicePort1, } + vmService.Spec.IPFamilies = []corev1.IPFamily{ + corev1.IPv4Protocol, + corev1.IPv6Protocol, + } vm1 = &vmopv1.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{ @@ -535,6 +572,25 @@ func unitTestsReconcile() { }) }) + Context("when VirtualMachineService omits ipFamilies (simulate apiserver defaulting on fake client)", func() { + BeforeEach(func() { + vmService.Spec.IPFamilies = nil + policy := corev1.IPFamilyPolicyPreferDualStack + vmService.Spec.IPFamilyPolicy = &policy + initObjects = append(initObjects, vm1, vm3) + }) + + It("repopulates endpoints after defaulted ipFamilies are written to the Service", func() { + simulateAPIServerDefaultedIPFamilies(ctx, objKey) + Expect(reconciler.ReconcileNormal(vmServiceCtx)).To(Succeed()) + Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + Expect(endpoints.Subsets).To(HaveLen(1)) + subset := endpoints.Subsets[0] + Expect(subset.Addresses).To(HaveLen(1)) + assertEPAddrFromVM(subset.Addresses[0], vm1) + }) + }) + Context("When multiple VMs match label selector", func() { BeforeEach(func() { initObjects = append(initObjects, vm1, vm2, vm3) @@ -642,6 +698,229 @@ func unitTestsReconcile() { }) }) + Context("Endpoint filtering based on Service IPFamilies", func() { + var dualStackVM *vmopv1.VirtualMachine + var ipv4VM *vmopv1.VirtualMachine + var ipv6VM *vmopv1.VirtualMachine + + BeforeEach(func() { + dualStackVM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dual-stack-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP4: "192.168.1.1", + PrimaryIP6: "2001:db8::1", + }, + }, + } + ipv4VM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ipv4-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP4: "192.168.1.2", + }, + }, + } + ipv6VM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ipv6-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP6: "2001:db8::2", + }, + }, + } + initObjects = append(initObjects, dualStackVM, ipv4VM, ipv6VM) + }) + + It("IPv4-only Service only includes IPv4 endpoints", func() { + vmService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol} + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + + allAddresses := collectAllEndpointIPs(endpoints) + Expect(allAddresses).To(ContainElements("192.168.1.1", "192.168.1.2")) + Expect(allAddresses).NotTo(ContainElements("2001:db8::1", "2001:db8::2")) + }) + + It("IPv6-only Service only includes IPv6 endpoints", func() { + vmService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv6Protocol} + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + + allAddresses := collectAllEndpointIPs(endpoints) + Expect(allAddresses).To(ContainElements("2001:db8::1", "2001:db8::2")) + Expect(allAddresses).NotTo(ContainElements("192.168.1.1", "192.168.1.2")) + }) + + It("Dual-stack Service includes both IPv4 and IPv6 endpoints", func() { + vmService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + + allAddresses := collectAllEndpointIPs(endpoints) + Expect(allAddresses).To(ContainElements("192.168.1.1", "192.168.1.2", "2001:db8::1", "2001:db8::2")) + }) + }) + + Context("Endpoint filtering based on Service IPFamilyPolicy", func() { + var dualStackVM *vmopv1.VirtualMachine + var ipv4VM *vmopv1.VirtualMachine + var ipv6VM *vmopv1.VirtualMachine + + BeforeEach(func() { + dualStackVM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dual-stack-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP4: "192.168.1.1", + PrimaryIP6: "2001:db8::1", + }, + }, + } + ipv4VM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ipv4-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP4: "192.168.1.2", + }, + }, + } + ipv6VM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ipv6-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP6: "2001:db8::2", + }, + }, + } + initObjects = append(initObjects, dualStackVM, ipv4VM, ipv6VM) + }) + + It("SingleStack policy with IPv4 clusterIP only includes IPv4 endpoints", func() { + service := &corev1.Service{} + policy := corev1.IPFamilyPolicySingleStack + vmService.Spec.IPFamilyPolicy = &policy + vmService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol} + // Create service first to get clusterIP + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, service)).To(Succeed()) + // Set IPv4 clusterIP + service.Spec.ClusterIPs = []string{"10.0.0.1"} + Expect(ctx.Client.Update(ctx, service)).To(Succeed()) + + err = reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + + allAddresses := collectReadyEndpointIPs(endpoints) + Expect(allAddresses).To(ContainElements("192.168.1.1", "192.168.1.2")) + Expect(allAddresses).NotTo(ContainElements("2001:db8::1", "2001:db8::2")) + }) + + It("PreferDualStack policy includes both IPv4 and IPv6 endpoints", func() { + policy := corev1.IPFamilyPolicyPreferDualStack + vmService.Spec.IPFamilyPolicy = &policy + vmService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + + allAddresses := collectReadyEndpointIPs(endpoints) + Expect(allAddresses).To(ContainElements("192.168.1.1", "192.168.1.2", "2001:db8::1", "2001:db8::2")) + }) + + It("RequireDualStack policy includes both IPv4 and IPv6 endpoints", func() { + policy := corev1.IPFamilyPolicyRequireDualStack + vmService.Spec.IPFamilyPolicy = &policy + vmService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + + allAddresses := collectReadyEndpointIPs(endpoints) + Expect(allAddresses).To(ContainElements("192.168.1.1", "192.168.1.2", "2001:db8::1", "2001:db8::2")) + }) + }) + + Context("VM created after VirtualMachineService", func() { + var newVM *vmopv1.VirtualMachine + + BeforeEach(func() { + vmService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol} + // Don't add VM to initObjects initially - simulate VM being created later + }) + + JustBeforeEach(func() { + // First reconciliation: Service created, but no VMs yet + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + // Service may be created or updated depending on test execution order + // Drain any events (Create or Update) to ensure event channel is ready + select { + case <-ctx.Events: + default: + } + Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + // Initially, endpoints should be empty + Expect(endpoints.Subsets).To(BeEmpty()) + + // Now create a VM that matches the selector (simulating VM created after service) + newVM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "new-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP4: "192.168.1.100", + PrimaryIP6: "2001:db8::100", + }, + }, + } + Expect(ctx.Client.Create(ctx, newVM)).To(Succeed()) + }) + + It("VM is added to endpoints with correct IP family filtering", func() { + // Reconcile again after VM is created (simulating controller watching VM changes) + err := reconciler.ReconcileNormal(vmServiceCtx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Client.Get(ctx, objKey, endpoints)).To(Succeed()) + + allAddresses := collectReadyEndpointIPs(endpoints) + Expect(allAddresses).To(ContainElement("192.168.1.100")) + Expect(allAddresses).NotTo(ContainElement("2001:db8::100")) + }) + }) + Context("Preserve VMs in Endpoints that have Probe but hasn't run yet", func() { BeforeEach(func() { vm1.UID = "abc" @@ -686,6 +965,158 @@ func unitTestsReconcile() { assertEPAddrFromVM(subset.NotReadyAddresses[0], vm2) }) }) + + Context("IPv6-only VM", func() { + var ipv6VM *vmopv1.VirtualMachine + + BeforeEach(func() { + vmService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv6Protocol} + ipv6VM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ipv6-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP6: "2001:db8::1", + }, + }, + } + initObjects = append(initObjects, ipv6VM) + }) + + It("Endpoint should contain IPv6 address", func() { + Expect(endpoints.Subsets).To(HaveLen(1)) + subset := endpoints.Subsets[0] + + Expect(subset.Ports).To(HaveLen(1)) + assertEPPortFromVMServicePort(subset.Ports[0], vmServicePort1) + + Expect(subset.Addresses).To(HaveLen(1)) + assertEPAddrFromVMWithIP(subset.Addresses[0], ipv6VM, "2001:db8::1") + Expect(subset.NotReadyAddresses).To(BeEmpty()) + }) + + It("Endpoint should NOT contain IPv4 address", func() { + Expect(endpoints.Subsets).To(HaveLen(1)) + subset := endpoints.Subsets[0] + + for _, addr := range subset.Addresses { + Expect(addr.IP).ToNot(Equal("")) + Expect(addr.IP).To(Equal("2001:db8::1")) + } + }) + }) + + Context("Dual-stack VM", func() { + var dualStackVM *vmopv1.VirtualMachine + + BeforeEach(func() { + dualStackVM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dual-stack-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP4: "192.168.1.100", + PrimaryIP6: "2001:db8::100", + }, + }, + } + initObjects = append(initObjects, dualStackVM) + }) + + It("Endpoints repack IPv4 and IPv6 into one canonical subset", func() { + Expect(endpoints.Subsets).To(HaveLen(1)) + subset := endpoints.Subsets[0] + Expect(subset.Ports).To(HaveLen(1)) + assertEPPortFromVMServicePort(subset.Ports[0], vmServicePort1) + + var ips []string + for _, addr := range subset.Addresses { + ips = append(ips, addr.IP) + } + Expect(ips).To(ContainElements("192.168.1.100", "2001:db8::100")) + + var v4Addr, v6Addr *corev1.EndpointAddress + for i := range subset.Addresses { + a := &subset.Addresses[i] + switch a.IP { + case "192.168.1.100": + v4Addr = a + case "2001:db8::100": + v6Addr = a + } + } + Expect(v4Addr).ToNot(BeNil()) + Expect(v6Addr).ToNot(BeNil()) + assertEPAddrFromVMWithIP(*v4Addr, dualStackVM, "192.168.1.100") + assertEPAddrFromVMWithIP(*v6Addr, dualStackVM, "2001:db8::100") + }) + }) + + Context("Mixed VMs (IPv4-only, IPv6-only, and dual-stack)", func() { + var ipv4VM, ipv6VM, dualStackVM *vmopv1.VirtualMachine + + BeforeEach(func() { + ipv4VM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ipv4-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP4: "192.168.1.10", + }, + }, + } + + ipv6VM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ipv6-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP6: "2001:db8::10", + }, + }, + } + + dualStackVM = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dual-stack-vm", + Namespace: vmService.Namespace, + Labels: vmLabels, + }, + Status: vmopv1.VirtualMachineStatus{ + Network: &vmopv1.VirtualMachineNetworkStatus{ + PrimaryIP4: "192.168.1.20", + PrimaryIP6: "2001:db8::20", + }, + }, + } + + initObjects = append(initObjects, ipv4VM, ipv6VM, dualStackVM) + }) + + It("All VMs appear in appropriate subsets based on their IP families", func() { + Expect(endpoints.Subsets).ToNot(BeEmpty()) + + allIPs := collectReadyEndpointIPs(endpoints) + // Should have: IPv4 from ipv4VM, IPv6 from ipv6VM, IPv4 and IPv6 from dualStackVM + Expect(allIPs).To(ContainElement("192.168.1.10"), "Should contain IPv4-only VM IP") + Expect(allIPs).To(ContainElement("2001:db8::10"), "Should contain IPv6-only VM IP") + Expect(allIPs).To(ContainElement("192.168.1.20"), "Should contain dual-stack VM IPv4") + Expect(allIPs).To(ContainElement("2001:db8::20"), "Should contain dual-stack VM IPv6") + Expect(allIPs).To(HaveLen(4), "Should have 4 total IP addresses") + }) + }) }) Context("Selectorless VirtualMachineService", func() { @@ -701,6 +1132,10 @@ func unitTestsReconcile() { vmService.Spec.Ports = []vmopv1.VirtualMachineServicePort{ vmServicePort1, } + vmService.Spec.IPFamilies = []corev1.IPFamily{ + corev1.IPv4Protocol, + corev1.IPv6Protocol, + } vm1 = &vmopv1.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{ @@ -994,6 +1429,23 @@ func nsxtLBProviderTestsReconcile() { }) } +// simulateAPIServerDefaultedIPFamilies writes spec.ipFamilies on the Service when it is unset. +// A real apiserver defaulting chain fills this after create; controller-runtime's fake client does not. +func simulateAPIServerDefaultedIPFamilies( + ctx *builder.UnitTestContextForController, + svcKey client.ObjectKey, +) { + svc := &corev1.Service{} + Expect(ctx.Client.Get(ctx, svcKey, svc)).To(Succeed()) + if len(svc.Spec.IPFamilies) > 0 { + return + } + // Align with single-stack envtest: PreferDualStack yields IPv4-only ipFamilies. + families := []corev1.IPFamily{corev1.IPv4Protocol} + svc.Spec.IPFamilies = families + Expect(ctx.Client.Update(ctx, svc)).To(Succeed()) +} + func expectEvent(ctx *builder.UnitTestContextForController, matcher types.GomegaMatcher) { var event string EventuallyWithOffset(1, ctx.Events).Should(Receive(&event)) @@ -1019,3 +1471,42 @@ func assertEPAddrFromVM( ExpectWithOffset(1, addr.TargetRef.Name).To(Equal(vm.Name)) ExpectWithOffset(1, addr.TargetRef.Namespace).To(Equal(vm.Namespace)) } + +// assertEPAddrFromVMWithIP validates that the endpoint address matches the expected IP +// and belongs to the specified VM. Supports both IPv4 and IPv6. +func assertEPAddrFromVMWithIP( + addr corev1.EndpointAddress, + vm *vmopv1.VirtualMachine, + expectedIP string) { + + ExpectWithOffset(1, vm.Status.Network).ToNot(BeNil()) + ExpectWithOffset(1, addr.IP).To(Equal(expectedIP)) + ExpectWithOffset(1, addr.TargetRef).ToNot(BeNil()) + ExpectWithOffset(1, addr.TargetRef.Name).To(Equal(vm.Name)) + ExpectWithOffset(1, addr.TargetRef.Namespace).To(Equal(vm.Namespace)) +} + +// collectReadyEndpointIPs returns all IPs from ready (Addresses) subsets of an Endpoints object. +func collectReadyEndpointIPs(endpoints *corev1.Endpoints) []string { + var ips []string + for _, subset := range endpoints.Subsets { + for _, addr := range subset.Addresses { + ips = append(ips, addr.IP) + } + } + return ips +} + +// collectAllEndpointIPs returns all IPs from both ready and not-ready subsets of an Endpoints object. +func collectAllEndpointIPs(endpoints *corev1.Endpoints) []string { + var ips []string + for _, subset := range endpoints.Subsets { + for _, addr := range subset.Addresses { + ips = append(ips, addr.IP) + } + for _, addr := range subset.NotReadyAddresses { + ips = append(ips, addr.IP) + } + } + return ips +} diff --git a/controllers/virtualmachinesnapshot/virtualmachinesnapshot_controller.go b/controllers/virtualmachinesnapshot/virtualmachinesnapshot_controller.go index 663c113c4..333279e1a 100644 --- a/controllers/virtualmachinesnapshot/virtualmachinesnapshot_controller.go +++ b/controllers/virtualmachinesnapshot/virtualmachinesnapshot_controller.go @@ -60,7 +60,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, ) diff --git a/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_controller.go b/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_controller.go index cf9bdcf63..794c79a3a 100644 --- a/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_controller.go +++ b/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_controller.go @@ -49,7 +49,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, ) diff --git a/controllers/virtualmachinewebconsolerequest/webconsolerequest_controller.go b/controllers/virtualmachinewebconsolerequest/webconsolerequest_controller.go index ca74edea7..4a9186939 100644 --- a/controllers/virtualmachinewebconsolerequest/webconsolerequest_controller.go +++ b/controllers/virtualmachinewebconsolerequest/webconsolerequest_controller.go @@ -47,7 +47,7 @@ func addToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ctx.VMProvider, ) diff --git a/controllers/vspherepolicy/policyevaluation/policyevaluation_controller.go b/controllers/vspherepolicy/policyevaluation/policyevaluation_controller.go index 84fb69b32..63dd57895 100644 --- a/controllers/vspherepolicy/policyevaluation/policyevaluation_controller.go +++ b/controllers/vspherepolicy/policyevaluation/policyevaluation_controller.go @@ -50,7 +50,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err ctx, mgr.GetClient(), ctrl.Log.WithName("controllers").WithName(controlledTypeName), - record.New(mgr.GetEventRecorderFor(controllerNameLong)), + record.New(mgr.GetEventRecorder(controllerNameLong)), ) return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/vspherepolicy/policyevaluation/policyevaluation_controller_test.go b/controllers/vspherepolicy/policyevaluation/policyevaluation_controller_test.go index e35fa5d53..1c81fae9f 100644 --- a/controllers/vspherepolicy/policyevaluation/policyevaluation_controller_test.go +++ b/controllers/vspherepolicy/policyevaluation/policyevaluation_controller_test.go @@ -14,7 +14,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - apirecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -97,7 +97,7 @@ var _ = Describe("Reconcile", func() { ctx, client, log.Log.WithName("test"), - record.New(apirecord.NewFakeRecorder(100)), + record.New(events.NewFakeRecorder(100)), ) }) diff --git a/external/image-registry-operator/api/v1alpha1/condition_constants.go b/external/image-registry-operator/api/v1alpha1/condition_constants.go new file mode 100644 index 000000000..bae88bb46 --- /dev/null +++ b/external/image-registry-operator/api/v1alpha1/condition_constants.go @@ -0,0 +1,81 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +// Common ConditionTypes used by Image Registry Operator API objects. +const ( + // ReadyCondition defines the Ready condition type that summarizes the operational state of an Image Registry Operator API object. + ReadyCondition ConditionType = "Ready" +) + +// Common Condition.Reason used by Image Registry Operator API objects. +const ( + // DeletingReason (Severity=Info) documents a condition not in Status=True because the object is currently being deleted. + DeletingReason = "Deleting" + + // DeletedReason (Severity=Error) documents a condition not in Status=True because the underlying object was deleted. + DeletedReason = "Deleted" +) + +// Condition.Reasons related to ClusterContentLibraryItem or ContentLibraryItem API objects. +const ( + ClusterContentLibraryRefValidationFailedReason = "ClusterContentLibraryRefValidationFailed" + ContentLibraryRefValidationFailedReason = "ContentLibraryRefValidationFailed" + ContentLibraryItemFileUnavailableReason = "ContentLibraryItemFileUnavailable" +) + +// ConditionTypes used by ContentLibraryItemImportRequest API objects. +const ( + ContentLibraryItemImportRequestSourceValid ConditionType = "SourceValid" + ContentLibraryItemImportRequestTargetValid ConditionType = "TargetValid" + ContentLibraryItemImportRequestContentLibraryItemCreated ConditionType = "ContentLibraryItemCreated" + ContentLibraryItemImportRequestTemplateUploaded ConditionType = "TemplateUploaded" + ContentLibraryItemImportRequestContentLibraryItemReady ConditionType = "ContentLibraryItemReady" + ContentLibraryItemImportRequestComplete ConditionType = "Complete" +) + +// Condition.Reasons related to ContentLibraryItemImportRequest API objects. +const ( + // SourceURLInvalidReason documents that the source URL specified in the ContentLibraryItemImportRequest is invalid. + SourceURLInvalidReason = "SourceURLInvalid" + + // SourceURLSchemeInvalidReason documents that the scheme in the source URL specified in the + // ContentLibraryItemImportRequest is invalid. + SourceURLSchemeInvalidReason = "SourceURLSchemeInvalid" + + // SourceURLHostInvalidReason documents that the host in the source URL specified in the + // ContentLibraryItemImportRequest is invalid. + SourceURLHostInvalidReason = "SourceURLHostInvalid" + + // SourceSSLCertificateUntrustedReason documents that the SSL certificate served at the source URL is not trusted by vSphere. + SourceSSLCertificateUntrustedReason = "SourceSSLCertificateUntrusted" + + // TargetLibraryInvalidReason documents that the target ContentLibrary specified in the + // ContentLibraryItemImportRequest is invalid. + TargetLibraryInvalidReason = "TargetLibraryInvalid" + + // TargetLibraryItemInvalidReason documents that the specified target content library item in the + // ContentLibraryItemImportRequest is invalid. + TargetLibraryItemInvalidReason = "TargetLibraryItemInvalid" + + // TargetLibraryItemCreationFailureReason documents that the creation of the target ContentLibraryItem has failed. + TargetLibraryItemCreationFailureReason = "TargetLibraryItemCreationFailure" + + // TargetLibraryItemUnavailableReason documents that the target ContentLibraryItem resource is not available in the + // namespace yet. + TargetLibraryItemUnavailableReason = "TargetLibraryItemUnavailable" + + // UploadInProgressReason documents that the files are in progress being uploaded to the target ContentLibraryItem. + UploadInProgressReason = "UploadInProgress" + + // UploadFailureReason documents that uploading files to the target content library item has failed. + UploadFailureReason = "UploadFailure" + + // UploadSessionExpiredReason documents that the uploading session for this ContentLibraryItemImportRequest has expired. + UploadSessionExpiredReason = "UploadSessionExpired" + + // TargetLibraryItemNotReadyReason documents that the target ContentLibraryItem resource is not in ready status yet. + TargetLibraryItemNotReadyReason = "TargetLibraryItemNotReady" +) diff --git a/external/image-registry-operator/api/v1alpha1/condition_types.go b/external/image-registry-operator/api/v1alpha1/condition_types.go new file mode 100644 index 000000000..c8455fdfa --- /dev/null +++ b/external/image-registry-operator/api/v1alpha1/condition_types.go @@ -0,0 +1,82 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2020 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ConditionSeverity expresses the severity of a Condition Type failing. +type ConditionSeverity string + +const ( + // ConditionSeverityError specifies that a condition with `Status=False` is an error. + ConditionSeverityError ConditionSeverity = "Error" + + // ConditionSeverityWarning specifies that a condition with `Status=False` is a warning. + ConditionSeverityWarning ConditionSeverity = "Warning" + + // ConditionSeverityInfo specifies that a condition with `Status=False` is informative. + ConditionSeverityInfo ConditionSeverity = "Info" + + // ConditionSeverityNone should apply only to conditions with `Status=True`. + ConditionSeverityNone ConditionSeverity = "" +) + +// ConditionType is a valid value for Condition.Type. +type ConditionType string + +// Condition defines an observation of an Image Registry Operator API resource operational state. +type Condition struct { + // Type of condition in CamelCase or in foo.example.com/CamelCase. + // Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + // can be useful (see .node.status.conditions), the ability to deconflict is important. + // +required + Type ConditionType `json:"type"` + + // Status of the condition, one of True, False, Unknown. + // +required + Status corev1.ConditionStatus `json:"status"` + + // Severity provides an explicit classification of Reason code, so the users or machines can immediately + // understand the current situation and act accordingly. + // The Severity field MUST be set only when Status=False. + // +optional + Severity ConditionSeverity `json:"severity,omitempty"` + + // Last time the condition transitioned from one status to another. + // This should be when the underlying condition changed. If that is not known, then using the time when + // the API field changed is acceptable. + // +required + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + + // The reason for the condition's last transition in CamelCase. + // The specific API may choose whether or not this field is considered a guaranteed API. + // This field may not be empty. + // +optional + Reason string `json:"reason,omitempty"` + + // A human readable message indicating details about the transition. + // This field may be empty. + // +optional + Message string `json:"message,omitempty"` +} + +// Conditions provide observations of the operational state of a Image Registry Operator API resource. +type Conditions []Condition diff --git a/external/image-registry-operator/api/v1alpha1/contentlibrary_types.go b/external/image-registry-operator/api/v1alpha1/contentlibrary_types.go new file mode 100644 index 000000000..0865fa4e1 --- /dev/null +++ b/external/image-registry-operator/api/v1alpha1/contentlibrary_types.go @@ -0,0 +1,285 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// ContentLibraryType is a constant type that indicates the type of a content library in vCenter. +type ContentLibraryType string + +const ( + // ContentLibraryTypeLocal indicates a local content library in vCenter. + ContentLibraryTypeLocal ContentLibraryType = "Local" + + // ContentLibraryTypeSubscribed indicates a subscribed content library in vCenter. + ContentLibraryTypeSubscribed ContentLibraryType = "Subscribed" +) + +// StorageBackingType is a constant type that indicates the type of the storage backing for a content library in vCenter. +type StorageBackingType string + +const ( + // StorageBackingTypeDatastore indicates a datastore backed content library in vCenter. + StorageBackingTypeDatastore StorageBackingType = "Datastore" + + // StorageBackingTypeOther indicates a remote file system backed content library in vCenter. + // Supports NFS and SMB remote file systems. + StorageBackingTypeOther StorageBackingType = "Other" +) + +// State is a constant type that indicates the current state of a content library in vCenter. +type State string + +const ( + // StateActive indicates the library state when the library should be fully functional, this is the default library + // state when a library is created. + StateActive State = "Active" + + // StateInMaintenance indicates the library state when the library is in maintenance. This can happen when the library + // is storage migrated to a different datastore, in which case content from the library may not be accessible and + // operations mutating library content will be disallowed. + StateInMaintenance State = "InMaintenance" +) + +// StorageBacking describes the default storage backing which is available for the library. +type StorageBacking struct { + // Type indicates the type of storage where the content would be stored. + // +kubebuilder:validation:Enum=Datastore;Other + // +required + Type StorageBackingType `json:"type"` + + // DatastoreID indicates the identifier of the datastore used to store the content + // in the library for the "Datastore" storageType in vCenter. + // +optional + DatastoreID string `json:"datastoreID,omitempty"` +} + +// ResourceNamingStrategy represents a naming strategy for item resources in a content library in vCenter. +type ResourceNamingStrategy string + +const ( + // ResourceNamingStrategyFromItemID indicates the naming strategy that generates the item resource name from the item + // identifier for items in a content library. This is the default naming strategy if not specified on a content library. + ResourceNamingStrategyFromItemID ResourceNamingStrategy = "FROM_ITEM_ID" + + // ResourceNamingStrategyPreferItemSourceID indicates the naming strategy that generates the item resource name from the + // source identifier of the item if it belongs to a subscribed content library, otherwise the item resource name will + // be generated from the item identifier for items in a content library. + ResourceNamingStrategyPreferItemSourceID ResourceNamingStrategy = "PREFER_ITEM_SOURCE_ID" +) + +// SubscriptionInfo defines how the subscribed library synchronizes to a remote source. +type SubscriptionInfo struct { + // URL of the endpoint where the metadata for the remotely published library is being served. + // The value from PublishInfo.URL of the published library should be used while creating a subscribed library. + // +required + URL string `json:"URL"` + + // OnDemand indicates whether a library item’s content will be synchronized only on demand. + // +required + OnDemand bool `json:"onDemand"` + + // AutomaticSync indicates whether the library should participate in automatic library synchronization. + // +required + AutomaticSync bool `json:"automaticSync"` +} + +// PublishInfo defines how the library is published so that it can be subscribed to by a remote subscribed library. +type PublishInfo struct { + // Published indicates if the local library is published so that it can be subscribed to by a remote subscribed library. + // +required + Published bool `json:"published"` + + // URL to which the library metadata is published by the vSphere Content Library Service. + // This value can be used to set the SubscriptionInfo.URL property when creating a subscribed library. + // +required + URL string `json:"URL"` +} + +type BaseContentLibrarySpec struct { + // UUID is the identifier which uniquely identifies the library in vCenter. This field is immutable. + // +required + UUID types.UID `json:"uuid"` + + // +optional + // +kubebuilder:default=FROM_ITEM_ID + + // ResourceNamingStrategy defines the naming strategy for item resources in this content library. If not specified, + // naming strategy FROM_ITEM_ID will be used to generate item resource names. This field is immutable. + // +optional + // +kubebuilder:validation:Enum=FROM_ITEM_ID;PREFER_ITEM_SOURCE_ID + ResourceNamingStrategy ResourceNamingStrategy `json:"resourceNamingStrategy,omitempty"` +} + +// ContentLibrarySpec defines the desired state of a ContentLibrary. +type ContentLibrarySpec struct { + BaseContentLibrarySpec `json:",inline"` + + // Writable flag indicates if users can create new library items in this library. + // +required + Writable bool `json:"writable"` + + // AllowImport flag indicates if users can import OVF/OVA templates from remote HTTPS URLs + // as new content library items in this library. + // +optional + // +kubebuilder:default=false + AllowImport bool `json:"allowImport,omitempty"` +} + +// ContentLibraryStatus defines the observed state of ContentLibrary. +type ContentLibraryStatus struct { + // Name specifies the name of the content library in vCenter. + // +optional + Name string `json:"name,omitempty"` + + // Description is a human-readable description for this library in vCenter. + // +optional + Description string `json:"description,omitempty"` + + // +kubebuilder:validation:Enum=Local;Subscribed + // +optional + Type ContentLibraryType `json:"type,omitempty"` + + // StorageBacking indicates the default storage backing available for this library in vCenter. + // +optional + StorageBacking *StorageBacking `json:"storageBacking,omitempty"` + + // Version is a number that can identify metadata changes. This value is incremented when the library + // properties such as name or description are changed in vCenter. + // +optional + Version string `json:"version,omitempty"` + + // Published indicates how the library is published so that it can be subscribed to by a remote subscribed library. + // +optional + PublishInfo *PublishInfo `json:"publishInfo,omitempty"` + + // SubscriptionInfo defines how the subscribed library synchronizes to a remote source. + // This field is populated only if Type=Subscribed. + // +optional + SubscriptionInfo *SubscriptionInfo `json:"subscriptionInfo,omitempty"` + + // SecurityPolicyID defines the security policy applied to this library. + // Setting this field will make the library secure. + // +optional + SecurityPolicyID string `json:"securityPolicyID,omitempty"` + + // State indicates the state of this library. + // +kubebuilder:validation:Enum=Active;InMaintenance + // +optional + State State `json:"state,omitempty"` + + // ServerGUID indicates the unique identifier of the vCenter server where the library exists. + // +optional + ServerGUID string `json:"serverGUID,omitempty"` + + // CreationTime indicates the date and time when this library was created in vCenter. + // +optional + CreationTime metav1.Time `json:"creationTime,omitempty"` + + // LastModifiedTime indicates the date and time when this library was last updated in vCenter. + // This field is updated only when the library properties are changed. This field is not updated when a library + // item is added, modified or deleted or its content is changed. + // +optional + LastModifiedTime metav1.Time `json:"lastModifiedTime,omitempty"` + + // LastSyncTime indicates the date and time when this library was last synchronized in vCenter. + // This field applies only if the library is of the "Subscribed" Type. + // +optional + LastSyncTime metav1.Time `json:"lastSyncTime,omitempty"` + + // Conditions describes the current condition information of the ContentLibrary. + // +optional + Conditions Conditions `json:"conditions,omitempty"` +} + +func (contentLibrary *ContentLibrary) GetConditions() Conditions { + return contentLibrary.Status.Conditions +} + +func (contentLibrary *ContentLibrary) SetConditions(conditions Conditions) { + contentLibrary.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,shortName=lib;library +// +kubebuilder:printcolumn:name="vSphereName",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".status.type" +// +kubebuilder:printcolumn:name="Writable",type="boolean",JSONPath=".spec.writable" +// +kubebuilder:printcolumn:name="AllowImport",type="boolean",JSONPath=".spec.allowImport" +// +kubebuilder:printcolumn:name="StorageType",type="string",JSONPath=".status.storageBacking.type" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// ContentLibrary is the schema for the content library API. +// Currently, ContentLibrary is immutable to end users. +type ContentLibrary struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ContentLibrarySpec `json:"spec,omitempty"` + Status ContentLibraryStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ContentLibraryList contains a list of ContentLibrary. +type ContentLibraryList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ContentLibrary `json:"items"` +} + +// ClusterContentLibrarySpec defines the desired state of a ClusterContentLibrary. +type ClusterContentLibrarySpec struct { + BaseContentLibrarySpec `json:",inline"` +} + +func (ccl *ClusterContentLibrary) GetConditions() Conditions { + return ccl.Status.Conditions +} + +func (ccl *ClusterContentLibrary) SetConditions(conditions Conditions) { + ccl.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=clib;clibrary +// +kubebuilder:printcolumn:name="vSphereName",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".status.type" +// +kubebuilder:printcolumn:name="StorageType",type="string",JSONPath=".status.storageBacking.type" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// ClusterContentLibrary is the schema for the cluster scoped content library API. +// Currently, ClusterContentLibrary is immutable to end users. +type ClusterContentLibrary struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ClusterContentLibrarySpec `json:"spec,omitempty"` + Status ContentLibraryStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ClusterContentLibraryList contains a list of ClusterContentLibrary. +type ClusterContentLibraryList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ClusterContentLibrary `json:"items"` +} + +func init() { + objectTypes = append( + objectTypes, + &ContentLibrary{}, + &ContentLibraryList{}, + &ClusterContentLibrary{}, + &ClusterContentLibraryList{}, + ) +} diff --git a/external/image-registry-operator/api/v1alpha1/contentlibraryitem_types.go b/external/image-registry-operator/api/v1alpha1/contentlibraryitem_types.go new file mode 100644 index 000000000..273244677 --- /dev/null +++ b/external/image-registry-operator/api/v1alpha1/contentlibraryitem_types.go @@ -0,0 +1,256 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// ContentLibraryItemType is a constant for the type of a content library item in vCenter. +type ContentLibraryItemType string + +// CertVerificationStatus is a constant for the certificate verification status of a content library item in vCenter. +type CertVerificationStatus string + +const ( + // ContentLibraryItemTypeOvf indicates an OVF content library item in vCenter. + ContentLibraryItemTypeOvf ContentLibraryItemType = "OVF" + + // ContentLibraryItemTypeIso indicates an ISO content library item in vCenter. + ContentLibraryItemTypeIso ContentLibraryItemType = "ISO" + + // CertVerificationStatusNotAvailable indicates the certificate verification status is not available. + CertVerificationStatusNotAvailable CertVerificationStatus = "NOT_AVAILABLE" + + // CertVerificationStatusVerified indicates the library item has been fully validated during importing or file syncing. + CertVerificationStatusVerified CertVerificationStatus = "VERIFIED" + + // CertVerificationStatusInternal indicates the library item is cloned/created through vCenter. + CertVerificationStatusInternal CertVerificationStatus = "INTERNAL" + + // CertVerificationStatusVerificationFailure indicates certificate or manifest validation failed on the library item. + CertVerificationStatusVerificationFailure CertVerificationStatus = "VERIFICATION_FAILURE" + + // CertVerificationStatusVerificationInProgress indicates the library item certificate verification is in progress. + CertVerificationStatusVerificationInProgress CertVerificationStatus = "VERIFICATION_IN_PROGRESS" + + // CertVerificationStatusUntrusted indicates the certificate used to sign the library item is not trusted. + CertVerificationStatusUntrusted CertVerificationStatus = "UNTRUSTED" +) + +// CertificateVerificationInfo shows the certificate verification status and the signing certificate. +type CertificateVerificationInfo struct { + // Status shows the certificate verification status of the library item. + // +kubebuilder:validation:Enum=NOT_AVAILABLE;VERIFIED;INTERNAL;VERIFICATION_FAILURE;VERIFICATION_IN_PROGRESS;UNTRUSTED + // +optional + Status CertVerificationStatus `json:"status,omitempty"` + + // CertChain shows the signing certificate chain in base64 encoding if the library item is signed. + // +optional + CertChain []string `json:"certChain,omitempty"` +} + +// FileInfo represents the information of a file in a content library item in vCenter. +type FileInfo struct { + // +required + + // Name specifies the name of the file in vCenter. + Name string `json:"name"` + + // +required + + // SizeInBytes indicates the library item file size in bytes on storage in vCenter. + SizeInBytes resource.Quantity `json:"sizeInBytes"` + + // +required + + // Version indicates the version of the library item file in vCenter. + // This value is incremented when a new copy of the file is uploaded to vCenter. + Version string `json:"version"` + + // +required + // +kubebuilder:default=false + + // Cached indicates if the library item file is on storage in vCenter. + Cached bool `json:"cached"` + + // +optional + + // StorageURI identifies the file on the storage backing. It is specific to + // the storage backing and available after the file is cached in vCenter. + // This URL is useful for creating a device that is backed by this file + // (i.e. mounting an ISO file via a virtual CD-ROM device). + StorageURI string `json:"storageURI,omitempty"` +} + +// ContentLibraryItemSpec defines the desired state of a ContentLibraryItem. +type ContentLibraryItemSpec struct { + // UUID is the identifier which uniquely identifies the library item in vCenter. This field is immutable. + // +required + UUID types.UID `json:"uuid"` +} + +// ContentLibraryItemStatus defines the observed state of ContentLibraryItem. +type ContentLibraryItemStatus struct { + // Name specifies the name of the content library item in vCenter specified by the user. + // +optional + Name string `json:"name,omitempty"` + + // ContentLibraryRef refers to the ContentLibrary custom resource that this item belongs to. + // +optional + ContentLibraryRef *NameAndKindRef `json:"contentLibraryRef,omitempty"` + + // Description is a human-readable description for this library item. + // +optional + Description string `json:"description,omitempty"` + + // MetadataVersion indicates the version of the library item metadata in vCenter. + // This value is incremented when the library item properties such as name or description are changed in vCenter. + // +optional + MetadataVersion string `json:"metadataVersion,omitempty"` + + // ContentVersion indicates the version of the library item content in vCenter. + // This value is incremented when the files comprising the content library item are changed in vCenter. + // +optional + ContentVersion string `json:"contentVersion,omitempty"` + + // Type indicates the type of the library item in vCenter. + // +kubebuilder:validation:Enum=OVF;ISO + // +optional + Type ContentLibraryItemType `json:"type,omitempty"` + + // SourceID indicates the source identifier of the library item if the item belongs to a subscribed content library. + // +optional + SourceID types.UID `json:"sourceID,omitempty"` + + // SizeInBytes indicates the library item size in bytes on storage in vCenter. + // +optional + SizeInBytes resource.Quantity `json:"sizeInBytes,omitempty"` + + // Cached indicates if the library item files are on storage in vCenter. + // +optional + // +kubebuilder:default=false + Cached bool `json:"cached,omitempty"` + + // SecurityCompliance shows the security compliance of the library item. + // +optional + SecurityCompliance *bool `json:"securityCompliance,omitempty"` + + // CertificateVerificationInfo shows the certificate verification status and the signing certificate. + // +optional + CertificateVerificationInfo *CertificateVerificationInfo `json:"certificateVerificationInfo,omitempty"` + + // FileInfo represents zero, one or more files belonging to the content library item in vCenter. + // +optional + FileInfo []FileInfo `json:"fileInfo,omitempty"` + + // CreationTime indicates the date and time when this library item was created in vCenter. + // +optional + CreationTime metav1.Time `json:"creationTime,omitempty"` + + // LastModifiedTime indicates the date and time when this library item was last updated in vCenter. + // This field is updated when the library item properties are changed or the file content is changed. + // +optional + LastModifiedTime metav1.Time `json:"lastModifiedTime,omitempty"` + + // LastSyncTime indicates the date and time when this library item was last synchronized in vCenter. + // This field applies only to the library items belonging to the library of Type=Subscribed. + // +optional + LastSyncTime metav1.Time `json:"lastSyncTime,omitempty"` + + // Conditions describes the current condition information of the ContentLibraryItem. + // +optional + Conditions Conditions `json:"conditions,omitempty"` +} + +func (contentLibraryItem *ContentLibraryItem) GetConditions() Conditions { + return contentLibraryItem.Status.Conditions +} + +func (contentLibraryItem *ContentLibraryItem) SetConditions(conditions Conditions) { + contentLibraryItem.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,shortName=libitem +// +kubebuilder:printcolumn:name="vSphereName",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="ContentLibraryRef",type="string",JSONPath=".status.contentLibraryRef.name" +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".status.type" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(.type=='Ready')].status" +// +kubebuilder:printcolumn:name="Cached",type="boolean",JSONPath=".status.cached" +// +kubebuilder:printcolumn:name="SizeInBytes",type="string",JSONPath=".status.sizeInBytes" +// +kubebuilder:printcolumn:name="SecurityCompliant",type="boolean",JSONPath=".status.securityCompliance" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// ContentLibraryItem is the schema for the content library item API. +// Currently, ContentLibraryItem is immutable to end users. +type ContentLibraryItem struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ContentLibraryItemSpec `json:"spec,omitempty"` + Status ContentLibraryItemStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ContentLibraryItemList contains a list of ContentLibraryItem. +type ContentLibraryItemList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ContentLibraryItem `json:"items"` +} + +func (cclItem *ClusterContentLibraryItem) GetConditions() Conditions { + return cclItem.Status.Conditions +} + +func (cclItem *ClusterContentLibraryItem) SetConditions(conditions Conditions) { + cclItem.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=clibitem +// +kubebuilder:printcolumn:name="vSphereName",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="ClusterContentLibraryRef",type="string",JSONPath=".status.contentLibraryRef.name" +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".status.type" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(.type=='Ready')].status" +// +kubebuilder:printcolumn:name="Cached",type="boolean",JSONPath=".status.cached" +// +kubebuilder:printcolumn:name="SizeInBytes",type="string",JSONPath=".status.sizeInBytes" +// +kubebuilder:printcolumn:name="SecurityCompliant",type="boolean",JSONPath=".status.securityCompliance" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// ClusterContentLibraryItem is the schema for the content library item API at the cluster scope. +// Currently, ClusterContentLibraryItem is immutable to end users. +type ClusterContentLibraryItem struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ContentLibraryItemSpec `json:"spec,omitempty"` + Status ContentLibraryItemStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ClusterContentLibraryItemList contains a list of ClusterContentLibraryItem. +type ClusterContentLibraryItemList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ClusterContentLibraryItem `json:"items"` +} + +func init() { + objectTypes = append( + objectTypes, + &ContentLibraryItem{}, + &ContentLibraryItemList{}, + &ClusterContentLibraryItem{}, + &ClusterContentLibraryItemList{}, + ) +} diff --git a/external/image-registry-operator/api/v1alpha1/contentlibraryitemimportrequest_types.go b/external/image-registry-operator/api/v1alpha1/contentlibraryitemimportrequest_types.go new file mode 100644 index 000000000..213b61592 --- /dev/null +++ b/external/image-registry-operator/api/v1alpha1/contentlibraryitemimportrequest_types.go @@ -0,0 +1,259 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// TransferStatus is a constant that indicates the transfer state of a file. +type TransferStatus string + +const ( + // TransferStatusWaiting indicates that the file is waiting to be transferred. + TransferStatusWaiting TransferStatus = "waiting" + + // TransferStatusTransferring indicates that the data of the file is being transferred. + TransferStatusTransferring TransferStatus = "transferring" + + // TransferStatusValidating indicates that the file is being validated. + TransferStatusValidating TransferStatus = "validating" + + // TransferStatusReady indicates that the file has been fully transferred and is ready to be used. + TransferStatusReady TransferStatus = "ready" + + // TransferStatusError indicates there was an error transferring or validating the file. + TransferStatusError TransferStatus = "error" +) + +// ContentLibraryItemImportRequestSource contains the specification of the source for the import request. +type ContentLibraryItemImportRequestSource struct { + // +required + + // URL is the endpoint that points to a file that is to be imported as a new Content Library Item in + // the target vSphere Content Library. If the target item type is ContentLibraryItemTypeOvf, the URL + // should point to an OVF descriptor file (.ovf), an OVA file (.ova), or an ISO file (.iso). Otherwise, + // the SourceValid condition will become false in the status. + URL string `json:"url"` + + // +optional + + // PEM encoded SSL Certificate for this endpoint specified by the URL. It is only used for HTTPS connections. + // If set, the remote endpoint's SSL certificate is only accepted if it matches this certificate, and no other + // certificate validation is performed. + // If unset, the remote endpoint's SSL certificate must be trusted by vSphere trusted root CA certificates, + // otherwise the SSL certification verification may fail and thus fail the import request. + SSLCertificate string `json:"sslCertificate,omitempty"` + + // +optional + + // Checksum contains the checksum algorithm and value calculated for the + // file specified in the URL. If omitted, the import request will not verify + // the checksum of the file. + Checksum *Checksum `json:"checksum,omitempty"` +} + +// Checksum contains the checksum value and algorithm used to calculate that +// value. +type Checksum struct { + // +optional + // +kubebuilder:validation:Enum=SHA256;SHA512 + // +kubebuilder:default=SHA256 + + // Algorithm is the algorithm used to calculate the checksum. Supported + // algorithms are "SHA256" and "SHA512". If omitted, "SHA256" will be used + // as the default algorithm. + Algorithm string `json:"algorithm"` + + // +required + + // Value is the checksum value calculated by the specified algorithm. + Value string `json:"value"` +} + +// ContentLibraryItemImportRequestTargetItem contains the specification of the target +// content library item for the import request. +type ContentLibraryItemImportRequestTargetItem struct { + // Name is the name of the new content library item that will be created in vSphere. + // If omitted, the content library item will be created with the same name as the name + // of the image specified in the spec.source.url in the specified vSphere Content Library. + // If an item with the same name already exists in the specified vSphere Content Library, + // the TargetValid condition will become false in the status. + // +optional + Name string `json:"name,omitempty"` + + // Description is a description for a vSphere Content Library Item. + // +optional + Description string `json:"description,omitempty"` + + // Type is the type of the new content library item that will be created in vSphere. + // Currently only ContentLibraryItemTypeOvf is supported, if it is omitted or other item type + // is specified, the TargetValid condition will become false in the status. For the item type + // of ContentLibraryItemTypeOvf, it is required that the default OVF security policy is configured + // on the target content library for the import request, otherwise the TargetValid condition will + // become false in the status. + // +optional + Type ContentLibraryItemType `json:"type,omitempty"` +} + +// ContentLibraryItemImportRequestTarget is the target specification of an import request. +type ContentLibraryItemImportRequestTarget struct { + // Item contains information about the content library item to which + // the template will be imported in vSphere. + // If omitted, the content library item will be created with the same name as the name + // of the image specified in the spec.source.url in the specified vSphere Content Library. + // If an item with the same name already exists in the specified vSphere Content Library, + // the TargetValid condition will become false in the status. + // +optional + Item ContentLibraryItemImportRequestTargetItem `json:"item,omitempty"` + + // Library contains information about the library in which the library item + // will be created in vSphere. + // +required + Library LocalObjectRef `json:"library"` +} + +// ContentLibraryItemImportRequestSpec defines the desired state of a +// ContentLibraryItemImportRequest. +type ContentLibraryItemImportRequestSpec struct { + // Source is the source of the import request which includes an external URL + // pointing to a VM image template. + // Source and Target will be immutable if the SourceValid and TargetValid conditions are true. + // +required + Source ContentLibraryItemImportRequestSource `json:"source"` + + // Target is the target of the import request which includes the content library item + // information and a ContentLibrary resource. + // Source and Target will be immutable if the SourceValid and TargetValid conditions are true. + // +required + Target ContentLibraryItemImportRequestTarget `json:"target"` + + // TTLSecondsAfterFinished is the time-to-live duration for how long this + // resource will be allowed to exist once the import operation + // completes. After the TTL expires, the resource will be automatically + // deleted without the user having to take any direct action. + // If this field is unset then the request resource will not be + // automatically deleted. If this field is set to zero then the request + // resource is eligible for deletion immediately after it finishes. + // +optional + // +kubebuilder:validation:Minimum=0 + TTLSecondsAfterFinished *int64 `json:"ttlSecondsAfterFinished,omitempty"` +} + +// ContentLibraryItemFileUploadStatus indicates the upload status of files belonging to the template. +type ContentLibraryItemFileUploadStatus struct { + // SessionUUID is the identifier that uniquely identifies the file upload session on the library item in vSphere. + // +required + SessionUUID types.UID `json:"sessionUUID,omitempty"` + + // FileUploads list the transfer statuses of files being uploaded and tracked by the upload session. + // +optional + FileUploads []FileTransferStatus `json:"fileUploads,omitempty"` +} + +// FileTransferStatus indicates the transfer status of a file belonging to a library item. +type FileTransferStatus struct { + // Name specifies the name of the file that is transferred. + // +required + Name string `json:"name"` + + // Status indicates the transfer status of the file. + // +required + Status TransferStatus `json:"transferStatus"` + + // BytesTransferred indicates the number of bytes of this file that have been received by the server. + // +optional + BytesTransferred *int64 `json:"bytesTransferred,omitempty"` + + // Size indicates the file size in bytes as received by the server, this won't be available + // until the transfer status is ready. + // +optional + Size *int64 `json:"size,omitempty"` + + // ErrorMessage describes the details about the transfer error if the transfer status is error. + // +optional + ErrorMessage string `json:"errorMessage,omitempty"` +} + +// ContentLibraryItemImportRequestStatus defines the observed state of a +// ContentLibraryItemImportRequest. +type ContentLibraryItemImportRequestStatus struct { + // ItemRef is the reference to the target ContentLibraryItem resource of the import request. + // If the ContentLibraryItemImportRequest is deleted when the import operation fails or before + // the Complete condition is set to true, the import operation will be cancelled in vSphere + // and the corresponding vSphere Content Library Item will be deleted. + // +optional + ItemRef *LocalObjectRef `json:"itemRef,omitempty"` + + // CompletionTime represents time when the request was completed. + // The value of this field should be equal to the value of the + // LastTransitionTime for the status condition Type=Complete. + // +optional + CompletionTime metav1.Time `json:"completionTime,omitempty"` + + // StartTime represents time when the request was acknowledged by the + // controller. + // +optional + StartTime metav1.Time `json:"startTime,omitempty"` + + // FileUpload indicates the upload status of files belonging to the template. + // +optional + FileUploadStatus *ContentLibraryItemFileUploadStatus `json:"fileUploadStatus,omitempty"` + + // Conditions describes the current condition information of the ContentLibraryItemImportRequest. + // The conditions present will be: + // * SourceValid + // * TargetValid + // * ContentLibraryItemCreated + // * TemplateUploaded + // * ContentLibraryItemReady + // * Complete + // +optional + Conditions []Condition `json:"conditions,omitempty"` +} + +func (clItemImportRequest *ContentLibraryItemImportRequest) GetConditions() Conditions { + return clItemImportRequest.Status.Conditions +} + +func (clItemImportRequest *ContentLibraryItemImportRequest) SetConditions(conditions Conditions) { + clItemImportRequest.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,shortName=libitemimport +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="ContentLibraryRef",type="string",JSONPath=".spec.target.library.name" +// +kubebuilder:printcolumn:name="ContentLibraryItemRef",type="string",JSONPath=".status.itemRef.name" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(.type=='Complete')].status" + +// ContentLibraryItemImportRequest defines the information necessary to import a VM image +// template as a ContentLibraryItem to a Content Library in vSphere. +type ContentLibraryItemImportRequest struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ContentLibraryItemImportRequestSpec `json:"spec,omitempty"` + Status ContentLibraryItemImportRequestStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ContentLibraryItemImportRequestList contains a list of +// ContentLibraryItemImportRequest resources. +type ContentLibraryItemImportRequestList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ContentLibraryItemImportRequest `json:"items"` +} + +func init() { + objectTypes = append( + objectTypes, + &ContentLibraryItemImportRequest{}, + &ContentLibraryItemImportRequestList{}, + ) +} diff --git a/external/image-registry-operator/api/v1alpha1/doc.go b/external/image-registry-operator/api/v1alpha1/doc.go new file mode 100644 index 000000000..e35e749da --- /dev/null +++ b/external/image-registry-operator/api/v1alpha1/doc.go @@ -0,0 +1,11 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +// +k8s:openapi-gen=true +// +kubebuilder:object:generate=true +// +groupName=imageregistry.vmware.com +// +k8s:conversion-gen=github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha2 + +// Package v1alpha1 is one of the schemas for Image Registry Operator. +package v1alpha1 diff --git a/external/image-registry-operator/api/v1alpha1/groupversion_info.go b/external/image-registry-operator/api/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..fdf908a5c --- /dev/null +++ b/external/image-registry-operator/api/v1alpha1/groupversion_info.go @@ -0,0 +1,36 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName specifies the group name used to register the objects. +const GroupName = "imageregistry.vmware.com" + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} + + // schemeBuilder is used to add go types to the GroupVersionKind scheme. + schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = schemeBuilder.AddToScheme + + objectTypes = []runtime.Object{} + + // localSchemeBuilder is used for type conversions. + localSchemeBuilder = schemeBuilder +) + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(GroupVersion, objectTypes...) + metav1.AddToGroupVersion(scheme, GroupVersion) + return nil +} diff --git a/external/image-registry-operator/api/v1alpha1/types.go b/external/image-registry-operator/api/v1alpha1/types.go new file mode 100644 index 000000000..8be972faf --- /dev/null +++ b/external/image-registry-operator/api/v1alpha1/types.go @@ -0,0 +1,36 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +// NameAndKindRef describes a reference to another object in the same +// namespace as the referrer. The reference can be just a name but may also +// include the referred resource's Kind. +type NameAndKindRef struct { + // Kind is a string value representing the kind of resource to which this + // object refers. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + // +optional + Kind string `json:"kind"` + + // Name refers to a unique resource in the current namespace. + // More info: http://kubernetes.io/docs/user-guide/identifiers#names + Name string `json:"name"` +} + +// LocalObjectRef describes a reference to another object in the same namespace as the referrer. +type LocalObjectRef struct { + // APIVersion defines the versioned schema of this representation of an + // object. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + APIVersion string `json:"apiVersion"` + + // Kind is a string value representing the kind of resource to which this + // object refers. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + Kind string `json:"kind"` + + // Name refers to a unique resource in the current namespace. + // More info: http://kubernetes.io/docs/user-guide/identifiers#names + Name string `json:"name"` +} diff --git a/external/image-registry-operator/api/v1alpha1/zz_generated.deepcopy.go b/external/image-registry-operator/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..b41464f57 --- /dev/null +++ b/external/image-registry-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,775 @@ +//go:build !ignore_autogenerated + +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BaseContentLibrarySpec) DeepCopyInto(out *BaseContentLibrarySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BaseContentLibrarySpec. +func (in *BaseContentLibrarySpec) DeepCopy() *BaseContentLibrarySpec { + if in == nil { + return nil + } + out := new(BaseContentLibrarySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificateVerificationInfo) DeepCopyInto(out *CertificateVerificationInfo) { + *out = *in + if in.CertChain != nil { + in, out := &in.CertChain, &out.CertChain + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificateVerificationInfo. +func (in *CertificateVerificationInfo) DeepCopy() *CertificateVerificationInfo { + if in == nil { + return nil + } + out := new(CertificateVerificationInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Checksum) DeepCopyInto(out *Checksum) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Checksum. +func (in *Checksum) DeepCopy() *Checksum { + if in == nil { + return nil + } + out := new(Checksum) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterContentLibrary) DeepCopyInto(out *ClusterContentLibrary) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterContentLibrary. +func (in *ClusterContentLibrary) DeepCopy() *ClusterContentLibrary { + if in == nil { + return nil + } + out := new(ClusterContentLibrary) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterContentLibrary) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterContentLibraryItem) DeepCopyInto(out *ClusterContentLibraryItem) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterContentLibraryItem. +func (in *ClusterContentLibraryItem) DeepCopy() *ClusterContentLibraryItem { + if in == nil { + return nil + } + out := new(ClusterContentLibraryItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterContentLibraryItem) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterContentLibraryItemList) DeepCopyInto(out *ClusterContentLibraryItemList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterContentLibraryItem, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterContentLibraryItemList. +func (in *ClusterContentLibraryItemList) DeepCopy() *ClusterContentLibraryItemList { + if in == nil { + return nil + } + out := new(ClusterContentLibraryItemList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterContentLibraryItemList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterContentLibraryList) DeepCopyInto(out *ClusterContentLibraryList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterContentLibrary, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterContentLibraryList. +func (in *ClusterContentLibraryList) DeepCopy() *ClusterContentLibraryList { + if in == nil { + return nil + } + out := new(ClusterContentLibraryList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterContentLibraryList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterContentLibrarySpec) DeepCopyInto(out *ClusterContentLibrarySpec) { + *out = *in + out.BaseContentLibrarySpec = in.BaseContentLibrarySpec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterContentLibrarySpec. +func (in *ClusterContentLibrarySpec) DeepCopy() *ClusterContentLibrarySpec { + if in == nil { + return nil + } + out := new(ClusterContentLibrarySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Condition) DeepCopyInto(out *Condition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Condition. +func (in *Condition) DeepCopy() *Condition { + if in == nil { + return nil + } + out := new(Condition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in Conditions) DeepCopyInto(out *Conditions) { + { + in := &in + *out = make(Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Conditions. +func (in Conditions) DeepCopy() Conditions { + if in == nil { + return nil + } + out := new(Conditions) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibrary) DeepCopyInto(out *ContentLibrary) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibrary. +func (in *ContentLibrary) DeepCopy() *ContentLibrary { + if in == nil { + return nil + } + out := new(ContentLibrary) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibrary) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItem) DeepCopyInto(out *ContentLibraryItem) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItem. +func (in *ContentLibraryItem) DeepCopy() *ContentLibraryItem { + if in == nil { + return nil + } + out := new(ContentLibraryItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibraryItem) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemFileUploadStatus) DeepCopyInto(out *ContentLibraryItemFileUploadStatus) { + *out = *in + if in.FileUploads != nil { + in, out := &in.FileUploads, &out.FileUploads + *out = make([]FileTransferStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemFileUploadStatus. +func (in *ContentLibraryItemFileUploadStatus) DeepCopy() *ContentLibraryItemFileUploadStatus { + if in == nil { + return nil + } + out := new(ContentLibraryItemFileUploadStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequest) DeepCopyInto(out *ContentLibraryItemImportRequest) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequest. +func (in *ContentLibraryItemImportRequest) DeepCopy() *ContentLibraryItemImportRequest { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibraryItemImportRequest) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestList) DeepCopyInto(out *ContentLibraryItemImportRequestList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ContentLibraryItemImportRequest, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestList. +func (in *ContentLibraryItemImportRequestList) DeepCopy() *ContentLibraryItemImportRequestList { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibraryItemImportRequestList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestSource) DeepCopyInto(out *ContentLibraryItemImportRequestSource) { + *out = *in + if in.Checksum != nil { + in, out := &in.Checksum, &out.Checksum + *out = new(Checksum) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestSource. +func (in *ContentLibraryItemImportRequestSource) DeepCopy() *ContentLibraryItemImportRequestSource { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestSpec) DeepCopyInto(out *ContentLibraryItemImportRequestSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + out.Target = in.Target + if in.TTLSecondsAfterFinished != nil { + in, out := &in.TTLSecondsAfterFinished, &out.TTLSecondsAfterFinished + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestSpec. +func (in *ContentLibraryItemImportRequestSpec) DeepCopy() *ContentLibraryItemImportRequestSpec { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestStatus) DeepCopyInto(out *ContentLibraryItemImportRequestStatus) { + *out = *in + if in.ItemRef != nil { + in, out := &in.ItemRef, &out.ItemRef + *out = new(LocalObjectRef) + **out = **in + } + in.CompletionTime.DeepCopyInto(&out.CompletionTime) + in.StartTime.DeepCopyInto(&out.StartTime) + if in.FileUploadStatus != nil { + in, out := &in.FileUploadStatus, &out.FileUploadStatus + *out = new(ContentLibraryItemFileUploadStatus) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestStatus. +func (in *ContentLibraryItemImportRequestStatus) DeepCopy() *ContentLibraryItemImportRequestStatus { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestTarget) DeepCopyInto(out *ContentLibraryItemImportRequestTarget) { + *out = *in + out.Item = in.Item + out.Library = in.Library +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestTarget. +func (in *ContentLibraryItemImportRequestTarget) DeepCopy() *ContentLibraryItemImportRequestTarget { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestTargetItem) DeepCopyInto(out *ContentLibraryItemImportRequestTargetItem) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestTargetItem. +func (in *ContentLibraryItemImportRequestTargetItem) DeepCopy() *ContentLibraryItemImportRequestTargetItem { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestTargetItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemList) DeepCopyInto(out *ContentLibraryItemList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ContentLibraryItem, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemList. +func (in *ContentLibraryItemList) DeepCopy() *ContentLibraryItemList { + if in == nil { + return nil + } + out := new(ContentLibraryItemList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibraryItemList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemSpec) DeepCopyInto(out *ContentLibraryItemSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemSpec. +func (in *ContentLibraryItemSpec) DeepCopy() *ContentLibraryItemSpec { + if in == nil { + return nil + } + out := new(ContentLibraryItemSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemStatus) DeepCopyInto(out *ContentLibraryItemStatus) { + *out = *in + if in.ContentLibraryRef != nil { + in, out := &in.ContentLibraryRef, &out.ContentLibraryRef + *out = new(NameAndKindRef) + **out = **in + } + out.SizeInBytes = in.SizeInBytes.DeepCopy() + if in.SecurityCompliance != nil { + in, out := &in.SecurityCompliance, &out.SecurityCompliance + *out = new(bool) + **out = **in + } + if in.CertificateVerificationInfo != nil { + in, out := &in.CertificateVerificationInfo, &out.CertificateVerificationInfo + *out = new(CertificateVerificationInfo) + (*in).DeepCopyInto(*out) + } + if in.FileInfo != nil { + in, out := &in.FileInfo, &out.FileInfo + *out = make([]FileInfo, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.CreationTime.DeepCopyInto(&out.CreationTime) + in.LastModifiedTime.DeepCopyInto(&out.LastModifiedTime) + in.LastSyncTime.DeepCopyInto(&out.LastSyncTime) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemStatus. +func (in *ContentLibraryItemStatus) DeepCopy() *ContentLibraryItemStatus { + if in == nil { + return nil + } + out := new(ContentLibraryItemStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryList) DeepCopyInto(out *ContentLibraryList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ContentLibrary, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryList. +func (in *ContentLibraryList) DeepCopy() *ContentLibraryList { + if in == nil { + return nil + } + out := new(ContentLibraryList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibraryList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibrarySpec) DeepCopyInto(out *ContentLibrarySpec) { + *out = *in + out.BaseContentLibrarySpec = in.BaseContentLibrarySpec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibrarySpec. +func (in *ContentLibrarySpec) DeepCopy() *ContentLibrarySpec { + if in == nil { + return nil + } + out := new(ContentLibrarySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryStatus) DeepCopyInto(out *ContentLibraryStatus) { + *out = *in + if in.StorageBacking != nil { + in, out := &in.StorageBacking, &out.StorageBacking + *out = new(StorageBacking) + **out = **in + } + if in.PublishInfo != nil { + in, out := &in.PublishInfo, &out.PublishInfo + *out = new(PublishInfo) + **out = **in + } + if in.SubscriptionInfo != nil { + in, out := &in.SubscriptionInfo, &out.SubscriptionInfo + *out = new(SubscriptionInfo) + **out = **in + } + in.CreationTime.DeepCopyInto(&out.CreationTime) + in.LastModifiedTime.DeepCopyInto(&out.LastModifiedTime) + in.LastSyncTime.DeepCopyInto(&out.LastSyncTime) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryStatus. +func (in *ContentLibraryStatus) DeepCopy() *ContentLibraryStatus { + if in == nil { + return nil + } + out := new(ContentLibraryStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FileInfo) DeepCopyInto(out *FileInfo) { + *out = *in + out.SizeInBytes = in.SizeInBytes.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileInfo. +func (in *FileInfo) DeepCopy() *FileInfo { + if in == nil { + return nil + } + out := new(FileInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FileTransferStatus) DeepCopyInto(out *FileTransferStatus) { + *out = *in + if in.BytesTransferred != nil { + in, out := &in.BytesTransferred, &out.BytesTransferred + *out = new(int64) + **out = **in + } + if in.Size != nil { + in, out := &in.Size, &out.Size + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileTransferStatus. +func (in *FileTransferStatus) DeepCopy() *FileTransferStatus { + if in == nil { + return nil + } + out := new(FileTransferStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalObjectRef) DeepCopyInto(out *LocalObjectRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalObjectRef. +func (in *LocalObjectRef) DeepCopy() *LocalObjectRef { + if in == nil { + return nil + } + out := new(LocalObjectRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NameAndKindRef) DeepCopyInto(out *NameAndKindRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NameAndKindRef. +func (in *NameAndKindRef) DeepCopy() *NameAndKindRef { + if in == nil { + return nil + } + out := new(NameAndKindRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PublishInfo) DeepCopyInto(out *PublishInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PublishInfo. +func (in *PublishInfo) DeepCopy() *PublishInfo { + if in == nil { + return nil + } + out := new(PublishInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageBacking) DeepCopyInto(out *StorageBacking) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageBacking. +func (in *StorageBacking) DeepCopy() *StorageBacking { + if in == nil { + return nil + } + out := new(StorageBacking) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubscriptionInfo) DeepCopyInto(out *SubscriptionInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubscriptionInfo. +func (in *SubscriptionInfo) DeepCopy() *SubscriptionInfo { + if in == nil { + return nil + } + out := new(SubscriptionInfo) + in.DeepCopyInto(out) + return out +} diff --git a/external/image-registry-operator/api/v1alpha2/condition_constants.go b/external/image-registry-operator/api/v1alpha2/condition_constants.go new file mode 100644 index 000000000..d96bfc5fb --- /dev/null +++ b/external/image-registry-operator/api/v1alpha2/condition_constants.go @@ -0,0 +1,81 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha2 + +// Common ConditionTypes used by Image Registry Operator API objects. +const ( + // ReadyCondition defines the Ready condition type that summarizes the operational state of an Image Registry Operator API object. + ReadyCondition = "Ready" +) + +// Common Condition.Reason used by Image Registry Operator API objects. +const ( + // DeletingReason (Severity=Info) documents a condition not in Status=True because the object is currently being deleted. + DeletingReason = "Deleting" + + // DeletedReason (Severity=Error) documents a condition not in Status=True because the underlying object was deleted. + DeletedReason = "Deleted" +) + +// Condition.Reasons related to ClusterContentLibraryItem or ContentLibraryItem API objects. +const ( + ClusterContentLibraryRefValidationFailedReason = "ClusterContentLibraryRefValidationFailed" + ContentLibraryRefValidationFailedReason = "ContentLibraryRefValidationFailed" + ContentLibraryItemFileUnavailableReason = "ContentLibraryItemFileUnavailable" +) + +// ConditionTypes used by ContentLibraryItemImportRequest API objects. +const ( + ContentLibraryItemImportRequestSourceValid = "SourceValid" + ContentLibraryItemImportRequestTargetValid = "TargetValid" + ContentLibraryItemImportRequestContentLibraryItemCreated = "ContentLibraryItemCreated" + ContentLibraryItemImportRequestTemplateUploaded = "TemplateUploaded" + ContentLibraryItemImportRequestContentLibraryItemReady = "ContentLibraryItemReady" + ContentLibraryItemImportRequestComplete = "Complete" +) + +// Condition.Reasons related to ContentLibraryItemImportRequest API objects. +const ( + // SourceURLInvalidReason documents that the source URL specified in the ContentLibraryItemImportRequest is invalid. + SourceURLInvalidReason = "SourceURLInvalid" + + // SourceURLSchemeInvalidReason documents that the scheme in the source URL specified in the + // ContentLibraryItemImportRequest is invalid. + SourceURLSchemeInvalidReason = "SourceURLSchemeInvalid" + + // SourceURLHostInvalidReason documents that the host in the source URL specified in the + // ContentLibraryItemImportRequest is invalid. + SourceURLHostInvalidReason = "SourceURLHostInvalid" + + // SourceSSLCertificateUntrustedReason documents that the SSL certificate served at the source URL is not trusted by vSphere. + SourceSSLCertificateUntrustedReason = "SourceSSLCertificateUntrusted" + + // TargetLibraryInvalidReason documents that the target ContentLibrary specified in the + // ContentLibraryItemImportRequest is invalid. + TargetLibraryInvalidReason = "TargetLibraryInvalid" + + // TargetLibraryItemInvalidReason documents that the specified target content library item in the + // ContentLibraryItemImportRequest is invalid. + TargetLibraryItemInvalidReason = "TargetLibraryItemInvalid" + + // TargetLibraryItemCreationFailureReason documents that the creation of the target ContentLibraryItem has failed. + TargetLibraryItemCreationFailureReason = "TargetLibraryItemCreationFailure" + + // TargetLibraryItemUnavailableReason documents that the target ContentLibraryItem resource is not available in the + // namespace yet. + TargetLibraryItemUnavailableReason = "TargetLibraryItemUnavailable" + + // UploadInProgressReason documents that the files are in progress being uploaded to the target ContentLibraryItem. + UploadInProgressReason = "UploadInProgress" + + // UploadFailureReason documents that uploading files to the target content library item has failed. + UploadFailureReason = "UploadFailure" + + // UploadSessionExpiredReason documents that the uploading session for this ContentLibraryItemImportRequest has expired. + UploadSessionExpiredReason = "UploadSessionExpired" + + // TargetLibraryItemNotReadyReason documents that the target ContentLibraryItem resource is not in ready status yet. + TargetLibraryItemNotReadyReason = "TargetLibraryItemNotReady" +) diff --git a/external/image-registry-operator/api/v1alpha2/contentlibrary_types.go b/external/image-registry-operator/api/v1alpha2/contentlibrary_types.go new file mode 100644 index 000000000..8b7bce8d2 --- /dev/null +++ b/external/image-registry-operator/api/v1alpha2/contentlibrary_types.go @@ -0,0 +1,353 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +kubebuilder:validation:Enum=Datastore;Other + +// StorageBackingType is a constant type that indicates the type of the storage +// backing for a content library in vCenter. +type StorageBackingType string + +const ( + // StorageBackingTypeDatastore indicates a content library backed by a + // datastore. + StorageBackingTypeDatastore StorageBackingType = "Datastore" + + // StorageBackingTypeOther indicates a content library backed by an NFS or + // SMB file system. + StorageBackingTypeOther StorageBackingType = "Other" +) + +// +kubebuilder:validation:Enum=Active;InMaintenance + +// State is a constant type that indicates the current state of a content +// library in vCenter. +type State string + +const ( + // StateActive indicates the library state when the library should be fully + // functional, this is the default library state when a library is created. + StateActive State = "Active" + + // StateInMaintenance indicates the library state when the library is in + // maintenance. This can happen when the library is storage migrated to a + // different datastore, in which case content from the library may not be + // accessible and operations mutating library content will be disallowed. + StateInMaintenance State = "InMaintenance" +) + +// StorageBacking describes the default storage backing which is available for +// the library. +type StorageBacking struct { + // Type indicates the type of storage where the content would be stored. + Type StorageBackingType `json:"type"` + + // +optional + + // DatastoreID indicates the identifier of the datastore used to store the + // content in the library for the "Datastore" storageType in vCenter. + DatastoreID string `json:"datastoreID,omitempty"` +} + +// +kubebuilder:validation:Enum=FromItemID;PreferItemSourceID + +// ResourceNamingStrategy represents a naming strategy for item resources in a +// content library in vCenter. +type ResourceNamingStrategy string + +const ( + // ResourceNamingStrategyFromItemID indicates the naming strategy that + // generates the item resource name from the item identifier for items in a + // content library. This is the default naming strategy if not specified on + // a content library. + ResourceNamingStrategyFromItemID ResourceNamingStrategy = "FromItemID" + + // ResourceNamingStrategyPreferItemSourceID indicates the naming strategy + // that generates the item resource name from the source identifier of the + // item if it belongs to a subscribed content library, otherwise the item + // resource name will be generated from the item identifier for items in a + // content library. + ResourceNamingStrategyPreferItemSourceID ResourceNamingStrategy = "PreferItemSourceID" +) + +// SubscriptionInfo defines how the subscribed library synchronizes to a remote +// source. +type SubscriptionInfo struct { + // URL of the endpoint where the metadata for the remotely published library + // is being served. + // The value from PublishInfo.URL of the published library should be used + // while creating a subscribed library. + URL string `json:"url"` + + // OnDemand indicates whether a library item’s content will be synchronized + // only on demand. + OnDemand bool `json:"onDemand"` + + // AutomaticSync indicates whether the library should participate in + // automatic library synchronization. + AutomaticSync bool `json:"automaticSync"` +} + +// PublishInfo defines how the library is published so that it can be subscribed +// to by a remote subscribed library. +type PublishInfo struct { + // Published indicates if the local library is published so that it can be + // subscribed to by a remote subscribed library. + Published bool `json:"published"` + + // URL to which the library metadata is published by the vSphere Content + // Library Service. + // This value can be used to set the SubscriptionInfo.URL property when + // creating a subscribed library. + URL string `json:"url"` +} + +// +kubebuilder:validation:Enum=ContentLibrary;Inventory + +// LibraryType defines the types of libraries. +type LibraryType string + +const ( + // LibraryTypeContentLibrary describes a classic vCenter content library + // whose library items are surfaced as ContentLibraryItem resources. + LibraryTypeContentLibrary LibraryType = "ContentLibrary" + + // LibraryTypeInventory describes a folder in the vCenter inventory whose + // virtual machines are surfaced as ContentLibraryItem resources. + LibraryTypeInventory LibraryType = "Inventory" +) + +type BaseContentLibrarySpec struct { + // ID describes the unique identifier used to find the library in vCenter. + // + // Please note this value may differ depending on spec.type: + // - Type=ContentLibrary -- ID is a content library UUID. + // - Type=Inventory -- ID is a vSphere folder managed object ID. + ID string `json:"id"` + + // +optional + // +kubebuilder:default=ContentLibrary + + // Type describes the type of library. + // + // Defaults to ContentLibrary. + Type LibraryType `json:"libraryType,omitempty"` + + // +optional + // +kubebuilder:default=FromItemID + + // ResourceNamingStrategy describes the naming strategy for item resources + // in this content library. + // + // This field is immutable and defaults to FromItemID. + // + // Please note, this is optional and not present on all libraries. + ResourceNamingStrategy ResourceNamingStrategy `json:"resourceNamingStrategy,omitempty"` +} + +// ContentLibrarySpec defines the desired state of a ContentLibrary. +type ContentLibrarySpec struct { + BaseContentLibrarySpec `json:",inline"` + + // +optional + + // StorageClass describes the name of the StorageClass used when publishing + // images to this library. + // + // Please note, this is optional and not present on all libraries. + StorageClass string `json:"storageClass,omitempty"` + + // +optional + + // AllowDelete describes whether or not it is possible to delete items from + // this library. + AllowDelete bool `json:"allowDelete,omitempty"` + + // +optional + + // AllowImport describes whether or not it is possible to import remote + // OVA/OVF/ISO content to this library. + AllowImport bool `json:"allowImport,omitempty"` + + // +optional + + // AllowPublish describes whether or not it is possible to publish new items + // to this library. + AllowPublish bool `json:"allowPublish,omitempty"` +} + +// ContentLibraryStatus defines the observed state of ContentLibrary. +type ContentLibraryStatus struct { + // +optional + + // Name describes the display name for the library. + Name string `json:"name,omitempty"` + + // +optional + + // Description describes a human-readable description for the library. + Description string `json:"description,omitempty"` + + // +optional + + // StorageBackings describes the default storage backing for the library. + StorageBackings []StorageBacking `json:"storageBacking,omitempty"` + + // +optional + + // Version describes an optional value that tracks changes to the library's + // metadata, such as its name or description. + // + // Please note, this is optional and not present on all libraries. + Version string `json:"version,omitempty"` + + // +optional + + // PublishInfo describes how the library is published. + // + // Please note, this is only applicable for published libraries. + PublishInfo *PublishInfo `json:"publishInfo,omitempty"` + + // +optional + + // SubscriptionInfo describes how the library is subscribed. + // + // This field is only present for subscribed libraries. + SubscriptionInfo *SubscriptionInfo `json:"subscriptionInfo,omitempty"` + + // +optional + + // SecurityPolicyID describes the security policy applied to the library. + // + // Please note, this is optional and not present on all libraries. + SecurityPolicyID string `json:"securityPolicyID,omitempty"` + + // +optional + + // State describes the current state of the library. + State State `json:"state,omitempty"` + + // +optional + + // ServerGUID describes the unique identifier of the vCenter server where + // the library exists. + ServerGUID string `json:"serverGUID,omitempty"` + + // +optional + + // CreationTime describes the date and time when this library was created + // in vCenter. + CreationTime metav1.Time `json:"creationTime,omitempty"` + + // +optional + + // LastModifiedTime describes the date and time when the library was last + // updated. + // This field is updated only when the library properties are changed. + // This field is not updated when a library item is added, modified, + // deleted, or its content is changed. + LastModifiedTime metav1.Time `json:"lastModifiedTime,omitempty"` + + // +optional + + // LastSyncTime describes the date and time when this library was last + // synchronized. + // + // Please note, this is only applicable for subscribed libraries. + LastSyncTime metav1.Time `json:"lastSyncTime,omitempty"` + + // +optional + + // Conditions describes the current condition information of the library. + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +func (contentLibrary ContentLibrary) GetConditions() []metav1.Condition { + return contentLibrary.Status.Conditions +} + +func (contentLibrary *ContentLibrary) SetConditions(conditions []metav1.Condition) { + contentLibrary.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:resource:scope=Namespaced,shortName=lib;library +// +kubebuilder:printcolumn:name="DisplayName",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="AllowPublish",type="boolean",JSONPath=".spec.allowPublish" +// +kubebuilder:printcolumn:name="AllowImport",type="boolean",JSONPath=".spec.allowImport" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// ContentLibrary is the schema for the content library API. +type ContentLibrary struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ContentLibrarySpec `json:"spec,omitempty"` + Status ContentLibraryStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ContentLibraryList contains a list of ContentLibrary. +type ContentLibraryList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ContentLibrary `json:"items"` +} + +// ClusterContentLibrarySpec defines the desired state of a ClusterContentLibrary. +type ClusterContentLibrarySpec struct { + BaseContentLibrarySpec `json:",inline"` +} + +func (ccl ClusterContentLibrary) GetConditions() []metav1.Condition { + return ccl.Status.Conditions +} + +func (ccl *ClusterContentLibrary) SetConditions(conditions []metav1.Condition) { + ccl.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:resource:scope=Cluster,shortName=clib;clibrary +// +kubebuilder:printcolumn:name="DisplayName",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// ClusterContentLibrary is the schema for the cluster scoped content library +// API. +type ClusterContentLibrary struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ClusterContentLibrarySpec `json:"spec,omitempty"` + Status ContentLibraryStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ClusterContentLibraryList contains a list of ClusterContentLibrary. +type ClusterContentLibraryList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ClusterContentLibrary `json:"items"` +} + +func init() { + objectTypes = append( + objectTypes, + &ContentLibrary{}, + &ContentLibraryList{}, + &ClusterContentLibrary{}, + &ClusterContentLibraryList{}, + ) +} diff --git a/external/image-registry-operator/api/v1alpha2/contentlibraryitem_types.go b/external/image-registry-operator/api/v1alpha2/contentlibraryitem_types.go new file mode 100644 index 000000000..54c2aac6e --- /dev/null +++ b/external/image-registry-operator/api/v1alpha2/contentlibraryitem_types.go @@ -0,0 +1,311 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha2 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +kubebuilder:validation:Enum=OVF;ISO;VM + +// ContentLibraryItemType describes the type of library item. +type ContentLibraryItemType string + +const ( + // ContentLibraryItemTypeOvf describes an OVF library item. + ContentLibraryItemTypeOvf ContentLibraryItemType = "OVF" + + // ContentLibraryItemTypeIso describes an ISO library item. + ContentLibraryItemTypeIso ContentLibraryItemType = "ISO" + + // ContentLibraryItemTypeVM describes a VM library item. + ContentLibraryItemTypeVM ContentLibraryItemType = "VM" +) + +// +kubebuilder:validation:Enum=NotAvailable;Verified;Internal;VerificationFailure;VerificationInProgress;Untrusted + +// CertVerificationStatus is a constant for the certificate verification status +// of a content library item. +type CertVerificationStatus string + +const ( + // CertVerificationStatusNotAvailable indicates the certificate verification + // status is not available. + CertVerificationStatusNotAvailable CertVerificationStatus = "NotAvailable" + + // CertVerificationStatusVerified indicates the library item has been fully + // validated during importing or file syncing. + CertVerificationStatusVerified CertVerificationStatus = "Verified" + + // CertVerificationStatusInternal indicates the library item is + // cloned/created through vCenter. + CertVerificationStatusInternal CertVerificationStatus = "Internal" + + // CertVerificationStatusVerificationFailure indicates certificate or + // manifest validation failed on the library item. + CertVerificationStatusVerificationFailure CertVerificationStatus = "VerificationFailure" + + // CertVerificationStatusVerificationInProgress indicates the library item + // certificate verification is in progress. + CertVerificationStatusVerificationInProgress CertVerificationStatus = "VerificationInProgress" + + // CertVerificationStatusUntrusted indicates the certificate used to sign + // the library item is not trusted. + CertVerificationStatusUntrusted CertVerificationStatus = "Untrusted" +) + +// CertificateVerificationInfo shows the certificate verification status and the +// signing certificate. +type CertificateVerificationInfo struct { + // +optional + + // Status describes the certificate verification status of the library item. + Status CertVerificationStatus `json:"status,omitempty"` + + // +optional + + // CertChain describes the signing certificate chain in base64 encoding if + // the library item is signed. + CertChain []string `json:"certChain,omitempty"` +} + +// FileInfo represents the information of a file in a content library item in +// vCenter. +type FileInfo struct { + // Name describes the name of the file. + Name string `json:"name"` + + // +optional + + // SizeInBytes describes the total number of bytes used by the file on disk. + SizeInBytes resource.Quantity `json:"sizeInBytes,omitempty"` + + // +optional + + // Version describes the version of the library item file. + // This value is incremented when the file is modified on disk. + Version string `json:"version,omitempty"` + + // +optional + + // Cached describes whether or not the file is available locally on disk. + Cached bool `json:"cached,omitempty"` + + // +optional + + // StorageURI describes the fully-qualified path to the file on a datastore, + // ex. "[my-datastore-1] library-1/item-1/file1.vmdk". + // This URL is useful for creating a device that is backed by this file + // (i.e. mounting an ISO file via a virtual CD-ROM device). + StorageURI string `json:"storageURI,omitempty"` +} + +// ContentLibraryItemSpec defines the desired state of a ContentLibraryItem. +type ContentLibraryItemSpec struct { + // ID describes the unique identifier used to find the library item in + // vCenter. + // + // Please note this value depends on the type of the library to which this + // item belongs: + // - LibraryType=ContentLibrary -- ID is a content library item UUID. + // - LibraryType=Inventory -- ID is a vSphere VM managed object ID. + ID string `json:"id"` + + // +required + + // LibraryName describes the name of the library to which this item belongs. + // Namespace-scoped items belong to namespace-scoped libraries and + // cluster-scoped items belong to cluster-scoped libraries. + LibraryName string `json:"libraryName"` +} + +// ContentLibraryItemStatus defines the observed state of ContentLibraryItem. +type ContentLibraryItemStatus struct { + // +optional + + // Name describes the name of the underlying item in vCenter. + Name string `json:"name,omitempty"` + + // +optional + + // Description is a human-readable description for this library item. + Description string `json:"description,omitempty"` + + // +optional + + // Version describes a value that tracks changes to the library item's + // metadata, such as its name or description. + // + // Please note, this is optional and not present on all items. + Version string `json:"version,omitempty"` + + // +optional + + // ContentVersion describes a value that tracks changes to the library + // item's content. + // + // Please note, this is optional and not present on all items. + ContentVersion string `json:"contentVersion,omitempty"` + + // +optional + + // Type describes the type of the library item. + Type ContentLibraryItemType `json:"type,omitempty"` + + // +optional + + // SourceID describes a unique piece of information used to identify a + // library item on the remote side a subscribed library. + // + // Please note, this is only applicable to items from subscribed libraries. + SourceID string `json:"sourceID,omitempty"` + + // +optional + + // SizeInBytes describes the total number of bytes used by the library item + // on disk. + SizeInBytes resource.Quantity `json:"sizeInBytes,omitempty"` + + // +optional + + // Cached describes if the library item files are available on disk. + Cached bool `json:"cached,omitempty"` + + // +optional + + // SecurityCompliance describes the optional security compliance of the + // library item. + // + // Please note, this is optional and not present on all items. + SecurityCompliance *bool `json:"securityCompliance,omitempty"` + + // +optional + + // CertificateVerificationInfo describes the certificate verification status + // and signing certificate. + // + // Please note, this is optional and not present on all items. + CertificateVerificationInfo *CertificateVerificationInfo `json:"certificateVerificationInfo,omitempty"` + + // +optional + + // FileInfo describes the files that belong to the library item. + FileInfo []FileInfo `json:"fileInfo,omitempty"` + + // +optional + + // CreationTime describes when the library item was created. + CreationTime metav1.Time `json:"creationTime,omitempty"` + + // +optional + + // LastModifiedTime describes when the library item's metadata or content + // were last modified. + LastModifiedTime metav1.Time `json:"lastModifiedTime,omitempty"` + + // +optional + + // LastSyncTime describes when the library item was last synchronized. + // + // Please note, this is only applicable to items from subscribed libraries. + LastSyncTime metav1.Time `json:"lastSyncTime,omitempty"` + + // +optional + + // Conditions describes the current condition information of the library + // item. + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +func (contentLibraryItem ContentLibraryItem) GetConditions() []metav1.Condition { + return contentLibraryItem.Status.Conditions +} + +func (contentLibraryItem *ContentLibraryItem) SetConditions(conditions []metav1.Condition) { + contentLibraryItem.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:resource:scope=Namespaced,shortName=libitem +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".status.type" +// +kubebuilder:printcolumn:name="DisplayName",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="LibraryName",type="string",JSONPath=".spec.libraryName" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(.type=='Ready')].status" +// +kubebuilder:printcolumn:name="Cached",type="boolean",JSONPath=".status.cached" +// +kubebuilder:printcolumn:name="Size",type="string",JSONPath=".status.sizeInBytes" +// +kubebuilder:printcolumn:name="SecurityCompliant",type="boolean",JSONPath=".status.securityCompliance" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// ContentLibraryItem is the schema for the content library item API. +// Currently, ContentLibraryItem is immutable to end users. +type ContentLibraryItem struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ContentLibraryItemSpec `json:"spec,omitempty"` + Status ContentLibraryItemStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ContentLibraryItemList contains a list of ContentLibraryItem. +type ContentLibraryItemList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ContentLibraryItem `json:"items"` +} + +func (cclItem ClusterContentLibraryItem) GetConditions() []metav1.Condition { + return cclItem.Status.Conditions +} + +func (cclItem *ClusterContentLibraryItem) SetConditions(conditions []metav1.Condition) { + cclItem.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:resource:scope=Cluster,shortName=clibitem +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".status.type" +// +kubebuilder:printcolumn:name="DisplayName",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="LibraryName",type="string",JSONPath=".status.libraryName" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(.type=='Ready')].status" +// +kubebuilder:printcolumn:name="Cached",type="boolean",JSONPath=".status.cached" +// +kubebuilder:printcolumn:name="Size",type="string",JSONPath=".status.sizeInBytes" +// +kubebuilder:printcolumn:name="SecurityCompliant",type="boolean",JSONPath=".status.securityCompliance" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// ClusterContentLibraryItem is the schema for the content library item API at the cluster scope. +// Currently, ClusterContentLibraryItem is immutable to end users. +type ClusterContentLibraryItem struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ContentLibraryItemSpec `json:"spec,omitempty"` + Status ContentLibraryItemStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ClusterContentLibraryItemList contains a list of ClusterContentLibraryItem. +type ClusterContentLibraryItemList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ClusterContentLibraryItem `json:"items"` +} + +func init() { + objectTypes = append( + objectTypes, + &ContentLibraryItem{}, + &ContentLibraryItemList{}, + &ClusterContentLibraryItem{}, + &ClusterContentLibraryItemList{}, + ) +} diff --git a/external/image-registry-operator/api/v1alpha2/contentlibraryitemimportrequest_types.go b/external/image-registry-operator/api/v1alpha2/contentlibraryitemimportrequest_types.go new file mode 100644 index 000000000..1633a81c8 --- /dev/null +++ b/external/image-registry-operator/api/v1alpha2/contentlibraryitemimportrequest_types.go @@ -0,0 +1,291 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// +kubebuilder:validation:Enum=Waiting;Transferring;Validating;Ready;Error + +// TransferStatus is a constant that indicates the transfer state of a file. +type TransferStatus string + +const ( + // TransferStatusWaiting indicates that the file is waiting to be + // transferred. + TransferStatusWaiting TransferStatus = "Waiting" + + // TransferStatusTransferring indicates that the data of the file is being + // transferred. + TransferStatusTransferring TransferStatus = "Transferring" + + // TransferStatusValidating indicates that the file is being validated. + TransferStatusValidating TransferStatus = "Validating" + + // TransferStatusReady indicates that the file has been fully transferred + // and is ready to be used. + TransferStatusReady TransferStatus = "Ready" + + // TransferStatusError indicates there was an error transferring or + // validating the file. + TransferStatusError TransferStatus = "Error" +) + +// ContentLibraryItemImportRequestSource contains the specification of the +// source for the import request. +type ContentLibraryItemImportRequestSource struct { + // URL is the endpoint that points to a file that is to be imported as a new + // Content Library Item in the target vSphere Content Library. If the target + // item type is ContentLibraryItemTypeOvf, the URL should point to an OVF + // descriptor file (.ovf), an OVA file (.ova), or an ISO file (.iso). + // Otherwise, the SourceValid condition will become false in the status. + URL string `json:"url"` + + // +optional + + // PEM encoded SSL Certificate for this endpoint specified by the URL. It is + // only used for HTTPS connections. + // If set, the remote endpoint's SSL certificate is only accepted if it + // matches this certificate, and no other certificate validation is + // performed. + // If unset, the remote endpoint's SSL certificate must be trusted by + // vSphere trusted root CA certificates, otherwise the SSL certification + // verification may fail and thus fail the import request. + SSLCertificate string `json:"sslCertificate,omitempty"` + + // +optional + + // Checksum contains the checksum algorithm and value calculated for the + // file specified in the URL. If omitted, the import request will not verify + // the checksum of the file. + Checksum *Checksum `json:"checksum,omitempty"` +} + +// Checksum contains the checksum value and algorithm used to calculate that +// value. +type Checksum struct { + // +optional + // +kubebuilder:validation:Enum=SHA256;SHA512 + // +kubebuilder:default=SHA256 + + // Algorithm is the algorithm used to calculate the checksum. Supported + // algorithms are "SHA256" and "SHA512". If omitted, "SHA256" will be used + // as the default algorithm. + Algorithm string `json:"algorithm"` + + // Value is the checksum value calculated by the specified algorithm. + Value string `json:"value"` +} + +// ContentLibraryItemImportRequestTargetItem contains the specification of the +// target content library item for the import request. +type ContentLibraryItemImportRequestTargetItem struct { + // +optional + + // Name is the name of the new content library item that will be created + // in vSphere. + // If omitted, the content library item will be created with the same name + // as the name of the image specified in the spec.source.url in the + // specified library. + // If an item with the same name already exists in the specified library, + // the TargetValid condition will become false in the + // status. + Name string `json:"name,omitempty"` + + // +optional + + // Description is a description for a library item. + Description string `json:"description,omitempty"` + + // +optional + + // Type is the type of the new library item that will be created. + // + // The valid types depend on the type of underlying library: + // - LibraryType=ContentLibrary -- OVF + // - LibraryType=ContentLibrary -- ISO + // + // If omitted or the type is invalid, the TargetValid condition will be + // false. + // + // For the item type OVF, the default OVF security policy must be configured + // on the target library, otherwise the TargetValid condition will be false. + Type ContentLibraryItemType `json:"type,omitempty"` +} + +// ContentLibraryItemImportRequestTarget is the target specification of an +// import request. +type ContentLibraryItemImportRequestTarget struct { + // +optional + + // Item contains information about the library item to which the item will + // be imported in vSphere. + // + // If omitted, the library item will be created with the same name as the + // name of the image specified in the spec.source.url in the specified + // library. + // + // If an item with the same name already exists in the specified library, + // the TargetValid condition will be false. + Item ContentLibraryItemImportRequestTargetItem `json:"item,omitempty"` + + // Library describes the name of the library in which the item will be + // created. + LibraryName string `json:"library"` +} + +// ContentLibraryItemImportRequestSpec defines the desired state of a +// ContentLibraryItemImportRequest. +type ContentLibraryItemImportRequestSpec struct { + // Source is the source of the import request which includes an external URL + // pointing to a VM image template. + // Source and Target will be immutable if the SourceValid and TargetValid + // conditions are true. + Source ContentLibraryItemImportRequestSource `json:"source"` + + // Target is the target of the import request which includes the content + // library item information and a ContentLibrary resource. + // Source and Target will be immutable if the SourceValid and TargetValid + // conditions are true. + Target ContentLibraryItemImportRequestTarget `json:"target"` + + // +optional + // +kubebuilder:validation:Minimum=0 + + // TTLSecondsAfterFinished is the time-to-live duration for how long this + // resource will be allowed to exist once the import operation + // completes. After the TTL expires, the resource will be automatically + // deleted without the user having to take any direct action. + // If this field is unset then the request resource will not be + // automatically deleted. If this field is set to zero then the request + // resource is eligible for deletion immediately after it finishes. + TTLSecondsAfterFinished *int64 `json:"ttlSecondsAfterFinished,omitempty"` +} + +// ContentLibraryItemFileUploadStatus indicates the upload status of files +// belonging to the template. +type ContentLibraryItemFileUploadStatus struct { + // SessionUUID is the identifier that uniquely identifies the file upload + // session on the library item in vSphere. + SessionUUID types.UID `json:"sessionUUID,omitempty"` + + // +optional + + // FileUploads list the transfer statuses of files being uploaded and + // tracked by the upload session. + FileUploads []FileTransferStatus `json:"fileUploads,omitempty"` +} + +// FileTransferStatus indicates the transfer status of a file belonging to a +// library item. +type FileTransferStatus struct { + // Name specifies the name of the file that is transferred. + Name string `json:"name"` + + // Status indicates the transfer status of the file. + Status TransferStatus `json:"transferStatus"` + + // +optional + + // BytesTransferred indicates the number of bytes of this file that have + // been received by the server. + BytesTransferred *int64 `json:"bytesTransferred,omitempty"` + + // +optional + + // Size indicates the file size in bytes as received by the server. + // This value will not be available until the transfer status is ready. + Size *int64 `json:"size,omitempty"` + + // +optional + + // ErrorMessage describes the details about the transfer error if the + // transfer status is error. + ErrorMessage string `json:"errorMessage,omitempty"` +} + +// ContentLibraryItemImportRequestStatus defines the observed state of a +// ContentLibraryItemImportRequest. +type ContentLibraryItemImportRequestStatus struct { + // +optional + + // ItemName is the name to the target ContentLibraryItem resource created as + // a result of the import. + // If the ContentLibraryItemImportRequest is deleted when the import + // operation fails or before the Complete condition is set to true, the + // import operation will be cancelled in vSphere and the corresponding + // vSphere Content Library Item will be deleted. + ItemName string `json:"itemName,omitempty"` + + // +optional + + // CompletionTime represents time when the request was completed. + // The value of this field should be equal to the value of the + // LastTransitionTime for the status condition Type=Complete. + CompletionTime metav1.Time `json:"completionTime,omitempty"` + + // +optional + + // StartTime represents time when the request was acknowledged by the + // controller. + StartTime metav1.Time `json:"startTime,omitempty"` + + // +optional + + // FileUpload indicates the upload status of files belonging to the template. + FileUploadStatus *ContentLibraryItemFileUploadStatus `json:"fileUploadStatus,omitempty"` + + // +optional + + // Conditions describes the current condition information of the + // ContentLibraryItemImportRequest. + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +func (clItemImportRequest ContentLibraryItemImportRequest) GetConditions() []metav1.Condition { + return clItemImportRequest.Status.Conditions +} + +func (clItemImportRequest *ContentLibraryItemImportRequest) SetConditions(conditions []metav1.Condition) { + clItemImportRequest.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,shortName=libitemimport +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="ContentLibraryRef",type="string",JSONPath=".spec.target.libraryName" +// +kubebuilder:printcolumn:name="ContentLibraryItemRef",type="string",JSONPath=".status.itemName" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(.type=='Complete')].status" + +// ContentLibraryItemImportRequest defines the information necessary to import a VM image +// template as a ContentLibraryItem to a Content Library in vSphere. +type ContentLibraryItemImportRequest struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ContentLibraryItemImportRequestSpec `json:"spec,omitempty"` + Status ContentLibraryItemImportRequestStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ContentLibraryItemImportRequestList contains a list of +// ContentLibraryItemImportRequest resources. +type ContentLibraryItemImportRequestList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ContentLibraryItemImportRequest `json:"items"` +} + +func init() { + objectTypes = append( + objectTypes, + &ContentLibraryItemImportRequest{}, + &ContentLibraryItemImportRequestList{}, + ) +} diff --git a/external/image-registry-operator/api/v1alpha2/doc.go b/external/image-registry-operator/api/v1alpha2/doc.go new file mode 100644 index 000000000..abef05712 --- /dev/null +++ b/external/image-registry-operator/api/v1alpha2/doc.go @@ -0,0 +1,10 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +// +k8s:openapi-gen=true +// +kubebuilder:object:generate=true +// +groupName=imageregistry.vmware.com + +// Package v1alpha2 is one of the schemas for Image Registry Operator. +package v1alpha2 diff --git a/external/image-registry-operator/api/v1alpha2/groupversion_info.go b/external/image-registry-operator/api/v1alpha2/groupversion_info.go new file mode 100644 index 000000000..fcaabf175 --- /dev/null +++ b/external/image-registry-operator/api/v1alpha2/groupversion_info.go @@ -0,0 +1,33 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName specifies the group name used to register the objects. +const GroupName = "imageregistry.vmware.com" + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha2"} + + // schemeBuilder is used to add go types to the GroupVersionKind scheme. + schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = schemeBuilder.AddToScheme + + objectTypes = []runtime.Object{} +) + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(GroupVersion, objectTypes...) + metav1.AddToGroupVersion(scheme, GroupVersion) + return nil +} diff --git a/external/image-registry-operator/api/v1alpha2/zz_generated.deepcopy.go b/external/image-registry-operator/api/v1alpha2/zz_generated.deepcopy.go new file mode 100644 index 000000000..7950c33aa --- /dev/null +++ b/external/image-registry-operator/api/v1alpha2/zz_generated.deepcopy.go @@ -0,0 +1,698 @@ +//go:build !ignore_autogenerated + +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha2 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BaseContentLibrarySpec) DeepCopyInto(out *BaseContentLibrarySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BaseContentLibrarySpec. +func (in *BaseContentLibrarySpec) DeepCopy() *BaseContentLibrarySpec { + if in == nil { + return nil + } + out := new(BaseContentLibrarySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificateVerificationInfo) DeepCopyInto(out *CertificateVerificationInfo) { + *out = *in + if in.CertChain != nil { + in, out := &in.CertChain, &out.CertChain + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificateVerificationInfo. +func (in *CertificateVerificationInfo) DeepCopy() *CertificateVerificationInfo { + if in == nil { + return nil + } + out := new(CertificateVerificationInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Checksum) DeepCopyInto(out *Checksum) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Checksum. +func (in *Checksum) DeepCopy() *Checksum { + if in == nil { + return nil + } + out := new(Checksum) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterContentLibrary) DeepCopyInto(out *ClusterContentLibrary) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterContentLibrary. +func (in *ClusterContentLibrary) DeepCopy() *ClusterContentLibrary { + if in == nil { + return nil + } + out := new(ClusterContentLibrary) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterContentLibrary) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterContentLibraryItem) DeepCopyInto(out *ClusterContentLibraryItem) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterContentLibraryItem. +func (in *ClusterContentLibraryItem) DeepCopy() *ClusterContentLibraryItem { + if in == nil { + return nil + } + out := new(ClusterContentLibraryItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterContentLibraryItem) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterContentLibraryItemList) DeepCopyInto(out *ClusterContentLibraryItemList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterContentLibraryItem, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterContentLibraryItemList. +func (in *ClusterContentLibraryItemList) DeepCopy() *ClusterContentLibraryItemList { + if in == nil { + return nil + } + out := new(ClusterContentLibraryItemList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterContentLibraryItemList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterContentLibraryList) DeepCopyInto(out *ClusterContentLibraryList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterContentLibrary, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterContentLibraryList. +func (in *ClusterContentLibraryList) DeepCopy() *ClusterContentLibraryList { + if in == nil { + return nil + } + out := new(ClusterContentLibraryList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterContentLibraryList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterContentLibrarySpec) DeepCopyInto(out *ClusterContentLibrarySpec) { + *out = *in + out.BaseContentLibrarySpec = in.BaseContentLibrarySpec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterContentLibrarySpec. +func (in *ClusterContentLibrarySpec) DeepCopy() *ClusterContentLibrarySpec { + if in == nil { + return nil + } + out := new(ClusterContentLibrarySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibrary) DeepCopyInto(out *ContentLibrary) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibrary. +func (in *ContentLibrary) DeepCopy() *ContentLibrary { + if in == nil { + return nil + } + out := new(ContentLibrary) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibrary) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItem) DeepCopyInto(out *ContentLibraryItem) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItem. +func (in *ContentLibraryItem) DeepCopy() *ContentLibraryItem { + if in == nil { + return nil + } + out := new(ContentLibraryItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibraryItem) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemFileUploadStatus) DeepCopyInto(out *ContentLibraryItemFileUploadStatus) { + *out = *in + if in.FileUploads != nil { + in, out := &in.FileUploads, &out.FileUploads + *out = make([]FileTransferStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemFileUploadStatus. +func (in *ContentLibraryItemFileUploadStatus) DeepCopy() *ContentLibraryItemFileUploadStatus { + if in == nil { + return nil + } + out := new(ContentLibraryItemFileUploadStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequest) DeepCopyInto(out *ContentLibraryItemImportRequest) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequest. +func (in *ContentLibraryItemImportRequest) DeepCopy() *ContentLibraryItemImportRequest { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibraryItemImportRequest) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestList) DeepCopyInto(out *ContentLibraryItemImportRequestList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ContentLibraryItemImportRequest, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestList. +func (in *ContentLibraryItemImportRequestList) DeepCopy() *ContentLibraryItemImportRequestList { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibraryItemImportRequestList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestSource) DeepCopyInto(out *ContentLibraryItemImportRequestSource) { + *out = *in + if in.Checksum != nil { + in, out := &in.Checksum, &out.Checksum + *out = new(Checksum) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestSource. +func (in *ContentLibraryItemImportRequestSource) DeepCopy() *ContentLibraryItemImportRequestSource { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestSpec) DeepCopyInto(out *ContentLibraryItemImportRequestSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + out.Target = in.Target + if in.TTLSecondsAfterFinished != nil { + in, out := &in.TTLSecondsAfterFinished, &out.TTLSecondsAfterFinished + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestSpec. +func (in *ContentLibraryItemImportRequestSpec) DeepCopy() *ContentLibraryItemImportRequestSpec { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestStatus) DeepCopyInto(out *ContentLibraryItemImportRequestStatus) { + *out = *in + in.CompletionTime.DeepCopyInto(&out.CompletionTime) + in.StartTime.DeepCopyInto(&out.StartTime) + if in.FileUploadStatus != nil { + in, out := &in.FileUploadStatus, &out.FileUploadStatus + *out = new(ContentLibraryItemFileUploadStatus) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestStatus. +func (in *ContentLibraryItemImportRequestStatus) DeepCopy() *ContentLibraryItemImportRequestStatus { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestTarget) DeepCopyInto(out *ContentLibraryItemImportRequestTarget) { + *out = *in + out.Item = in.Item +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestTarget. +func (in *ContentLibraryItemImportRequestTarget) DeepCopy() *ContentLibraryItemImportRequestTarget { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemImportRequestTargetItem) DeepCopyInto(out *ContentLibraryItemImportRequestTargetItem) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemImportRequestTargetItem. +func (in *ContentLibraryItemImportRequestTargetItem) DeepCopy() *ContentLibraryItemImportRequestTargetItem { + if in == nil { + return nil + } + out := new(ContentLibraryItemImportRequestTargetItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemList) DeepCopyInto(out *ContentLibraryItemList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ContentLibraryItem, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemList. +func (in *ContentLibraryItemList) DeepCopy() *ContentLibraryItemList { + if in == nil { + return nil + } + out := new(ContentLibraryItemList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibraryItemList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemSpec) DeepCopyInto(out *ContentLibraryItemSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemSpec. +func (in *ContentLibraryItemSpec) DeepCopy() *ContentLibraryItemSpec { + if in == nil { + return nil + } + out := new(ContentLibraryItemSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryItemStatus) DeepCopyInto(out *ContentLibraryItemStatus) { + *out = *in + out.SizeInBytes = in.SizeInBytes.DeepCopy() + if in.SecurityCompliance != nil { + in, out := &in.SecurityCompliance, &out.SecurityCompliance + *out = new(bool) + **out = **in + } + if in.CertificateVerificationInfo != nil { + in, out := &in.CertificateVerificationInfo, &out.CertificateVerificationInfo + *out = new(CertificateVerificationInfo) + (*in).DeepCopyInto(*out) + } + if in.FileInfo != nil { + in, out := &in.FileInfo, &out.FileInfo + *out = make([]FileInfo, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.CreationTime.DeepCopyInto(&out.CreationTime) + in.LastModifiedTime.DeepCopyInto(&out.LastModifiedTime) + in.LastSyncTime.DeepCopyInto(&out.LastSyncTime) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryItemStatus. +func (in *ContentLibraryItemStatus) DeepCopy() *ContentLibraryItemStatus { + if in == nil { + return nil + } + out := new(ContentLibraryItemStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryList) DeepCopyInto(out *ContentLibraryList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ContentLibrary, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryList. +func (in *ContentLibraryList) DeepCopy() *ContentLibraryList { + if in == nil { + return nil + } + out := new(ContentLibraryList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ContentLibraryList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibrarySpec) DeepCopyInto(out *ContentLibrarySpec) { + *out = *in + out.BaseContentLibrarySpec = in.BaseContentLibrarySpec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibrarySpec. +func (in *ContentLibrarySpec) DeepCopy() *ContentLibrarySpec { + if in == nil { + return nil + } + out := new(ContentLibrarySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContentLibraryStatus) DeepCopyInto(out *ContentLibraryStatus) { + *out = *in + if in.StorageBackings != nil { + in, out := &in.StorageBackings, &out.StorageBackings + *out = make([]StorageBacking, len(*in)) + copy(*out, *in) + } + if in.PublishInfo != nil { + in, out := &in.PublishInfo, &out.PublishInfo + *out = new(PublishInfo) + **out = **in + } + if in.SubscriptionInfo != nil { + in, out := &in.SubscriptionInfo, &out.SubscriptionInfo + *out = new(SubscriptionInfo) + **out = **in + } + in.CreationTime.DeepCopyInto(&out.CreationTime) + in.LastModifiedTime.DeepCopyInto(&out.LastModifiedTime) + in.LastSyncTime.DeepCopyInto(&out.LastSyncTime) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentLibraryStatus. +func (in *ContentLibraryStatus) DeepCopy() *ContentLibraryStatus { + if in == nil { + return nil + } + out := new(ContentLibraryStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FileInfo) DeepCopyInto(out *FileInfo) { + *out = *in + out.SizeInBytes = in.SizeInBytes.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileInfo. +func (in *FileInfo) DeepCopy() *FileInfo { + if in == nil { + return nil + } + out := new(FileInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FileTransferStatus) DeepCopyInto(out *FileTransferStatus) { + *out = *in + if in.BytesTransferred != nil { + in, out := &in.BytesTransferred, &out.BytesTransferred + *out = new(int64) + **out = **in + } + if in.Size != nil { + in, out := &in.Size, &out.Size + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileTransferStatus. +func (in *FileTransferStatus) DeepCopy() *FileTransferStatus { + if in == nil { + return nil + } + out := new(FileTransferStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PublishInfo) DeepCopyInto(out *PublishInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PublishInfo. +func (in *PublishInfo) DeepCopy() *PublishInfo { + if in == nil { + return nil + } + out := new(PublishInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageBacking) DeepCopyInto(out *StorageBacking) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageBacking. +func (in *StorageBacking) DeepCopy() *StorageBacking { + if in == nil { + return nil + } + out := new(StorageBacking) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubscriptionInfo) DeepCopyInto(out *SubscriptionInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubscriptionInfo. +func (in *SubscriptionInfo) DeepCopy() *SubscriptionInfo { + if in == nil { + return nil + } + out := new(SubscriptionInfo) + in.DeepCopyInto(out) + return out +} diff --git a/external/image-registry-operator/go.mod b/external/image-registry-operator/go.mod new file mode 100644 index 000000000..deec1e056 --- /dev/null +++ b/external/image-registry-operator/go.mod @@ -0,0 +1,28 @@ +module github.com/vmware-tanzu/vm-operator/external/image-registry-operator + +go 1.26.1 + +require ( + k8s.io/api v0.35.3 + k8s.io/apimachinery v0.35.3 +) + +require ( + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/text v0.31.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect +) diff --git a/external/image-registry-operator/go.sum b/external/image-registry-operator/go.sum new file mode 100644 index 000000000..4dbbd404d --- /dev/null +++ b/external/image-registry-operator/go.sum @@ -0,0 +1,66 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ= +k8s.io/api v0.35.3/go.mod h1:9Y9tkBcFwKNq2sxwZTQh1Njh9qHl81D0As56tu42GA4= +k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8= +k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/external/mobility-operator/api/v1alpha2/common_types.go b/external/mobility-operator/api/v1alpha2/common_types.go new file mode 100644 index 000000000..8e0fea937 --- /dev/null +++ b/external/mobility-operator/api/v1alpha2/common_types.go @@ -0,0 +1,8 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. + +package v1alpha2 + +const ( + // ImportOperationLabelName is the label set on entities linked to an ImportOperation. + ImportOperationLabelName = GroupName + "/import-operation" +) diff --git a/external/mobility-operator/api/v1alpha2/groupversion_info.go b/external/mobility-operator/api/v1alpha2/groupversion_info.go new file mode 100644 index 000000000..df4352a37 --- /dev/null +++ b/external/mobility-operator/api/v1alpha2/groupversion_info.go @@ -0,0 +1,30 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. + +// Package v1alpha2 contains API Schema definitions for the mobility-operator v1alpha2 API group +// +kubebuilder:object:generate=true +// +groupName=mobility-operator.vmware.com +package v1alpha2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +const ( + // Version is the API Version. + Version = "v1alpha2" + + // GroupName is the name of the API group. + GroupName = "mobility-operator.vmware.com" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: GroupName, Version: Version} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/external/mobility-operator/api/v1alpha2/importoperation_types.go b/external/mobility-operator/api/v1alpha2/importoperation_types.go new file mode 100644 index 000000000..e0709a3d4 --- /dev/null +++ b/external/mobility-operator/api/v1alpha2/importoperation_types.go @@ -0,0 +1,519 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. + +package v1alpha2 + +import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ImportOperationSpecValid and ImportOperationPrecheckSucceeded Condition.Type and their corresponding Condition.Reasons. +const ( + // ImportOperationSpecValid is set to true only when ImportOperation.Spec has been validated. + ImportOperationSpecValid = "SpecValid" + + // ImportOperationPrecheckSucceeded is set to true only when ImportOperation.Spec has been validated. + ImportOperationPrecheckSucceeded = "PrecheckSucceeded" + + // ImportOperationVMwareToolsMissing indicates that the import operation failed because the virtual machine does not have VMware Tools installed. + ImportOperationVMwareToolsMissing = "VMwareToolsMissing" + + // ImportOperationUnsupportedGuestOS indicates that the import operation failed because the virtual machine has a guest OS that does not support Guest OS customization. + ImportOperationUnsupportedGuestOS = "UnsupportedGuestOS" + + // ImportOperationIPv6NotSupported indicates that the import operation failed because the virtual machine has an IPv6 network, which is not supported. + ImportOperationIPv6NotSupported = "IPv6NotSupported" + + // ImportOperationVirtualMachineManagedByFieldAlreadySet indicates that the import operation failed because the virtual machine is already managed by a VC Extension. + ImportOperationVirtualMachineManagedByFieldAlreadySet = "ManagedByAlreadySet" + + // ImportOperationVirtualMachineTemplateNotSupported indicates that the import operation failed because the virtual machine is a template. + ImportOperationVirtualMachineTemplateNotSupported = "VirtualMachineTemplateNotSupported" + + // ImportOperationVirtualMachineAlreadyExists indicates a virtual machine already exists at the target folder. + + ImportOperationVirtualMachineAlreadyExists = "VirtualMachineAlreadyExists" + + // ImportOperationVAppNotSupported indicates the VM is part of a vApp, which is not supported. + ImportOperationVAppNotSupported = "VAppNotSupported" + + // ImportOperationImprovedVirtualDisksNotSupported indicates a virtual machine contains Improved Virtual Disks. + ImportOperationImprovedVirtualDisksNotSupported = "ImprovedVirtualDisksNotSupported" + + // ImportOperationSuspendedVirtualMachineNotSupported indicates that Virtual Machine is in Suspended state. + ImportOperationSuspendedVirtualMachineNotSupported = "SuspendedVirtualMachineNotSupported" +) + +// ImportOperationVirtualMachineSetManagedBySucceeded Condition.Type and its corresponding Condition.Reasons. +const ( + // ImportOperationVirtualMachineSetManagedBySucceeded is set to true only when the virtual machine has been successfully set managed by VM Service. + ImportOperationVirtualMachineSetManagedBySucceeded = "VirtualMachineSetManagedBySucceeded" + + // ImportOperationVirtualMachineSetManagedByFailure indicates that there was a failure to set the virtual machine as managed by VM Service. + ImportOperationVirtualMachineSetManagedByFailure = "VirtualMachineSetManagedByFailure" +) + +// ImportOperationNetworkBackingReady Condition.Type and its corresponding Condition.Reasons. +const ( + // ImportOperationNetworkBackingReady is set to true only when the network backing is ready for use by the incoming virtual machine. + ImportOperationNetworkBackingReady = "NetworkBackingReady" + + // ImportOperationNetworkCreationFailure indicates that there was a failure to create the target network. + ImportOperationNetworkCreationFailure = "NetworkCreationFailure" + + // ImportOperationNetworkNotReady indicates that the target network is not ready for use. + ImportOperationNetworkNotReady = "NetworkNotReady" + + // ImportOperationNetworkBackingMissing indicates that the target network is marked as ready to be used, but its network device backing cannot be retrieved. + ImportOperationNetworkBackingMissing = "NetworkBackingMissing" +) + +// ImportOperationVirtualMachineReadyForImport Condition.Type and its corresponding Condition.Reasons. +const ( + // ImportOperationVirtualMachineReadyForImport is set to true only when the virtual machine has been successfully moved to a suitable infrastructure servicing the target namespace and is ready to be imported. + ImportOperationVirtualMachineReadyForImport = "VirtualMachineReadyForImport" + + // ImportOperationVirtualMachinePlacementFailure indicates that there was a failure to recommend the virtual machine placement. + ImportOperationVirtualMachinePlacementFailure = "VirtualMachinePlacementFailure" + + // ImportOperationVirtualMachineReconfigureFailure indicates that there was a failure to reconfigure the virtual machine. + ImportOperationVirtualMachineReconfigureFailure = "VirtualMachineReconfigureFailure" + + // ImportOperationVirtualMachineRelocateFailure indicates that there was a failure to relocate the virtual machine. + ImportOperationVirtualMachineRelocateFailure = "VirtualMachineRelocateFailure" + + // ImportOperationVirtualMachineResetPermissionFailure indicates that there was a failure to reset the virtual machine permissions. + ImportOperationVirtualMachineResetPermissionFailure = "VirtualMachineResetPermissionFailure" +) + +// ImportOperationGuestCustomization Condition.Type and its corresponding Condition.Reason +const ( + // ImportOperationGuestCustomization is set to true when the virtual machine is customized successfully. + ImportOperationGuestCustomization = "GuestCustomization" + + // ImportOperationGuestCustomizationVMCrNotFound indicates that the virtual machine resource was not found during guest customization. + ImportOperationGuestCustomizationVMCrNotFound = "GuestCustomizationVMCrNotFound" + + // ImportOperationGuestCustomizationFailed indicates that the guest customization failed in the guest OS. + ImportOperationGuestCustomizationFailed = "GuestCustomizationFailed" +) + +// ImportOperation Condition.Types without any corresponding Condition.Reasons. +const ( + // ImportOperationVirtualMachineReady is set to true when the virtual machine is ready to be consumed in the target namespace. + ImportOperationVirtualMachineReady = "VirtualMachineReady" + + // ImportOperationCompleted is set to true when all other conditions for the import operation have been set to true. + ImportOperationCompleted = "Completed" +) + +// ImportOperationVirtualMachineCreated Condition.Type and its corresponding Condition.Reason +const ( + // ImportOperationVirtualMachineCreated is set to true when the virtual machine has been successfully created in the target namespace. + ImportOperationVirtualMachineCreated = "VirtualMachineCreated" + + // ImportOperationVMCRNotFoundPostCreation indicate a failure when the VM CR was created but cannot be found post creation. + ImportOperationVMCRNotFoundPostCreation = "VMCRNotFoundPostCreation" + + // ImportOperationCreateVMCRFailure indicates that there was a failure to create VM CR. + ImportOperationCreateVMCRFailure = "CreateVMCRFailure" + + // ImportOperationBootstrapSpecCreationFailure indicates that there was a failure to create the bootstrap spec for the VM CR. + ImportOperationBootstrapSpecCreationFailure = "BootstrapSpecCreationFailure" + + // ImportOperationVirtualMachineCreationFailureNotFound indicates that the virtual machine resource was not found during creating VM CR. + ImportOperationVirtualMachineNotFound = "VirtualMachineNotFound" + + // ImportOperationGetVMCRFailure indicates that there was a failure to get VM CR. The real reason is unknown. + ImportOperationGetVMCRFailure = "GetVMCRFailure" +) + +// Condition.Type for Conditions related to rollback in ImportOperation.Status. +const ( + // RollbackVirtualMachineLocationCompleted is set to true only when the VM or VMs have been successfully rolled back + // to the original location. + RollbackVirtualMachineLocationCompleted = "RollbackVirtualMachineLocationCompleted" + + // RollbackVirtualMachinePropertyCompleted is set to true only when the VM or VMs have been successfully rolled back + // to the original property. + RollbackVirtualMachinePropertyCompleted = "RollbackVirtualMachinePropertyCompleted" + + // RollbackCustomResourceCompleted is set to true only when the custom resource created by the ImportOperation have been + // successfully cleaned up. + RollbackCustomResourceCompleted = "RollbackCustomResourceCompleted" + + // RollbackCompleted is set to true only when all other conditions present in the rollback have been set to true. + RollbackCompleted = "RollbackCompleted" +) + +// Condition.Reason for Condition.Type RollbackVirtualMachineLocationCompleted related to rollback in ImportOperation.Status. +const ( + // RollbackVirtualMachineCancelImportTaskFailure means there is a failure to cancel virtual machine import task during rollback. + RollbackVirtualMachineCancelImportTaskFailure = "RollbackVirtualMachineCancelImportTaskFailure" + + // RollbackVirtualMachineRelocateFailure means there is a failure to relocate virtual machine during rollback. + RollbackVirtualMachineRelocateFailure = "RollbackVirtualMachineRelocateFailure" + + // RollbackVirtualMachineReconfigureFailure means there is a failure to reconfigure virtual machine during rollback. + RollbackVirtualMachineReconfigureFailure = "RollbackVirtualMachineReconfigureFailure" +) + +// Condition.Reason for Condition.Type RollbackVirtualMachinePropertyCompleted related to rollback in ImportOperation.Status. +const ( + // RollbackVirtualMachinePermissionFailure means there is a failure to restore virtual machine permission during rollback. + RollbackVirtualMachinePermissionFailure = "RollbackVirtualMachinePermissionFailure" + + // RollbackVirtualMachineManagedByFailure means there is a failure to restore virtual machine managed by field during rollback. + RollbackVirtualMachineManagedByFailure = "RollbackVirtualMachineManagedByFailure" +) + +// Condition.Reason for Condition.Type RollbackCustomResourceCompleted related to rollback in ImportOperation.Status. +const ( + // RollbackSecretCleanupFailure means there is a failure to cleanup secret during rollback. + RollbackSecretCleanupFailure = "RollbackSecretCleanupFailure" + + // RollbackNetworkBackingCleanupFailure means there is a failure to cleanup network backing during rollback. + RollbackNetworkBackingCleanupFailure = "RollbackNetworkBackingCleanupFailure" +) + +// ProductIDSecretKeySelector references the ProductID value from a Secret resource. +type ProductIDSecretKeySelector struct { + // Name is the name of the secret. + Name string `json:"name"` + + // +kubebuilder:default=product_id + + // Key is the key in the secret that specifies the requested data. + Key string `json:"key"` +} + +// CustomizationSysprepUserData contains user data for customizing a Windows guest operating system. +// This struct maps to the UserData key in the sysprep.xml answer file. +type CustomizationSysprepUserData struct { + // FullName is the full name of the user. + FullName string `json:"fullName"` + + // OrgName is the name of the organization. + OrgName string `json:"orgName"` + + // +kubebuilder:validation:Pattern=`^([a-zA-Z0-9\p{S}\p{L}]{1,2}|(?:[a-zA-Z0-9\p{S}\p{L}][a-zA-Z0-9-\p{S}\p{L}]{0,13}[a-zA-Z0-9\p{S}\p{L}]))$` + + // ComputerName is the computer name of the Windows virtual machine. It must be 1 to 15 characters in length. + // + // Validation Rules: + // - If the name is 1 or 2 characters long, it can consist of letters (A–Z, a–z), numbers (0–9), or symbols. + // - If the name is between 3 and 15 characters: + // - It must start and end with a letter, number, or symbol (excluding hyphens). + // - Middle characters (if any) can be letters, numbers, symbols, or hyphens (-). + // - Hyphens are **not** allowed at the beginning or end of the name. + // + // - The name must not contain spaces or periods (.). + // + // Examples of valid ComputerName values: + // - "A" + // - "AB" + // - "Server1" + // - "Workstation-01" + // - "WEB_SERVER" + // - "Comp-Name" + // + // Examples of invalid ComputerName values: + // - "-" (single hyphen, not allowed) + // - "Server-" (ends with hyphen) + // - "-Server" (starts with hyphen) + // - "Server Name" (contains spaces) + // - "Server.Name" (contains periods) + // - "ThisNameIsWayTooLongForTheLimit" (exceeds 15 characters) + ComputerName string `json:"computerName"` + + // +optional + + // ProductID is a valid serial number. Microsoft Sysprep requires that a valid serial number be included in the answer file when mini-setup runs. This serial number is ignored if the original guest operating system was installed using a volume-licensed CD. + ProductID *ProductIDSecretKeySelector `json:"productID,omitempty"` +} + +// NetworkCustomization is used to configure the network identity of the importing virtual machine. +// +// This is required when importing a Windows virtual machine that needs automated network identity configuration via Guest OS customization. +type NetworkCustomization struct { + // +optional + + // CustomizationSysprepUserData contains the user data for sysprep customization. + CustomizationSysprepUserData *CustomizationSysprepUserData `json:"customizationSysprepUserData,omitempty"` +} + +// +kubebuilder:validation:Enum=Subnet;SubnetSet + +// SubnetType defines the type of Subnet +type SubnetType string + +const ( + // SubnetTypeSubnet in a VPC represents an independent layer 2 broadcast domain with its associated CIDR and properties like Access mode (network advertisement), DHCP configuration etc. + SubnetTypeSubnet SubnetType = "Subnet" + + // SubnetTypeSubnetSet is a scalable grouping of VPC subnets sharing the same properties, which will allow auto-scale of networking availability to connect workloads. + SubnetTypeSubnetSet SubnetType = "SubnetSet" +) + +// SubnetInfo defines the information identifying a Subnet. +type SubnetInfo struct { + // Name corresponds to the name of the Subnet. + Name string `json:"name"` + + // +kubebuilder:validation:Minimum=0 + + // DeviceKey corresponds to the device key of the virtual device of the VM. + DeviceKey int32 `json:"deviceKey"` + + // +optional + + // Type of the Subnet, indicating whether it is a Subnet or SubnetSet. + // If the name is unique in the available Subnet and SubnetSet entities, + // this field is optional. + Type SubnetType `json:"type,omitempty"` +} + +// CommitActionType defines the type of commit action for an import operation. +type CommitActionType string + +const ( + // CommitActionWait implies that the import operation will finish its operations in order to + // be ready to be in the supervisor, it will wait for a user to commit. + CommitActionWait CommitActionType = "Wait" + + // CommitActionAuto implies that the import operation will finish its operations to be used + // in Supervisor, it will be automatically committed. + CommitActionAuto CommitActionType = "Auto" +) + +// RollbackActionType defines the type of rollback action for an import operation. +type RollbackActionType string + +const ( + // RollbackActionImmediate implies that the import operation will perform a best effort to rollback + // the workload to its original state prior to the import operation request. + RollbackActionImmediate RollbackActionType = "Immediate" + + // RollbackActionComplete implies that the import operation will complete the rollback immediately, + // any ongoing rollback will be given up and be stopped forever. + RollbackActionComplete RollbackActionType = "Complete" +) + +// ControlActionSpec defines the desired control actions for an import operation. +type ControlActionSpec struct { + // +optional + // +kubebuilder:default=CommitActionWait + + // CommitAction allows the user to decide what commit action to take for an import operation. + + // The default commit action will be CommitActionWait. + CommitAction CommitActionType `json:"commitAction,omitempty"` + + // +optional + + // RollbackAction allows the user to decide what rollback action to take for an import operation. + + // If RollbackAction is unset, no rollback action will be taken. + // The default rollback action will be unset. + RollbackAction RollbackActionType `json:"rollbackAction,omitempty"` + + // +optional + // +kubebuilder:default=false + + // PrecheckOnly indicates whether the import operation should perform only the precheck phase. + // + // When PrecheckOnly is set to true, the import operation will run prechecks to detect as many potential blockers + // as possible without altering any infrastructure state, then pause without proceeding further in the workflow. + // When set to false, the import operation continues with the rest of the workflow after prechecks. + // + // PrecheckOnly may be updated from true to false to resume the workflow, but cannot be updated from false to true, + // as it would be semantically unclear to perform only a precheck after other workflow steps have already executed. + PrecheckOnly *bool `json:"precheckOnly,omitempty"` +} + +// ImportOperationSpec defines the desired state of import operation. +type ImportOperationSpec struct { + // +kubebuilder:validation:Required + + // VirtualMachineID is the unique identifier of the virtual machine in vSphere to be imported. + VirtualMachineID string `json:"virtualMachineID"` + + // +optional + + // StorageClass specifies the name of the StorageClass resource used to configure storage-related attributes of the importing virtual machine. If unset, and if the target namespace has only a single StorageClass for VM service, that StorageClass will be used by default. + // + // Note: The name of the StorageClass is derived from the vSphere Storage Policy name and conforms to Kubernetes RFC 1123 Label Names. The StorageClass resource in the target namespace is allocated by the vSphere Storage Policy added to the namespace and is subject to StoragePolicyQuota restrictions. + StorageClass string `json:"storageClass,omitempty"` + + // +optional + // +listType=map + // +listMapKey=deviceKey + + // List of network device keys to Subnet information specifying the Subnets + // to which the VM's network devices should be connected. + // + // The DeviceKey is the device key of the network device on the + // VM, and the value is the Subnet information. Each device key must be + // unique within the list; overlapping or duplicate device keys are not + // allowed. + SubnetMappings []SubnetInfo `json:"subnetMappings,omitempty"` + + // +optional + + // NetworkCustomization is used to configure the network identity of the importing virtual machine by assigning one from the Supervisor Workload Network via Guest OS customization. + NetworkCustomization *NetworkCustomization `json:"networkCustomization,omitempty"` + + // +optional + + // +kubebuilder:validation:Minimum=0 + + // TTLSecondsAfterFinished limits the lifetime of an import operation after completion (either Succeeded or Failed). If set, the import operation is eligible for automatic deletion TTLSecondsAfterFinished seconds after it finishes. When being deleted, its lifecycle guarantees (e.g., finalizers) will be honored. If unset, the import operation will not be automatically deleted. If set to zero, the import operation becomes eligible for immediate deletion upon completion. + TTLSecondsAfterFinished *int32 `json:"ttlSecondsAfterFinished,omitempty"` + + // +optional + + // ControlAction specifies the desired control action for an import operation. + + // If unset, the import operation will follow the same workflow as the v1alpha1 behavior: + // It will proceed with completing the import into Supervisor without pausing for pre-check or commit. + // If the import operation is deleted before ImportOperationVirtualMachineCreated is set to true, + // the system will attempt a best-effort rollback to restore the workload to its original state. + // When ControlAction is set, it implies that the import operation will take action specified by ControlAction. + + // Once ControlAction is unset, it can not be updated. + // The default control action will be unset, to keep backward compatibility, v1alpha1 API does not support control action. + ControlAction *ControlActionSpec `json:"controlAction,omitempty"` +} + +// +kubebuilder:validation:Enum=ImportRelocate;RollbackRelocate + +// TaskType defines the type for monitored task types. +type TaskType string + +const ( + // TaskImportRelocate tracks the relocate task type for importing the virtual machine into the target namespace. + TaskImportRelocate TaskType = "ImportRelocate" + + // TaskRollbackRelocate tracks the relocate task type for rolling back the virtual machine to its original infrastructure. + TaskRollbackRelocate TaskType = "RollbackRelocate" +) + +// TaskMonitoringInfo provides detailed tracking information about a task's execution and progress over time. It maintains key data points such as the task's unique identifier, the intervals at which the task's status is polled, the last time the status was checked, and the task's progress percentage. +type TaskMonitoringInfo struct { + // Type specifies the type of the task being monitored. + Type TaskType `json:"type"` + + // TaskID is the unique identifier of the task in vSphere. + TaskID string `json:"taskID"` + + // PollIntervalSeconds defines the interval, in seconds, between consecutive status retrieval attempts. + // The controller starts with an initial interval of 1 second. If the `LastProgress` value has not changed since the last poll, the controller increases the interval exponentially. + PollIntervalSeconds int32 `json:"pollIntervalSeconds"` + + // LastPollTime is the timestamp of the most recent status retrieval for the task. + LastPollTime metav1.Time `json:"lastPollTime"` + + // LastProgress represents the most recent progress percentage of the task, recorded during the last status poll. + LastProgress int32 `json:"lastProgress"` +} + +// ImportOperationStatus defines the observed state of import operation. +type ImportOperationStatus struct { + // +optional + // +listType=map + // +listMapKey=type + + // Conditions describes the current condition information of the import operation object. + Conditions []metav1.Condition `json:"conditions,omitempty"` + + // +optional + + // StartTime describes the time when the import operation controller started processing the import operation. It is represented in RFC3339 format and is in UTC. + StartTime *metav1.Time `json:"startTime,omitempty"` + + // +optional + + // CompletionTime describes the time when the import operation was completed. It is not guaranteed to be set in a happens-before order across separate ImportOperations. It is represented in RFC3339 format and is in UTC. + // + // The value of this field should be equal to the value of the LastTransitionTime for the status condition `Type=ImportOperationCompleted`. + CompletionTime *metav1.Time `json:"completionTime,omitempty"` + + // +optional + + // VirtualMachineName is the name of the imported virtual machine in the target namespace. + VirtualMachineName string `json:"virtualMachineName,omitempty"` + + // +optional + // +listType=map + // +listMapKey=type + + // TaskMonitor tracks the status of tasks responsible for moving the imported virtual machine to the target infrastructure. + // Each entry corresponds to a specific task type, and only the most recent task status for each type is retained. + TaskMonitor []TaskMonitoringInfo `json:"taskMonitor,omitempty"` + + // +optional + + // StateTransitions records the latest virtual machine state change or action for each `TransitionType` performed during the import process. Each `StateTransition` entry corresponds to a specific `TransitionType`, and only the most recent transition for each type is retained. + StateTransitions []StateTransition `json:"stateTransitions,omitempty"` +} + +// GetConditions returns the list of conditions for the import operation. +func (importOp *ImportOperation) GetConditions() []metav1.Condition { + return importOp.Status.Conditions +} + +// SetConditions updates the conditions for the import operation. +func (importOp *ImportOperation) SetConditions(conditions []metav1.Condition) { + importOp.Status.Conditions = conditions +} + +// GetStateTransitions returns the state transitions for this import operation. +func (importOp *ImportOperation) GetStateTransitions() []StateTransition { + return importOp.Status.StateTransitions +} + +// SetStateTransitions sets the state transitions for this import operation. +func (importOp *ImportOperation) SetStateTransitions(transitions []StateTransition) { + importOp.Status.StateTransitions = transitions +} + +// IsDeleteTimeBeforeNow returns true if current time exceeds TTLSecondsAfterFinished + completionTime. +func (importOp *ImportOperation) IsDeleteTimeBeforeNow() bool { + if importOp.Spec.TTLSecondsAfterFinished == nil { + return false + } + + duration := time.Duration(*importOp.Spec.TTLSecondsAfterFinished) * time.Second + deletionTime := importOp.Status.CompletionTime.Add(duration) + return deletionTime.Before(time.Now()) +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:resource:scope=Namespaced,shortName=importop +// +kubebuilder:printcolumn:name="VM Name",type=string,JSONPath=`.status.virtualMachineName` +// +kubebuilder:printcolumn:name="Completed",type=string,JSONPath=`.status.conditions[?(@.type=="Completed")].status` + +// ImportOperation represents an import operation for a virtual machine in the API. +type ImportOperation struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ImportOperationSpec `json:"spec,omitempty"` + Status ImportOperationStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ImportOperationList contains a list of ImportOperation resources. +type ImportOperationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ImportOperation `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ImportOperation{}, &ImportOperationList{}) +} diff --git a/external/mobility-operator/api/v1alpha2/state_transitions.go b/external/mobility-operator/api/v1alpha2/state_transitions.go new file mode 100644 index 000000000..37e2ef1e2 --- /dev/null +++ b/external/mobility-operator/api/v1alpha2/state_transitions.go @@ -0,0 +1,207 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. + +package v1alpha2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +kubebuilder:validation:Enum=ClearPermissions;Move + +// TransitionType defines the type for state transitions. +type TransitionType string + +const ( + // TransitionClearPermissions indicates a transition where the virtual machine's non-inherited permissions are cleared. + TransitionClearPermissions TransitionType = "ClearPermissions" + + // TransitionMove indicates a transition where the virtual machine is moved to a different location. + TransitionMove TransitionType = "Move" +) + +// Permission contains details for the ClearPermissions transition. +type Permission struct { + // Group indicates whether the principal refers to a group (`true`) or a user (`false`). + Group bool `json:"group"` + + // Principal specifies the name of the user or group whose permissions are being cleared. + Principal string `json:"principal"` + + // Propagate indicates whether this permission propagates down the hierarchy to sub-entities. + Propagate bool `json:"propagate"` + + // RoleID is the unique identifier of the role providing access in vSphere. + RoleID int32 `json:"roleID"` +} + +// DiskStorageInfo represents storage information for a specific virtual disk. +type DiskStorageInfo struct { + // Name is the device key for the virtual disk. + // +kubebuilder:validation:Required + Name string `json:"name"` + + StorageInfo `json:",inline"` +} + +// StorageInfo contains information about the storage configuration for a virtual machine. +type StorageInfo struct { + // DatastoreID is the unique identifier of the datastore in vSphere. + DatastoreID string `json:"datastoreID"` + + // +optional + + // ProfileIDs is a list of unique storage profile identifiers in vSphere applied to the storage. + ProfileIDs []string `json:"profileIDs,omitempty"` +} + +// DeviceBackingInfo provides information about an infrastructure device or resource that backs a device in a virtual machine. +type DeviceBackingInfo struct { + // DeviceName is the name of the device. + DeviceName string `json:"deviceName"` +} + +// NetworkBackingInfo contains the network backing information for a virtual Ethernet card of a virtual machine. +type NetworkBackingInfo struct { + DeviceBackingInfo `json:",inline"` + + // +optional + + // NetworkID is the unique identifier of the network in vSphere. + NetworkID string `json:"networkID,omitempty"` +} + +// DistributedVirtualSwitchPortConnection specifies the connection details for a port on a distributed virtual switch. +type DistributedVirtualSwitchPortConnection struct { + // +optional + + // PortgroupKey is the key of the port group. + PortgroupKey string `json:"portgroupKey,omitempty"` + + // SwitchUUID is the UUID of the distributed virtual switch. + SwitchUUID string `json:"switchUUID"` +} + +// DistributedVirtualPortBackingInfo contains the network backing information for a virtual Ethernet card that connects to a distributed virtual switch port or port group. +type DistributedVirtualPortBackingInfo struct { + // PortConnection specifies the connection to a distributed virtual switch port. + PortConnection DistributedVirtualSwitchPortConnection `json:"portConnection"` +} + +// OpaqueNetworkBackingInfo contains the network backing information for a virtual Ethernet card that connects to an opaque network. +type OpaqueNetworkBackingInfo struct { + // OpaqueNetworkID is the unique identifier of the opaque network in vSphere. + OpaqueNetworkID string `json:"opaqueNetworkID"` + + // OpaqueNetworkType is the type of the opaque network. For example, "nsx.LogicalSwitch". + OpaqueNetworkType string `json:"opaqueNetworkType"` +} + +// NetworkInfo is a union type that specifies the network backing information. +// Only one of the fields should be set to represent the network backing type. +type NetworkInfo struct { + // Name is the device key for Virtual Ethernet Card. + Name string `json:"name"` + + // +optional + + // NetworkBacking specifies a standard network backing. + NetworkBacking *NetworkBackingInfo `json:"networkBacking,omitempty"` + + // +optional + + // DistributedVirtualPortBacking specifies a distributed virtual port backing. + DistributedVirtualPortBacking *DistributedVirtualPortBackingInfo `json:"distributedVirtualPortBacking,omitempty"` + + // +optional + + // OpaqueNetworkBacking specifies an opaque network backing. + OpaqueNetworkBacking *OpaqueNetworkBackingInfo `json:"opaqueNetworkBacking,omitempty"` +} + +// Location specifies the details of a virtual machine's location within the infrastructure. +type Location struct { + // +optional + + // DatacenterID specifies the unique identifier of the datacenter in vSphere where the virtual machine is located. + DatacenterID string `json:"datacenterID,omitempty"` + + // +optional + + // HostID specifies the unique identifier of the host in vSphere where the virtual machine is located. + HostID string `json:"hostID,omitempty"` + + // +optional + + // FolderID specifies the unique identifier of the folder in vSphere where the virtual machine is located. + FolderID string `json:"folderID,omitempty"` + + // +optional + + // ResourcePoolID specifies the unique identifier of the resource pool in vSphere where the virtual machine is assigned. + ResourcePoolID string `json:"resourcePoolID,omitempty"` + + // +optional + + // HomeStorage contains storage information for the virtual machine's home files. + HomeStorage *StorageInfo `json:"homeStorage,omitempty"` + + // +optional + // +listType=map + // +listMapKey=name + + // DiskStorage contains storage information for the virtual machine's disks. + DiskStorage []DiskStorageInfo `json:"diskStorage,omitempty"` + + // +optional + // +listType=map + // +listMapKey=name + + // Network specifies the network configuration for the virtual machine's virtual Ethernet cards. + Network []NetworkInfo `json:"network,omitempty"` + + // +optional + + // PowerState contains the power state of the virtual machine. + + PowerState string `json:"powerState,omitempty"` +} + +// MoveDetails contains details about the virtual machine move transition, including the original and new locations. +type MoveDetails struct { + // +optional + + // Source specifies the original location of the virtual machine before the move. + Source *Location `json:"source,omitempty"` + + // +optional + + // Destination specifies the new location of the virtual machine after the move. + Destination *Location `json:"destination,omitempty"` +} + +// TransitionDetails contains detailed information specific to the type of transition. +// Only one of the fields should be set, corresponding to the `TransitionType`. +type TransitionDetails struct { + // +optional + + // ClearedPermissions contains the virtual machine's non-inherited permissions that were cleared during the import. + ClearedPermissions []Permission `json:"clearedPermissions,omitempty"` + + // +optional + + // Move contains details if the transition is of type `Move`. + Move *MoveDetails `json:"move,omitempty"` +} + +// StateTransition records a specific state change or action performed during the virtual machine import process. +type StateTransition struct { + // Type specifies the type of state transition. + Type TransitionType `json:"type"` + + // Timestamp is the time when the transition was recorded. + // This field is always set. + Timestamp metav1.Time `json:"timestamp"` + + // Details holds additional information about the transition. + Details *TransitionDetails `json:"details,omitempty"` +} diff --git a/external/mobility-operator/api/v1alpha2/zz_generated.deepcopy.go b/external/mobility-operator/api/v1alpha2/zz_generated.deepcopy.go new file mode 100644 index 000000000..062e90977 --- /dev/null +++ b/external/mobility-operator/api/v1alpha2/zz_generated.deepcopy.go @@ -0,0 +1,519 @@ +//go:build !ignore_autogenerated + +// Copyright (c) Broadcom. All Rights Reserved. + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha2 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControlActionSpec) DeepCopyInto(out *ControlActionSpec) { + *out = *in + if in.PrecheckOnly != nil { + in, out := &in.PrecheckOnly, &out.PrecheckOnly + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlActionSpec. +func (in *ControlActionSpec) DeepCopy() *ControlActionSpec { + if in == nil { + return nil + } + out := new(ControlActionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomizationSysprepUserData) DeepCopyInto(out *CustomizationSysprepUserData) { + *out = *in + if in.ProductID != nil { + in, out := &in.ProductID, &out.ProductID + *out = new(ProductIDSecretKeySelector) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomizationSysprepUserData. +func (in *CustomizationSysprepUserData) DeepCopy() *CustomizationSysprepUserData { + if in == nil { + return nil + } + out := new(CustomizationSysprepUserData) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceBackingInfo) DeepCopyInto(out *DeviceBackingInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceBackingInfo. +func (in *DeviceBackingInfo) DeepCopy() *DeviceBackingInfo { + if in == nil { + return nil + } + out := new(DeviceBackingInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DiskStorageInfo) DeepCopyInto(out *DiskStorageInfo) { + *out = *in + in.StorageInfo.DeepCopyInto(&out.StorageInfo) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DiskStorageInfo. +func (in *DiskStorageInfo) DeepCopy() *DiskStorageInfo { + if in == nil { + return nil + } + out := new(DiskStorageInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DistributedVirtualPortBackingInfo) DeepCopyInto(out *DistributedVirtualPortBackingInfo) { + *out = *in + out.PortConnection = in.PortConnection +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DistributedVirtualPortBackingInfo. +func (in *DistributedVirtualPortBackingInfo) DeepCopy() *DistributedVirtualPortBackingInfo { + if in == nil { + return nil + } + out := new(DistributedVirtualPortBackingInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DistributedVirtualSwitchPortConnection) DeepCopyInto(out *DistributedVirtualSwitchPortConnection) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DistributedVirtualSwitchPortConnection. +func (in *DistributedVirtualSwitchPortConnection) DeepCopy() *DistributedVirtualSwitchPortConnection { + if in == nil { + return nil + } + out := new(DistributedVirtualSwitchPortConnection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImportOperation) DeepCopyInto(out *ImportOperation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImportOperation. +func (in *ImportOperation) DeepCopy() *ImportOperation { + if in == nil { + return nil + } + out := new(ImportOperation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ImportOperation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImportOperationList) DeepCopyInto(out *ImportOperationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ImportOperation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImportOperationList. +func (in *ImportOperationList) DeepCopy() *ImportOperationList { + if in == nil { + return nil + } + out := new(ImportOperationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ImportOperationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImportOperationSpec) DeepCopyInto(out *ImportOperationSpec) { + *out = *in + if in.SubnetMappings != nil { + in, out := &in.SubnetMappings, &out.SubnetMappings + *out = make([]SubnetInfo, len(*in)) + copy(*out, *in) + } + if in.NetworkCustomization != nil { + in, out := &in.NetworkCustomization, &out.NetworkCustomization + *out = new(NetworkCustomization) + (*in).DeepCopyInto(*out) + } + if in.TTLSecondsAfterFinished != nil { + in, out := &in.TTLSecondsAfterFinished, &out.TTLSecondsAfterFinished + *out = new(int32) + **out = **in + } + if in.ControlAction != nil { + in, out := &in.ControlAction, &out.ControlAction + *out = new(ControlActionSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImportOperationSpec. +func (in *ImportOperationSpec) DeepCopy() *ImportOperationSpec { + if in == nil { + return nil + } + out := new(ImportOperationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImportOperationStatus) DeepCopyInto(out *ImportOperationStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.StartTime != nil { + in, out := &in.StartTime, &out.StartTime + *out = (*in).DeepCopy() + } + if in.CompletionTime != nil { + in, out := &in.CompletionTime, &out.CompletionTime + *out = (*in).DeepCopy() + } + if in.TaskMonitor != nil { + in, out := &in.TaskMonitor, &out.TaskMonitor + *out = make([]TaskMonitoringInfo, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.StateTransitions != nil { + in, out := &in.StateTransitions, &out.StateTransitions + *out = make([]StateTransition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImportOperationStatus. +func (in *ImportOperationStatus) DeepCopy() *ImportOperationStatus { + if in == nil { + return nil + } + out := new(ImportOperationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Location) DeepCopyInto(out *Location) { + *out = *in + if in.HomeStorage != nil { + in, out := &in.HomeStorage, &out.HomeStorage + *out = new(StorageInfo) + (*in).DeepCopyInto(*out) + } + if in.DiskStorage != nil { + in, out := &in.DiskStorage, &out.DiskStorage + *out = make([]DiskStorageInfo, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Network != nil { + in, out := &in.Network, &out.Network + *out = make([]NetworkInfo, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Location. +func (in *Location) DeepCopy() *Location { + if in == nil { + return nil + } + out := new(Location) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MoveDetails) DeepCopyInto(out *MoveDetails) { + *out = *in + if in.Source != nil { + in, out := &in.Source, &out.Source + *out = new(Location) + (*in).DeepCopyInto(*out) + } + if in.Destination != nil { + in, out := &in.Destination, &out.Destination + *out = new(Location) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MoveDetails. +func (in *MoveDetails) DeepCopy() *MoveDetails { + if in == nil { + return nil + } + out := new(MoveDetails) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkBackingInfo) DeepCopyInto(out *NetworkBackingInfo) { + *out = *in + out.DeviceBackingInfo = in.DeviceBackingInfo +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkBackingInfo. +func (in *NetworkBackingInfo) DeepCopy() *NetworkBackingInfo { + if in == nil { + return nil + } + out := new(NetworkBackingInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkCustomization) DeepCopyInto(out *NetworkCustomization) { + *out = *in + if in.CustomizationSysprepUserData != nil { + in, out := &in.CustomizationSysprepUserData, &out.CustomizationSysprepUserData + *out = new(CustomizationSysprepUserData) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkCustomization. +func (in *NetworkCustomization) DeepCopy() *NetworkCustomization { + if in == nil { + return nil + } + out := new(NetworkCustomization) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInfo) DeepCopyInto(out *NetworkInfo) { + *out = *in + if in.NetworkBacking != nil { + in, out := &in.NetworkBacking, &out.NetworkBacking + *out = new(NetworkBackingInfo) + **out = **in + } + if in.DistributedVirtualPortBacking != nil { + in, out := &in.DistributedVirtualPortBacking, &out.DistributedVirtualPortBacking + *out = new(DistributedVirtualPortBackingInfo) + **out = **in + } + if in.OpaqueNetworkBacking != nil { + in, out := &in.OpaqueNetworkBacking, &out.OpaqueNetworkBacking + *out = new(OpaqueNetworkBackingInfo) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInfo. +func (in *NetworkInfo) DeepCopy() *NetworkInfo { + if in == nil { + return nil + } + out := new(NetworkInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpaqueNetworkBackingInfo) DeepCopyInto(out *OpaqueNetworkBackingInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpaqueNetworkBackingInfo. +func (in *OpaqueNetworkBackingInfo) DeepCopy() *OpaqueNetworkBackingInfo { + if in == nil { + return nil + } + out := new(OpaqueNetworkBackingInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Permission) DeepCopyInto(out *Permission) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Permission. +func (in *Permission) DeepCopy() *Permission { + if in == nil { + return nil + } + out := new(Permission) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProductIDSecretKeySelector) DeepCopyInto(out *ProductIDSecretKeySelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProductIDSecretKeySelector. +func (in *ProductIDSecretKeySelector) DeepCopy() *ProductIDSecretKeySelector { + if in == nil { + return nil + } + out := new(ProductIDSecretKeySelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StateTransition) DeepCopyInto(out *StateTransition) { + *out = *in + in.Timestamp.DeepCopyInto(&out.Timestamp) + if in.Details != nil { + in, out := &in.Details, &out.Details + *out = new(TransitionDetails) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StateTransition. +func (in *StateTransition) DeepCopy() *StateTransition { + if in == nil { + return nil + } + out := new(StateTransition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageInfo) DeepCopyInto(out *StorageInfo) { + *out = *in + if in.ProfileIDs != nil { + in, out := &in.ProfileIDs, &out.ProfileIDs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageInfo. +func (in *StorageInfo) DeepCopy() *StorageInfo { + if in == nil { + return nil + } + out := new(StorageInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetInfo) DeepCopyInto(out *SubnetInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetInfo. +func (in *SubnetInfo) DeepCopy() *SubnetInfo { + if in == nil { + return nil + } + out := new(SubnetInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskMonitoringInfo) DeepCopyInto(out *TaskMonitoringInfo) { + *out = *in + in.LastPollTime.DeepCopyInto(&out.LastPollTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskMonitoringInfo. +func (in *TaskMonitoringInfo) DeepCopy() *TaskMonitoringInfo { + if in == nil { + return nil + } + out := new(TaskMonitoringInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TransitionDetails) DeepCopyInto(out *TransitionDetails) { + *out = *in + if in.ClearedPermissions != nil { + in, out := &in.ClearedPermissions, &out.ClearedPermissions + *out = make([]Permission, len(*in)) + copy(*out, *in) + } + if in.Move != nil { + in, out := &in.Move, &out.Move + *out = new(MoveDetails) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TransitionDetails. +func (in *TransitionDetails) DeepCopy() *TransitionDetails { + if in == nil { + return nil + } + out := new(TransitionDetails) + in.DeepCopyInto(out) + return out +} diff --git a/external/mobility-operator/go.mod b/external/mobility-operator/go.mod new file mode 100644 index 000000000..15895c7e4 --- /dev/null +++ b/external/mobility-operator/go.mod @@ -0,0 +1,27 @@ +module github.com/vmware-tanzu/vm-operator/external/mobility-operator + +go 1.26.1 + +require ( + k8s.io/apimachinery v0.35.3 + sigs.k8s.io/controller-runtime v0.23.3 +) + +require ( + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/text v0.31.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect +) diff --git a/external/mobility-operator/go.sum b/external/mobility-operator/go.sum new file mode 100644 index 000000000..e1c4518a1 --- /dev/null +++ b/external/mobility-operator/go.sum @@ -0,0 +1,80 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= +k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= +k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8= +k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80= +sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/external/net-operator/api/v1alpha1/aviloadbalancerconfig_types.go b/external/net-operator/api/v1alpha1/aviloadbalancerconfig_types.go new file mode 100644 index 000000000..caad6d4ad --- /dev/null +++ b/external/net-operator/api/v1alpha1/aviloadbalancerconfig_types.go @@ -0,0 +1,124 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// AviLoadBalancerLogLevel is a valid log level for the Avi Kubernetes Operator. +type AviLoadBalancerLogLevel string + +const ( + // AviLoadBalancerLogLevelInfo is the INFO log level for AKO. + AviLoadBalancerLogLevelInfo AviLoadBalancerLogLevel = "INFO" + // AviLoadBalancerLogLevelDebug is the DEBUG log level for AKO. + AviLoadBalancerLogLevelDebug AviLoadBalancerLogLevel = "DEBUG" + // AviLoadBalancerLogLevelWarn is the WARN log level for AKO. + AviLoadBalancerLogLevelWarn AviLoadBalancerLogLevel = "WARN" + // AviLoadBalancerLogLevelError is the ERROR log level for AKO. + AviLoadBalancerLogLevelError AviLoadBalancerLogLevel = "ERROR" +) + +// AviLoadBalancerIPAMType is the type of IPAM used by Avi. +type AviLoadBalancerIPAMType string + +const ( + // AviLoadBalancerSupervisorIPAM indicates that IPAM is provided by the + // Supervisor cluster. + AviLoadBalancerSupervisorIPAM AviLoadBalancerIPAMType = "supervisor" + // AviLoadBalancerControllerIPAM indicates that IPAM is provided by the Avi + // Controller. + AviLoadBalancerControllerIPAM AviLoadBalancerIPAMType = "controller" +) + +// AviLoadBalancerConfigSpec defines the configuration for an Avi load balancer. +// This specification is used to configure the resources the Avi Kubernetes +// Operator (AKO) requires in order to connect to the Avi load balancer. +type AviLoadBalancerConfigSpec struct { + // Server is the endpoint at which the Avi Controller REST API is available. + // The format is [SCHEME://]ADDRESS[:PORT], ex. https://10.10.10.10 + // * SCHEME may be http or https and defaults to https if the SCHEME is + // omitted + // * ADDRESS is the Avi Controller IP address or the Avi Cluster IP when + // two or more Avi Controllers are deployed in cluster mode. + // * PORT defaults to 80 when SCHEME is http and 443 when SCHEME is https. + Server string `json:"server"` + + // CloudName is used by the Avi Kubernetes Operator (AKO) when querying + // properties via the Avi REST API, ex. /api/cloud/?name=CLOUD_NAME. + // Defaults to Default-Cloud. + // +kubebuilder:default:=Default-Cloud + CloudName string `json:"cloudName,omitempty"` + + // AdvancedL4 is a flag that enables support for WCP in AKO. + // Defaults to true. + // +kubebuilder:default:=true + AdvancedL4 *bool `json:"advancedL4,omitempty"` + + // LogLevel specifies the log level used by AKO. + // +kubebuilder:default:=WARN + // +kubebuilder:validation:Enum=INFO;DEBUG;WARN;ERROR + LogLevel AviLoadBalancerLogLevel `json:"logLevel,omitempty"` + + // IPAMType is the type of IPAM used by the Avi Software Load Balancer. + // +kubebuilder:default:=controller + // +kubebuilder:validation:Enum=controller;supervisor + IPAMType AviLoadBalancerIPAMType `json:"ipamType,omitempty"` + + // CredentialSecretRef points to a Secret resource used to access and + // configure the Avi Controller. + // + // * certificateAuthorityData PEM-encoded certificate authority + // certificates + // * username Username used with basic authentication for + // the Avi REST API + // * password Password used with basic authentication for + // the Avi REST API + // + // The following YAML is an example secret: + // + // apiVersion: v1 + // kind: Secret + // metadata: + // name: avi-lb-config + // namespace: vmware-system-netop + // data: + // certificateAuthorityData: []byte + // username: []byte + // password: []byte + CredentialSecretRef ClientSecretReference `json:"credentialSecretRef"` +} + +// AviLoadBalancerConfigStatus is unused because AviLoadBalancerConfigSpec is +// purely a configuration resource. +type AviLoadBalancerConfigStatus struct { +} + +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster + +// AviLoadBalancerConfig is the Schema for the AviLoadBalancerConfigs API +type AviLoadBalancerConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AviLoadBalancerConfigSpec `json:"spec,omitempty"` + Status AviLoadBalancerConfigStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// AviLoadBalancerConfigList contains a list of AviLoadBalancerConfig +type AviLoadBalancerConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AviLoadBalancerConfig `json:"items"` +} + +func init() { + RegisterTypeWithScheme(&AviLoadBalancerConfig{}, &AviLoadBalancerConfigList{}) +} diff --git a/external/net-operator/api/v1alpha1/doc.go b/external/net-operator/api/v1alpha1/doc.go new file mode 100644 index 000000000..098bb067f --- /dev/null +++ b/external/net-operator/api/v1alpha1/doc.go @@ -0,0 +1,7 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// +k8s:openapi-gen=true +// +kubebuilder:object:generate=true +// +groupName=netoperator.vmware.com +package v1alpha1 diff --git a/external/net-operator/api/v1alpha1/foundationloadbalancerconfig_types.go b/external/net-operator/api/v1alpha1/foundationloadbalancerconfig_types.go new file mode 100644 index 000000000..27561a8e7 --- /dev/null +++ b/external/net-operator/api/v1alpha1/foundationloadbalancerconfig_types.go @@ -0,0 +1,278 @@ +// Copyright (c) 2024 Broadcom. All Rights Reserved. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // FoundationLoadBalancerConditionHealthy reflects the health status of the load balancer data-plane's runtime. + FoundationLoadBalancerConditionHealthy FoundationLoadBalancerConditionType = "Healthy" + // FoundationLoadBalancerConditionDeploymentStatusReady reflects the deployment status of the load balancer node(s). + FoundationLoadBalancerConditionDeploymentStatusReady FoundationLoadBalancerConditionType = "DeploymentStatusReady" + // FoundationLoadBalancerConditionOperationStatusReady reflects the operation status of the load balancer instance. + FoundationLoadBalancerConditionOperationStatusReady FoundationLoadBalancerConditionType = "OperationStatusReady" + + FoundationLoadBalancerSizeSmall FoundationLoadBalancerSize = "small" + FoundationLoadBalancerSizeMedium FoundationLoadBalancerSize = "medium" + FoundationLoadBalancerSizeLarge FoundationLoadBalancerSize = "large" + FoundationLoadBalancerSizeXL FoundationLoadBalancerSize = "xlarge" + + FoundationAvailabilityModeActivePassive FoundationLoadBalancerAvailabilityMode = "active-passive" + FoundationAvailabilityModeSingleNode FoundationLoadBalancerAvailabilityMode = "single-node" +) + +type FoundationLoadBalancerConditionType string +type FoundationLoadBalancerTopologyType string +type FoundationLoadBalancerSize string +type FoundationLoadBalancerAvailabilityMode string + +// Spec objects. Input for FLB deployment. + +// FoundationLoadBalancerDeploymentSpec describes how to deploy the load balancer. +type FoundationLoadBalancerDeploymentSpec struct { + // Size describes the node form factor. + // + // +kubebuilder:validation:Enum=small;medium;large;xlarge + // +kubebuilder:default:=small + Size FoundationLoadBalancerSize `json:"size"` + + // StoragePolicy is a vSphere Storage Policy ID which defines node storage placement. + StoragePolicy string `json:"storagePolicy"` + + // Version number desired by the operator. + // + // Defaults to the latest available. + // + // +optional + Version string `json:"version,omitempty"` + + // Zones contains the names of zones eligible for placing nodes. Zones must be one of the + // AvailabilityZones defined and eligible for placement on the cluster. + Zones []string `json:"zones"` + + // AvailabilityMode defines how the availability of the solution is deployed and configured. + // +kubebuilder:validation:Enum=active-passive;single-node + // +kubebuilder:default:=active-passive + AvailabilityMode FoundationLoadBalancerAvailabilityMode `json:"availabilityMode"` + + // ActivePassiveAvailabilityMode configures the load balancer in active-passive configuration. + // Active-passive configuration consists of a two node deployment with one node configured to + // actively service traffic with the second node in standby mode. When the service detects the + // active node is unhealthy, traffic will be moved to the passive node after a short delay. + // Connections may be dropped on fail-over. + // + // +optional + ActivePassiveAvailabilityMode *ActivePassiveAvailabilityMode `json:"activePassiveSpec,omitempty"` + + // SingleNodeAvailabilityMode deploys a single node to serve load balancer traffic. If the node + // fails, the service will attempt to redeploy it, but redeployment is best-effort and depends on + // the health of the underlying infrastructure. You must select + // + // +optional + SingleNodeAvailabilityMode *SingleNodeAvailabilityMode `json:"singleNodeSpec,omitempty"` +} + +// ActivePassiveAvailabilityMode deploys two nodes in Active-Passive mode where one node is set into +// active state and is responsible for serving traffic, and one node is passive - +// awaiting a fail-over event. When a fail-over occurs, connections to and from the load balancer +// may be reset. +type ActivePassiveAvailabilityMode struct { + // Replicas describes the total number of deployed nodes. Defaults to 2. + // + // +kubebuilder:validation:Maximum=2 + // +kubebuilder:default:=2 + Replicas uint32 `json:"replicas"` +} + +// SingleNodeAvailabilityMode defines single node configuration. Single node configuration involves +// trading availability in return for reduced resource consumption. Upon node failure, redeployment will +// be attempted on a best-effort basis. +type SingleNodeAvailabilityMode struct { + // Replicas describes the total number of deployed nodes. Defaults to 1. + // + // +kubebuilder:validation:Maximum=1 + // +kubebuilder:default:=1 + Replicas uint32 `json:"replicas"` +} + +// Status objects. Specs are realized into Statuses. + +// FoundationLoadBalancerNodeStatus describes the per-node status of the load balancer. +type FoundationLoadBalancerNodeStatus struct { + // NodeID is a node's unique identifier. + NodeID string `json:"nodeID"` + + // ManagementNetworkInterface defines the management NetworkInterface if it exists. + // + // +optional + ManagementNetworkInterface NetworkInterfaceReference `json:"managementNetworkInterface,omitempty"` + + // WorkloadNetworkInterface defines the workload NetworkInterfaces if they exist. + // + // +optional + WorkloadNetworkInterfaces []NetworkInterfaceReference `json:"workloadNetworkInterfaces,omitempty"` + + // VIPNetworkInterface is the interface bound to the Virtual IP Network. + VIPNetworkInterface NetworkInterfaceReference `json:"vipNetworkInterface"` +} + +// FoundationLoadBalancerConfigStatus describes the observed state of the Foundation Load Balancer. +type FoundationLoadBalancerConfigStatus struct { + // Version describes the current version of the Foundation Load Balancer. + // + // +optional + Version string `json:"version,omitempty"` + + // Nodes list specific information about each deployed node. + // + // +optional + Nodes []FoundationLoadBalancerNodeStatus `json:"nodes,omitempty"` + + // VirtualServerIPPoolsUtilization describes the current states of virtual server IP addresses utilization. + // + // +optional + VirtualServerIPPoolsUtilization VirtualIPPoolsUtilization `json:"virtualServerIPPoolsUtilization,omitempty"` + + // Conditions describes states of the load balancer at specific points in time. + // + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// VirtualIPPoolsUtilization defines the IP addresses utilization for virtual IPPools resource. +type VirtualIPPoolsUtilization struct { + // IPsAllocated represents the total number of virtual IP addresses currently allocated to services. + // + // +optional + IPsAllocated int64 `json:"ipsAllocated,omitempty"` + + // IPsAvailable represents the total number of virtual IP addresses eligible to be used for services. + // + // +optional + IPsAvailable int64 `json:"ipsAvailable,omitempty"` +} + +// FoundationLoadBalancerConfigSpec defines the configuration for a vSphere Foundation Load Balancer. +// This specification is used to configure the resources for the load balancer on vCenter Server. +type FoundationLoadBalancerConfigSpec struct { + // DeploymentSpec describes sizing and placement constraints of the load balancer. + DeploymentSpec FoundationLoadBalancerDeploymentSpec `json:"deploymentSpec"` + + // ManagementNetwork points to the Network used to program node management network interfaces. + // + // If unset, the VirtualIPNetwork will be used for management traffic. + // + // +optional + ManagementNetwork *NetworkReference `json:"managementNetwork,omitempty"` + + // WorkloadNetwork points to the Network used to program node workload network interfaces. + // + // If unset, workload data traffic will be routed out of the same NIF bound to VirtualIPNetwork. + // + // +kubebuilder:validation:MaxItems:=1 + // +optional + WorkloadNetworks []NetworkReference `json:"workloadNetworks,omitempty"` + + // VirtualIPNetwork points to the Network used to program node VIP network interfaces. + VirtualIPNetwork NetworkReference `json:"virtualIPNetwork"` + + // NetworkSpec contains values for configuring networks on the load balancer. + // If unset, default settings will be applied. + // + // +optional + NetworkSpec FoundationLoadBalancerNetworkConfigSpec `json:"networkSpec,omitempty"` +} + +// FoundationLoadBalancerNetworkConfigSpec contains values for configuring networks on the load balancer. +type FoundationLoadBalancerNetworkConfigSpec struct { + // VirtualServerIPPools are the list of IPPools that are + // used for load balancer IP addresses. + VirtualServerIPPools []IPPoolReference `json:"virtualServerIPPools"` + + // VirtualServerSubnets are the list of subnets specified in CIDR notation + // that are directly connected to the VirtualIPNetwork. + // + // The VirtualServerIPPools must fall within the subnet of the VirtualIPNetwork + // or one of these subnets. + // + // +kubebuilder:default:={} + // +optional + VirtualServerSubnets []string `json:"virtualServerSubnets"` + + // DNSServers is the list of servers used for DNS traffic. + // These servers must be reachable from the network configured + // for management traffic. + // + // +kubebuilder:default:={} + // +optional + DNSServers []string `json:"dnsServers"` + + // DNSSearchDomains are the domains resolvable on the specified DNSServers. + // + // +kubebuilder:default:={} + // +optional + DNSSearchDomains []string `json:"dnsSearchDomains"` + + // NTPServers are the servers used to sync time across nodes. + // These servers must be reachable from the network configured + // for management traffic. + // + // +kubebuilder:default:={} + // +optional + NTPServers []string `json:"ntpServers"` + + // SyslogEndpoint configures the syslog server. It accepts a protocol, host and port. + // If using TLS, you must configure a TLS CA that is capable of verifying the endpoint certificate. + // E.g. [protocol://]host[:port] + // This server must be reachable from the network configured for management traffic. + // + // If empty, data will be logged locally to load balancer nodes. + // Defaults to port 514 if using UDP and 6514 if using TLS. + // + // +optional + SyslogEndpoint string `json:"syslogEndpoint,omitempty"` + + // SyslogCertificateSecretName is the certificate required to verify + // the TLS syslog endpoint in PEM format. + // + // +optional + SyslogCertificate string `json:"syslogCertificate,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=flb + +// FoundationLoadBalancerConfig is the Schema for the FoundationLoadBalancerConfig API +type FoundationLoadBalancerConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FoundationLoadBalancerConfigSpec `json:"spec,omitempty"` + Status FoundationLoadBalancerConfigStatus `json:"status,omitempty"` +} + +func (flb *FoundationLoadBalancerConfig) GetConditions() []metav1.Condition { + return flb.Status.Conditions +} + +func (flb *FoundationLoadBalancerConfig) SetConditions(conditions []metav1.Condition) { + flb.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true + +// FoundationLoadBalancerConfigList contains a list of FoundationLoadBalancerConfig. +type FoundationLoadBalancerConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []FoundationLoadBalancerConfig `json:"items"` +} + +func init() { + RegisterTypeWithScheme(&FoundationLoadBalancerConfig{}, &FoundationLoadBalancerConfigList{}) +} diff --git a/external/net-operator/api/v1alpha1/groupversion_info.go b/external/net-operator/api/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..9648b465d --- /dev/null +++ b/external/net-operator/api/v1alpha1/groupversion_info.go @@ -0,0 +1,38 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName specifies the group name used to register the objects. +const GroupName = "netoperator.vmware.com" + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &runtime.SchemeBuilder{} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +// Resource takes an unqualified resource and returns a Group qualified GroupResource +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +// RegisterTypeWithScheme adds objects to the SchemeBuilder +func RegisterTypeWithScheme(object ...runtime.Object) { + SchemeBuilder.Register(func(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, object...) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil + }) +} diff --git a/external/net-operator/api/v1alpha1/haproxyloadbalancerconfig_types.go b/external/net-operator/api/v1alpha1/haproxyloadbalancerconfig_types.go new file mode 100644 index 000000000..852216046 --- /dev/null +++ b/external/net-operator/api/v1alpha1/haproxyloadbalancerconfig_types.go @@ -0,0 +1,92 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// HAProxyLoadBalancerConfigSpec defines the configuration for an HAProxyLoadBalancerConfig instance. +// The spec is used to configure the HAProxyLoadBalancer instance to correctly route traffic to services. +// This spec supports HAProxyLoadBalancerConfig Dataplane API 2.0+ sidecar +type HAProxyLoadBalancerConfigSpec struct { + // EndPointURLs is a list of the addresses for the DataPlane API servers used + // to configure HAProxy. + // One or more DataPlane API endpoints are possible due to the following topologies: + // Single Node Topology + // Multi-Node Active/Passive Topology + // The strings should include the host, port, and API version, ex.: + // https://hostname:port/v1 + // +kubebuilder:validation:MinItems=1 + EndPointURLs []string `json:"endPointURLs"` + + // ServerName is used to verify the hostname on the returned + // certificates. It is also included + // in the client's handshake to support virtual hosting unless it is + // an IP address. + // Defaults to the host part parsed from Server + // +optional + ServerName string `json:"serverName,omitempty"` + + // CredentialSecretRef is an object name of kind Secret. + // It will be used to access and configure the HAProxy load balancer DataPlane API servers. + // The following fields are optional: + // + // * certificateAuthorityData - CertificateAuthorityData contains PEM-encoded certificate authority certificates. + // + // * clientCertificateData - ClientCertificateData contains PEM-encoded data from a client cert file. + // + // * clientKeyData - ClientKeyData contains PEM-encoded data from a client key file for TLS. + // + // * username - Username is the username for basic authentication. Defaults to "client". + // + // * password - Password is the password for basic authentication. Defaults to "cert". + // + // Sample of a secret: + // + // apiVersion: v1 + // kind: Secret + // metadata: + // name: haproxy-lb-config + // namespace: vmware-system-netop + // data: + // certificateAuthorityData: + // clientCertificateData: + // clientKeyData: + // username: + // password: + // +optional + CredentialSecretRef ClientSecretReference `json:"credentialSecretRef,omitempty"` +} + +// HAProxyLoadBalancerConfigStatus is unused. This is because HAProxyLoadBalancerConfig is purely a configuration resource +type HAProxyLoadBalancerConfigStatus struct { +} + +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster + +// HAProxyLoadBalancerConfig is the Schema for the HAProxyLoadBalancerConfigs API +type HAProxyLoadBalancerConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec HAProxyLoadBalancerConfigSpec `json:"spec,omitempty"` + Status HAProxyLoadBalancerConfigStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// HAProxyLoadBalancerConfigList contains a list of HAProxyLoadBalancerConfig +type HAProxyLoadBalancerConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []HAProxyLoadBalancerConfig `json:"items"` +} + +func init() { + RegisterTypeWithScheme(&HAProxyLoadBalancerConfig{}, &HAProxyLoadBalancerConfigList{}) +} diff --git a/external/net-operator/api/v1alpha1/ipaddressallocation_types.go b/external/net-operator/api/v1alpha1/ipaddressallocation_types.go new file mode 100644 index 000000000..56f137abe --- /dev/null +++ b/external/net-operator/api/v1alpha1/ipaddressallocation_types.go @@ -0,0 +1,91 @@ +// Copyright (c) 2020-2024 Broadcom. All Rights Reserved. + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// IPAddressAllocationFinalizer is a finalizer that allows the controller to perform cleanup +// of resources associated with an IPAddressAllocation before it is removed from the API Server. +const IPAddressAllocationFinalizer = "ipaddressallocation.netoperator.vmware.com" + +// IPAddressAllocationConditionType is a string type for the condition types of an IPAddressAllocation. +type IPAddressAllocationConditionType string + +const ( + // IPAddressAllocationReady indicates the IP has been successfully allocated. + IPAddressAllocationReady IPAddressAllocationConditionType = "Ready" + // IPAddressAllocationFail indicates an error was encountered during allocation. + IPAddressAllocationFail IPAddressAllocationConditionType = "Failure" +) + +// IPAddressAllocationConditionReason describes the reason for the last transition of a condition. +type IPAddressAllocationConditionReason string + +const ( + // IPAddressAllocationConditionInvalidRequestedIP is used when the IPAddressAllocation fails due to an invalid RequestedIP. + IPAddressAllocationConditionInvalidRequestedIP IPAddressAllocationConditionReason = "InvalidRequestedIP" + // IPAddressAllocationConditionFailureReasonCannotAllocIP is used when the IPAddressAllocation fails because an IP cannot be allocated. + IPAddressAllocationConditionFailureReasonCannotAllocIP IPAddressAllocationConditionReason = "CannotAllocIP" + // IPAddressAllocationConditionFailureReasonIPPoolRefRetrievalFailed is used when retrieval of the IPPoolRef has failed. + IPAddressAllocationConditionFailureReasonIPPoolRefRetrievalFailed IPAddressAllocationConditionReason = "IPPoolRefRetrievalFailed" +) + +// IPAddressAllocationCondition describes the state of an IPAddressAllocation at a specific point in time. +type IPAddressAllocationCondition struct { + // Type is the type of the condition. + Type IPAddressAllocationConditionType `json:"type"` + // Status reflects whether the condition is True, False, or Unknown. + Status corev1.ConditionStatus `json:"status"` + // LastTransitionTime is the timestamp of the last change to the condition's status. + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + // Reason provides a machine-readable explanation for the last status transition. + Reason IPAddressAllocationConditionReason `json:"reason,omitempty"` + // Message provides a human-readable explanation for the last status transition. + Message string `json:"message,omitempty"` +} + +// IPAddressAllocationSpec defines the desired state of an IPAddressAllocation, including the pool reference and an optional requested IP. +type IPAddressAllocationSpec struct { + // PoolRef is the reference to the network's IP pool within the namespace. + // It currently only supports reference to a Network. + PoolRef corev1.TypedLocalObjectReference `json:"poolRef"` + // RequestedIP is an optional field for a user to specify a particular IP they want to request. + // If omitted, the system will allocate a single IP address. + RequestedIP string `json:"requestedIP,omitempty"` +} + +// IPAddressAllocationStatus contains the current status of an IPAddressAllocation, including the allocated IP address and conditions. +type IPAddressAllocationStatus struct { + // IPAddress is the actually allocated IP address. + IPAddress string `json:"ipaddress,omitempty"` + // Conditions provide detailed information about the status of the allocation. + Conditions []IPAddressAllocationCondition `json:"conditions,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true + +// IPAddressAllocation represents a request for IP address allocation, including the desired state and current status. +type IPAddressAllocation struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec IPAddressAllocationSpec `json:"spec,omitempty"` + Status IPAddressAllocationStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// IPAddressAllocationList is a list of IPAddressAllocation objects. +type IPAddressAllocationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []IPAddressAllocation `json:"items"` +} + +// init function registers the IPAddressAllocation type with the scheme. +func init() { + RegisterTypeWithScheme(&IPAddressAllocation{}, &IPAddressAllocationList{}) +} diff --git a/external/net-operator/api/v1alpha1/ippool_types.go b/external/net-operator/api/v1alpha1/ippool_types.go new file mode 100644 index 000000000..2300b2470 --- /dev/null +++ b/external/net-operator/api/v1alpha1/ippool_types.go @@ -0,0 +1,116 @@ +// Copyright (c) 2020-2024 Broadcom. All Rights Reserved. + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// IPAMDisabledAnnotationKeyName is the name of the annotation added to +// GatewayClass resources that do not participate in net-operator's IPAM. +// The value does not need to be truthy; the presence of the key is what +// disables net-operator's IPAM for that GatewayClass. +const IPAMDisabledAnnotationKeyName = "netoperator.vmware.com/ipam-disabled" + +type IPPoolUsageLabelValue string + +const ( + // IPPoolUsageLabelKeyName is the name of a label used to indicate how IP pools + // should be used. To create an affinity, you must create a NetworkInterface with a + // label matching the intended use. For example, if you create a NetworkInterface with + // a label matching netoperator.vmware.com/ipam-usage=vip, then net operator + // will only provision from IPPools matching that label falling back to the general + // pool if needed unless IPPoolUsageAnnotationStrictKeyName is set. + IPPoolUsageLabelKeyName = "netoperator.vmware.com/ipam-usage" + + // IPPoolUsageAnnotationStrictKeyName indicates that an interface should not attempt + // to retrieve IPPools meant for general purpose consumption. For example, if "vip" is set, + // only IPPools matching the "vip" label will be used and "general" will not be used as a pool. + IPPoolUsageAnnotationStrictKeyName = "netoperator.vmware.com/ipam-strict-usage" + + // IPPoolUsageLabelGeneralValue indicates an IP pool can be used for any purpose. + // If a usage label is omitted from an IPPool, this value is implied. + IPPoolUsageLabelGeneralValue IPPoolUsageLabelValue = "general" + + // IPPoolUsageLabelVIPValue indicates an IP pool is reserved for a NetworkInterface + // which provisions virtual IP addresses. + IPPoolUsageLabelVIPValue IPPoolUsageLabelValue = "vip" +) + +type IPPoolConditionType string + +const ( + // IPPoolFull condition is added when no more IPs are free in the pool. + IPPoolFull IPPoolConditionType = "full" + // IPPoolReady condition is added when IPPool has been realized. + IPPoolReady IPPoolConditionType = "ready" + // IPPoolFail condition is added when an error was encountered in realizing. + IPPoolFail IPPoolConditionType = "failure" +) + +// IPPoolCondition describes the state of a IPPool at a certain point. +type IPPoolCondition struct { + // Type is the type of IPPool condition. + Type IPPoolConditionType `json:"type"` + // Status is the status of the condition. + // Can be True, False, Unknown. + Status corev1.ConditionStatus `json:"status"` + // Machine understandable string that gives the reason for condition's last transition. + Reason string `json:"reason,omitempty"` + // Human-readable message indicating details about last transition. + Message string `json:"message,omitempty"` +} + +// IPPoolSpec defines the desired state of IPPool. +type IPPoolSpec struct { + // StartingAddress represents the starting IP address of the pool. + StartingAddress string `json:"startingAddress"` + // AddressCount represents the number of IP addresses in the pool. + AddressCount int64 `json:"addressCount"` +} + +// IPPoolStatus defines the current state of IPPool. +type IPPoolStatus struct { + // Allocated represents the number of IP addresses currently allocated to services. + Allocated int64 `json:"allocated,omitempty"` + // Conditions is an array of current observed IPPool conditions. + Conditions []IPPoolCondition `json:"conditions,omitempty"` +} + +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster + +// IPPool is the Schema for the ippools API. +// It represents a pool of IP addresses that are owned and managed by the IPPool controller. +// Provider specific networks can associate themselves with IPPool objects to use +// network operator's IPAM implementation. +type IPPool struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec IPPoolSpec `json:"spec,omitempty"` + Status IPPoolStatus `json:"status,omitempty"` +} + +type IPPoolReference struct { + // Name of the IPPool resource being referenced. + Name string `json:"name"` + // API version of the referent. + APIVersion string `json:"apiVersion,omitempty"` +} + +// +kubebuilder:object:root=true + +// IPPoolList contains a list of IPPool +type IPPoolList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []IPPool `json:"items"` +} + +func init() { + RegisterTypeWithScheme(&IPPool{}, &IPPoolList{}) +} diff --git a/external/net-operator/api/v1alpha1/loadbalancerconfig_types.go b/external/net-operator/api/v1alpha1/loadbalancerconfig_types.go new file mode 100644 index 000000000..6d07dd28f --- /dev/null +++ b/external/net-operator/api/v1alpha1/loadbalancerconfig_types.go @@ -0,0 +1,116 @@ +// Copyright (c) 2020-2024 Broadcom. All Rights Reserved. + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ClientSecretReference contains info to locate an object of Kind Secret +// which contains credential specifications for a load balancer. +type ClientSecretReference struct { + // Name is the name of resource being referenced. + Name string `json:"name"` + // Namespace of the resource being referenced. If empty, cluster scoped resource is assumed. + // +kubebuilder:default:=default + Namespace string `json:"namespace,omitempty"` +} + +// LoadBalancerConfigConditionType is used as a typed string for representing +// LoadBalancerConfig.Status.Conditions. +type LoadBalancerConfigConditionType string + +const ( + // LoadBalancerConfigReady is added when the LoadBalancerConfig object has been successfully realized + LoadBalancerConfigReady LoadBalancerConfigConditionType = "Ready" + // LoadBalancerConfigFailure is added if any failure is encountered while realizing LoadBalancerConfig object + LoadBalancerConfigFailure LoadBalancerConfigConditionType = "Failure" + // LoadBalancerConfigIPPoolPressure condition status is set to True when IPPool is low on free IPs. + LoadBalancerConfigIPPoolPressure LoadBalancerConfigConditionType = "IPPoolPressure" +) + +// LoadBalancerConfigCondition describes the state of a LoadBalancerConfig at a certain point +type LoadBalancerConfigCondition struct { + // Type is the type of load balancer condition + // Can be Ready or Failure + Type LoadBalancerConfigConditionType `json:"type"` + // Status is the status of the condition + // Can be True, False, Unknown + Status corev1.ConditionStatus `json:"status"` + // Machine understandable string that gives the reason for the condition's last transition + // +optional + Reason string `json:"reason,omitempty"` + // Human-readable message indicating details about last transition + // +optional + Message string `json:"message,omitempty"` + // Provides a timestamp for when the LoadBalancerConfig object last transitioned from one status to another + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty" patchStrategy:"replace"` +} + +// LoadBalancerConfigProviderReference represents the specific load balancer instance that needs to be configured +type LoadBalancerConfigProviderReference struct { + // APIGroup is the group for the resource being referenced + APIGroup string `json:"apiGroup"` + // Kind is the type of resource being referenced + Kind string `json:"kind"` + // Name is the name of resource being referenced + Name string `json:"name"` + // API version of the referent + APIVersion string `json:"apiVersion,omitempty"` +} + +type LoadBalancerConfigType string + +const ( + // LoadBalancerConfigTypeHAProxy is the LoadBalancerConfigType for HAProxy. + LoadBalancerConfigTypeHAProxy LoadBalancerConfigType = "haproxy" + + // LoadBalancerConfigTypeAvi is the LoadBalancerConfigType for Avi. + LoadBalancerConfigTypeAvi LoadBalancerConfigType = "avi" + + // LoadBalancerConfigTypeFoundation is the FoundationLoadBalancerConfigType for VCF Foundation Load Balancer. + LoadBalancerConfigTypeFoundation LoadBalancerConfigType = "foundation" +) + +// LoadBalancerConfigSpec defines the desired state of LoadBalancerConfig +type LoadBalancerConfigSpec struct { + // Type describes type of load balancer. + // +kubebuilder:validation:Enum=haproxy;avi;foundation + Type LoadBalancerConfigType `json:"type"` + // ProviderRef is reference to a load balancer provider object that provides the details for this type of load balancer + ProviderRef LoadBalancerConfigProviderReference `json:"providerRef"` +} + +// LoadBalancerConfigStatus defines the observed state of LoadBalancerConfig +type LoadBalancerConfigStatus struct { + // Conditions is an array of current observed load balancer conditions + Conditions []LoadBalancerConfigCondition `json:"conditions,omitempty"` +} + +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster + +// LoadBalancerConfig is the Schema for the LoadBalancerConfigs API +type LoadBalancerConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LoadBalancerConfigSpec `json:"spec,omitempty"` + Status LoadBalancerConfigStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// LoadBalancerConfigList contains a list of LoadBalancerConfig +type LoadBalancerConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LoadBalancerConfig `json:"items"` +} + +func init() { + RegisterTypeWithScheme(&LoadBalancerConfig{}, &LoadBalancerConfigList{}) +} diff --git a/external/net-operator/api/v1alpha1/network_types.go b/external/net-operator/api/v1alpha1/network_types.go new file mode 100644 index 000000000..8efddb88c --- /dev/null +++ b/external/net-operator/api/v1alpha1/network_types.go @@ -0,0 +1,92 @@ +// Copyright (c) 2020-2024 Broadcom. All Rights Reserved. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NetworkProviderReference contains info to locate a network provider object. +type NetworkProviderReference struct { + // APIGroup is the group for the resource being referenced. + APIGroup string `json:"apiGroup"` + // Kind is the type of resource being referenced. + Kind string `json:"kind"` + // Name is the name of resource being referenced. + Name string `json:"name"` + // Namespace of the resource being referenced. If empty, cluster scoped resource is assumed. + Namespace string `json:"namespace,omitempty"` + // API version of the referent. + APIVersion string `json:"apiVersion,omitempty"` +} + +// NetworkType is used to type the constants describing possible network types. +type NetworkType string + +const ( + // NetworkTypeNSXT is the network type describing NSX-T. + NetworkTypeNSXT = NetworkType("nsx-t") + + // NetworkTypeVDS is the network type describing VSphere Distributed Switch. + NetworkTypeVDS = NetworkType("vsphere-distributed") + + // NetworkTypeNSXTVPC is the network type describing NSX-T VPC. + NetworkTypeNSXTVPC = NetworkType("nsx-t_vpc") +) + +// NetworkSpec defines the state of Network. +type NetworkSpec struct { + // Type describes type of Network. Supported values are nsx-t, vsphere-distributed. + Type NetworkType `json:"type"` + // ProviderRef is reference to a network provider object that provides this type of network. + ProviderRef NetworkProviderReference `json:"providerRef"` + // DNS is a list of DNS server IPs to associate with network interfaces on this network. + DNS []string `json:"dns,omitempty"` + // DNSSearchDomains is a list of DNS search domains to associate with network interfaces on this network. + DNSSearchDomains []string `json:"dnsSearchDomains,omitempty"` + // NTP is a list of NTP server DNS names or IP addresses to use on this network. + NTP []string `json:"ntp,omitempty"` +} + +// NetworkStatus is unused. This is because Network is purely a configuration resource. +type NetworkStatus struct { +} + +// NetworkReference is an object that points to a Network. +type NetworkReference struct { + // Kind is the type of resource being referenced. + Kind string `json:"kind"` + // Name is the name of resource being referenced. + Name string `json:"name"` + // APIVersion of the referent. + // + // +optional + APIVersion string `json:"apiVersion,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true + +// Network is the Schema for the networks API. +// A Network describes type, class and common attributes of a network available +// in a namespace. A NetworkInterface resource references a Network. +type Network struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NetworkSpec `json:"spec,omitempty"` + Status NetworkStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NetworkList contains a list of Network +type NetworkList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Network `json:"items"` +} + +func init() { + RegisterTypeWithScheme(&Network{}, &NetworkList{}) +} diff --git a/external/net-operator/api/v1alpha1/networkinterface_types.go b/external/net-operator/api/v1alpha1/networkinterface_types.go new file mode 100644 index 000000000..2aa95d281 --- /dev/null +++ b/external/net-operator/api/v1alpha1/networkinterface_types.go @@ -0,0 +1,180 @@ +// Copyright (c) 2020-2024 Broadcom. All Rights Reserved. + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // NetworkInterfaceFinalizer allows the Controller to clean up resources associated + // with a NetworkInterface before removing it from the API Server. + NetworkInterfaceFinalizer = "networkinterface.netoperator.vmware.com" + + // NetworkInterfaceClientManagedAnnotation annotations means the NetworkInterface is + // client managed and the Controller will not reconcile it. The value does not need + // to be truthy; the presence of the key is what disables reconciliation. + NetworkInterfaceClientManagedAnnotation = "networkinterface.netoperator.vmware.com/client-managed" +) + +// IPConfig represents an IP configuration. +type IPConfig struct { + // IP setting. + IP string `json:"ip"` + // IPFamily specifies the IP family (IPv4 vs IPv6) the IP belongs to. + IPFamily corev1.IPFamily `json:"ipFamily"` + // Gateway setting. + Gateway string `json:"gateway"` + // SubnetMask setting. + SubnetMask string `json:"subnetMask"` +} + +// NetworkInterfaceProviderReference contains info to locate a network interface provider object. +type NetworkInterfaceProviderReference struct { + // APIGroup is the group for the resource being referenced. + APIGroup string `json:"apiGroup"` + // Kind is the type of resource being referenced + Kind string `json:"kind"` + // Name is the name of resource being referenced + Name string `json:"name"` + // API version of the referent. + APIVersion string `json:"apiVersion,omitempty"` +} + +type NetworkInterfaceConditionType string + +const ( + // NetworkInterfaceReady is added when all network settings have been updated and the network + // interface is ready to be used. + NetworkInterfaceReady NetworkInterfaceConditionType = "Ready" + // NetworkInterfaceFailure is added when network provider plugin returns an error. + NetworkInterfaceFailure NetworkInterfaceConditionType = "Failure" +) + +type NetworkInterfaceConditionReason string + +const ( + // NetworkInterfaceFailureReasonCannotAllocIP indicates NetworkInterface is in failed state because an + // IPConfig cannot be allocated. + NetworkInterfaceFailureReasonCannotAllocIP NetworkInterfaceConditionReason = "CannotAllocIP" + // NetworkInterfaceFailureReasonCannotAllocPort indicates NetworkInterface is in failed state because + // port cannot be allocated for network interface on the network. + NetworkInterfaceFailureReasonCannotAllocPort NetworkInterfaceConditionReason = "CannotAllocPort" +) + +// NetworkInterfaceCondition describes the state of a NetworkInterface at a certain point. +type NetworkInterfaceCondition struct { + // Type is the type of network interface condition. + Type NetworkInterfaceConditionType `json:"type"` + // Status is the status of the condition. + // Can be True, False, Unknown. + Status corev1.ConditionStatus `json:"status"` + // LastTransitionTime is the timestamp corresponding to the last status + // change of this condition. + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + // Machine understandable string that gives the reason for condition's last transition. + Reason NetworkInterfaceConditionReason `json:"reason,omitempty"` + // Human-readable message indicating details about last transition. + Message string `json:"message,omitempty"` +} + +// NetworkInterfaceStatus defines the observed state of NetworkInterface. +// Once NetworkInterfaceReady condition is True, it should contain configuration to use to place +// a VM/Pod/Container's nic on the specified network. +type NetworkInterfaceStatus struct { + // Conditions is an array of current observed network interface conditions. + Conditions []NetworkInterfaceCondition `json:"conditions,omitempty"` + // IPConfigs is an array of IP configurations for the network interface. + IPConfigs []IPConfig `json:"ipConfigs,omitempty"` + // MacAddress setting for the network interface. + MacAddress string `json:"macAddress,omitempty"` + // ExternalID is a network provider specific identifier assigned to the network interface. + ExternalID string `json:"externalID,omitempty"` + // NetworkID is an network provider specific identifier for the network backing the network + // interface. + NetworkID string `json:"networkID,omitempty"` + // PortID is a network provider specific port identifier allocated for this network interface on + // the backing network. It is only valid on requested node and is set only if port allocation + // was requested. + PortID string `json:"portID,omitempty"` + // ConnectionID is a network provider specific port connection identifier allocated for this + // network interface on the backing network. It is only valid on requested node and is set + // only if port allocation was requested. + ConnectionID string `json:"connectionID,omitempty"` +} + +type NetworkInterfaceType string + +const ( + // NetworkInterfaceTypeVMXNet3 is for a VMXNET3 device. + NetworkInterfaceTypeVMXNet3 = NetworkInterfaceType("vmxnet3") +) + +// NetworkInterfacePortAllocation describes the settings for network interface port allocation request. +type NetworkInterfacePortAllocation struct { + // NodeName is the node where port must be allocated for this network interface. + NodeName string `json:"nodeName"` +} + +// NetworkInterfaceSpec defines the desired state of NetworkInterface. +type NetworkInterfaceSpec struct { + // NetworkName refers to a NetworkObject in the same namespace. + NetworkName string `json:"networkName,omitempty"` + // Type is the type of NetworkInterface. Supported values are vmxnet3. + Type NetworkInterfaceType `json:"type,omitempty"` + // ProviderRef is a reference to a provider specific network interface object + // that specifies the network interface configuration. + // If unset, default configuration is assumed. + ProviderRef *NetworkInterfaceProviderReference `json:"providerRef,omitempty"` + // PortAllocation is a request to allocate a port for this network interface on the backing network. + // This feature is currently supported only if backing network type is NetworkTypeVDS. In all other + // cases this field is ignored. Typically this is done implicitly by vCenter Server at the time + // of attaching a network interface to a network and should be left unset. This is used primarily when + // attachment of network interface to the network is done without vCenter Server's knowledge. + PortAllocation *NetworkInterfacePortAllocation `json:"portAllocation,omitempty"` + // ExternalID describes a value that will be surfaced as status.externalID. + // If this field is omitted, then it is up to the underlying network + // provider to surface any information in status.externalID. + // +optional + ExternalID string `json:"externalID,omitempty"` +} + +// NetworkInterfaceReference is an object that points to a NetworkInterface. +type NetworkInterfaceReference struct { + // Kind is the type of resource being referenced. + Kind string `json:"kind"` + // Name is the name of resource being referenced. + Name string `json:"name"` + // APIVersion of the referent. + // + // +optional + APIVersion string `json:"apiVersion,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true + +// NetworkInterface is the Schema for the networkinterfaces API. +// A NetworkInterface represents a user's request for network configuration to use to place a +// VM/Pod/Container's nic on a specified network. +type NetworkInterface struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NetworkInterfaceSpec `json:"spec,omitempty"` + Status NetworkInterfaceStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NetworkInterfaceList contains a list of NetworkInterface +type NetworkInterfaceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetworkInterface `json:"items"` +} + +func init() { + RegisterTypeWithScheme(&NetworkInterface{}, &NetworkInterfaceList{}) +} diff --git a/external/net-operator/api/v1alpha1/vmxnet3networkinterface_types.go b/external/net-operator/api/v1alpha1/vmxnet3networkinterface_types.go new file mode 100644 index 000000000..c8d61eeba --- /dev/null +++ b/external/net-operator/api/v1alpha1/vmxnet3networkinterface_types.go @@ -0,0 +1,48 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VMXNET3NetworkInterfaceSpec defines the desired state of VMXNET3NetworkInterface. +type VMXNET3NetworkInterfaceSpec struct { + // UPTCompatibilityEnabled indicates whether UPT(Universal Pass-through) compatibility is enabled + // on this network interface. + UPTCompatibilityEnabled bool `json:"uptCompatibilityEnabled,omitempty"` + // WakeOnLanEnabled indicates whether wake-on-LAN is enabled on this network interface. Clients + // can set this property to selectively enable or disable wake-on-LAN. + WakeOnLanEnabled bool `json:"wakeOnLanEnabled,omitempty"` +} + +// VMXNET3NetworkInterfaceStatus is unused. VMXNET3NetworkInterface is a configuration only resource. +type VMXNET3NetworkInterfaceStatus struct { +} + +// +genclient +// +kubebuilder:object:root=true + +// VMXNET3NetworkInterface is the Schema for the vmxnet3networkinterfaces API. +// It represents configuration of a vSphere VMXNET3 type network interface card. +type VMXNET3NetworkInterface struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec VMXNET3NetworkInterfaceSpec `json:"spec,omitempty"` + Status VMXNET3NetworkInterfaceStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// VMXNET3NetworkInterfaceList contains a list of VMXNET3NetworkInterface +type VMXNET3NetworkInterfaceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []VMXNET3NetworkInterface `json:"items"` +} + +func init() { + RegisterTypeWithScheme(&VMXNET3NetworkInterface{}, &VMXNET3NetworkInterfaceList{}) +} diff --git a/external/net-operator/api/v1alpha1/vspheredistributednetwork_types.go b/external/net-operator/api/v1alpha1/vspheredistributednetwork_types.go new file mode 100644 index 000000000..38135ae47 --- /dev/null +++ b/external/net-operator/api/v1alpha1/vspheredistributednetwork_types.go @@ -0,0 +1,101 @@ +// Copyright (c) 2020-2024 Broadcom. All Rights Reserved. + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type VSphereDistributedNetworkConditionType string + +const ( + // VSphereDistributedNetworkPortGroupFailure is added when PortGroupID specified either doesn't exist, or + // there was an error in communicating with vCenter Server. + VSphereDistributedNetworkPortGroupFailure VSphereDistributedNetworkConditionType = "PortGroupFailure" + // VSphereDistributedNetworkIPPoolInvalid is added when no valid IPPool references exists. + VSphereDistributedNetworkIPPoolInvalid VSphereDistributedNetworkConditionType = "IPPoolInvalid" + // VsphereDistributedNetworkIPPoolPressure condition status is set to True when IPPool is low on free IPs. + VsphereDistributedNetworkIPPoolPressure VSphereDistributedNetworkConditionType = "IPPoolPressure" +) + +type IPAssignmentModeType string + +const ( + // IPAssignmentModeDHCP indicates IP address is assigned dynamically using DHCP. + IPAssignmentModeDHCP IPAssignmentModeType = "dhcp" + // IPAssignmentModeStaticPool indicates IP address is assigned from a static pool of IP addresses. + IPAssignmentModeStaticPool IPAssignmentModeType = "staticpool" +) + +// VSphereDistributedNetworkCondition describes the state of a VSphereDistributedNetwork at a certain point. +type VSphereDistributedNetworkCondition struct { + // Type is the type of VSphereDistributedNetwork condition. + Type VSphereDistributedNetworkConditionType `json:"type"` + // Status is the status of the condition. + // Can be True, False, Unknown. + Status corev1.ConditionStatus `json:"status"` + // Machine understandable string that gives the reason for condition's last transition. + Reason string `json:"reason,omitempty"` + // Human-readable message indicating details about last transition. + Message string `json:"message,omitempty"` + // Provides a timestamp for when the VSphereDistributedNetwork object last transitioned from one status to another. + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty" patchStrategy:"replace"` +} + +// VSphereDistributedNetworkSpec defines the desired state of VSphereDistributedNetwork. +type VSphereDistributedNetworkSpec struct { + // PortGroupID is an existing vSphere Distributed PortGroup identifier. + PortGroupID string `json:"portGroupID"` + + // IPAssignmentMode to use for network interfaces. If unset, defaults to IPAssignmentModeStaticPool. + // In case of IPAssignmentModeDHCP, IPPools, Gateway and SubnetMask fields are ignored. + // +optional + IPAssignmentMode IPAssignmentModeType `json:"ipAssignmentMode,omitempty"` + + // IPPools references list of IPPool objects. This field should be set to empty list for + // IPAssignmentModeDHCP IPAssignmentMode. + IPPools []IPPoolReference `json:"ipPools"` + + // Gateway setting to use for network interfaces. This field should be set to empty string + // for IPAssignmentModeDHCP IPAssignmentMode. + Gateway string `json:"gateway"` + + // SubnetMask setting to use for network interfaces. This field should be set to empty string + // for IPAssignmentModeDHCP IPAssignmentMode. + SubnetMask string `json:"subnetMask"` +} + +// VSphereDistributedNetworkStatus defines the observed state of VSphereDistributedNetwork. +type VSphereDistributedNetworkStatus struct { + // Conditions is an array of current observed vSphere Distributed network conditions. + Conditions []VSphereDistributedNetworkCondition `json:"conditions,omitempty"` +} + +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster + +// VSphereDistributedNetwork represents schema for a network backed by a vSphere Distributed PortGroup on vSphere +// Distributed switch. +type VSphereDistributedNetwork struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec VSphereDistributedNetworkSpec `json:"spec,omitempty"` + Status VSphereDistributedNetworkStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// VSphereDistributedNetworkList contains a list of VSphereDistributedNetwork +type VSphereDistributedNetworkList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []VSphereDistributedNetwork `json:"items"` +} + +func init() { + RegisterTypeWithScheme(&VSphereDistributedNetwork{}, &VSphereDistributedNetworkList{}) +} diff --git a/external/net-operator/api/v1alpha1/zz_generated.deepcopy.go b/external/net-operator/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..2aa218d52 --- /dev/null +++ b/external/net-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,1364 @@ +//go:build !ignore_autogenerated + +// Copyright (c) 2020-2024 Broadcom. All Rights Reserved. + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ActivePassiveAvailabilityMode) DeepCopyInto(out *ActivePassiveAvailabilityMode) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ActivePassiveAvailabilityMode. +func (in *ActivePassiveAvailabilityMode) DeepCopy() *ActivePassiveAvailabilityMode { + if in == nil { + return nil + } + out := new(ActivePassiveAvailabilityMode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AviLoadBalancerConfig) DeepCopyInto(out *AviLoadBalancerConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AviLoadBalancerConfig. +func (in *AviLoadBalancerConfig) DeepCopy() *AviLoadBalancerConfig { + if in == nil { + return nil + } + out := new(AviLoadBalancerConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AviLoadBalancerConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AviLoadBalancerConfigList) DeepCopyInto(out *AviLoadBalancerConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AviLoadBalancerConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AviLoadBalancerConfigList. +func (in *AviLoadBalancerConfigList) DeepCopy() *AviLoadBalancerConfigList { + if in == nil { + return nil + } + out := new(AviLoadBalancerConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AviLoadBalancerConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AviLoadBalancerConfigSpec) DeepCopyInto(out *AviLoadBalancerConfigSpec) { + *out = *in + if in.AdvancedL4 != nil { + in, out := &in.AdvancedL4, &out.AdvancedL4 + *out = new(bool) + **out = **in + } + out.CredentialSecretRef = in.CredentialSecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AviLoadBalancerConfigSpec. +func (in *AviLoadBalancerConfigSpec) DeepCopy() *AviLoadBalancerConfigSpec { + if in == nil { + return nil + } + out := new(AviLoadBalancerConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AviLoadBalancerConfigStatus) DeepCopyInto(out *AviLoadBalancerConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AviLoadBalancerConfigStatus. +func (in *AviLoadBalancerConfigStatus) DeepCopy() *AviLoadBalancerConfigStatus { + if in == nil { + return nil + } + out := new(AviLoadBalancerConfigStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClientSecretReference) DeepCopyInto(out *ClientSecretReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientSecretReference. +func (in *ClientSecretReference) DeepCopy() *ClientSecretReference { + if in == nil { + return nil + } + out := new(ClientSecretReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FoundationLoadBalancerConfig) DeepCopyInto(out *FoundationLoadBalancerConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationLoadBalancerConfig. +func (in *FoundationLoadBalancerConfig) DeepCopy() *FoundationLoadBalancerConfig { + if in == nil { + return nil + } + out := new(FoundationLoadBalancerConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FoundationLoadBalancerConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FoundationLoadBalancerConfigList) DeepCopyInto(out *FoundationLoadBalancerConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FoundationLoadBalancerConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationLoadBalancerConfigList. +func (in *FoundationLoadBalancerConfigList) DeepCopy() *FoundationLoadBalancerConfigList { + if in == nil { + return nil + } + out := new(FoundationLoadBalancerConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FoundationLoadBalancerConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FoundationLoadBalancerConfigSpec) DeepCopyInto(out *FoundationLoadBalancerConfigSpec) { + *out = *in + in.DeploymentSpec.DeepCopyInto(&out.DeploymentSpec) + if in.ManagementNetwork != nil { + in, out := &in.ManagementNetwork, &out.ManagementNetwork + *out = new(NetworkReference) + **out = **in + } + if in.WorkloadNetworks != nil { + in, out := &in.WorkloadNetworks, &out.WorkloadNetworks + *out = make([]NetworkReference, len(*in)) + copy(*out, *in) + } + out.VirtualIPNetwork = in.VirtualIPNetwork + in.NetworkSpec.DeepCopyInto(&out.NetworkSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationLoadBalancerConfigSpec. +func (in *FoundationLoadBalancerConfigSpec) DeepCopy() *FoundationLoadBalancerConfigSpec { + if in == nil { + return nil + } + out := new(FoundationLoadBalancerConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FoundationLoadBalancerConfigStatus) DeepCopyInto(out *FoundationLoadBalancerConfigStatus) { + *out = *in + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]FoundationLoadBalancerNodeStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.VirtualServerIPPoolsUtilization = in.VirtualServerIPPoolsUtilization + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationLoadBalancerConfigStatus. +func (in *FoundationLoadBalancerConfigStatus) DeepCopy() *FoundationLoadBalancerConfigStatus { + if in == nil { + return nil + } + out := new(FoundationLoadBalancerConfigStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FoundationLoadBalancerDeploymentSpec) DeepCopyInto(out *FoundationLoadBalancerDeploymentSpec) { + *out = *in + if in.Zones != nil { + in, out := &in.Zones, &out.Zones + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ActivePassiveAvailabilityMode != nil { + in, out := &in.ActivePassiveAvailabilityMode, &out.ActivePassiveAvailabilityMode + *out = new(ActivePassiveAvailabilityMode) + **out = **in + } + if in.SingleNodeAvailabilityMode != nil { + in, out := &in.SingleNodeAvailabilityMode, &out.SingleNodeAvailabilityMode + *out = new(SingleNodeAvailabilityMode) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationLoadBalancerDeploymentSpec. +func (in *FoundationLoadBalancerDeploymentSpec) DeepCopy() *FoundationLoadBalancerDeploymentSpec { + if in == nil { + return nil + } + out := new(FoundationLoadBalancerDeploymentSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FoundationLoadBalancerNetworkConfigSpec) DeepCopyInto(out *FoundationLoadBalancerNetworkConfigSpec) { + *out = *in + if in.VirtualServerIPPools != nil { + in, out := &in.VirtualServerIPPools, &out.VirtualServerIPPools + *out = make([]IPPoolReference, len(*in)) + copy(*out, *in) + } + if in.VirtualServerSubnets != nil { + in, out := &in.VirtualServerSubnets, &out.VirtualServerSubnets + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DNSServers != nil { + in, out := &in.DNSServers, &out.DNSServers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DNSSearchDomains != nil { + in, out := &in.DNSSearchDomains, &out.DNSSearchDomains + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.NTPServers != nil { + in, out := &in.NTPServers, &out.NTPServers + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationLoadBalancerNetworkConfigSpec. +func (in *FoundationLoadBalancerNetworkConfigSpec) DeepCopy() *FoundationLoadBalancerNetworkConfigSpec { + if in == nil { + return nil + } + out := new(FoundationLoadBalancerNetworkConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FoundationLoadBalancerNodeStatus) DeepCopyInto(out *FoundationLoadBalancerNodeStatus) { + *out = *in + out.ManagementNetworkInterface = in.ManagementNetworkInterface + if in.WorkloadNetworkInterfaces != nil { + in, out := &in.WorkloadNetworkInterfaces, &out.WorkloadNetworkInterfaces + *out = make([]NetworkInterfaceReference, len(*in)) + copy(*out, *in) + } + out.VIPNetworkInterface = in.VIPNetworkInterface +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationLoadBalancerNodeStatus. +func (in *FoundationLoadBalancerNodeStatus) DeepCopy() *FoundationLoadBalancerNodeStatus { + if in == nil { + return nil + } + out := new(FoundationLoadBalancerNodeStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAProxyLoadBalancerConfig) DeepCopyInto(out *HAProxyLoadBalancerConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAProxyLoadBalancerConfig. +func (in *HAProxyLoadBalancerConfig) DeepCopy() *HAProxyLoadBalancerConfig { + if in == nil { + return nil + } + out := new(HAProxyLoadBalancerConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *HAProxyLoadBalancerConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAProxyLoadBalancerConfigList) DeepCopyInto(out *HAProxyLoadBalancerConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]HAProxyLoadBalancerConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAProxyLoadBalancerConfigList. +func (in *HAProxyLoadBalancerConfigList) DeepCopy() *HAProxyLoadBalancerConfigList { + if in == nil { + return nil + } + out := new(HAProxyLoadBalancerConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *HAProxyLoadBalancerConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAProxyLoadBalancerConfigSpec) DeepCopyInto(out *HAProxyLoadBalancerConfigSpec) { + *out = *in + if in.EndPointURLs != nil { + in, out := &in.EndPointURLs, &out.EndPointURLs + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.CredentialSecretRef = in.CredentialSecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAProxyLoadBalancerConfigSpec. +func (in *HAProxyLoadBalancerConfigSpec) DeepCopy() *HAProxyLoadBalancerConfigSpec { + if in == nil { + return nil + } + out := new(HAProxyLoadBalancerConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAProxyLoadBalancerConfigStatus) DeepCopyInto(out *HAProxyLoadBalancerConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAProxyLoadBalancerConfigStatus. +func (in *HAProxyLoadBalancerConfigStatus) DeepCopy() *HAProxyLoadBalancerConfigStatus { + if in == nil { + return nil + } + out := new(HAProxyLoadBalancerConfigStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocation) DeepCopyInto(out *IPAddressAllocation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocation. +func (in *IPAddressAllocation) DeepCopy() *IPAddressAllocation { + if in == nil { + return nil + } + out := new(IPAddressAllocation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPAddressAllocation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationCondition) DeepCopyInto(out *IPAddressAllocationCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationCondition. +func (in *IPAddressAllocationCondition) DeepCopy() *IPAddressAllocationCondition { + if in == nil { + return nil + } + out := new(IPAddressAllocationCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationList) DeepCopyInto(out *IPAddressAllocationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]IPAddressAllocation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationList. +func (in *IPAddressAllocationList) DeepCopy() *IPAddressAllocationList { + if in == nil { + return nil + } + out := new(IPAddressAllocationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPAddressAllocationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationSpec) DeepCopyInto(out *IPAddressAllocationSpec) { + *out = *in + in.PoolRef.DeepCopyInto(&out.PoolRef) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationSpec. +func (in *IPAddressAllocationSpec) DeepCopy() *IPAddressAllocationSpec { + if in == nil { + return nil + } + out := new(IPAddressAllocationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationStatus) DeepCopyInto(out *IPAddressAllocationStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]IPAddressAllocationCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationStatus. +func (in *IPAddressAllocationStatus) DeepCopy() *IPAddressAllocationStatus { + if in == nil { + return nil + } + out := new(IPAddressAllocationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPConfig) DeepCopyInto(out *IPConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPConfig. +func (in *IPConfig) DeepCopy() *IPConfig { + if in == nil { + return nil + } + out := new(IPConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPool) DeepCopyInto(out *IPPool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPool. +func (in *IPPool) DeepCopy() *IPPool { + if in == nil { + return nil + } + out := new(IPPool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPPool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolCondition) DeepCopyInto(out *IPPoolCondition) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolCondition. +func (in *IPPoolCondition) DeepCopy() *IPPoolCondition { + if in == nil { + return nil + } + out := new(IPPoolCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolList) DeepCopyInto(out *IPPoolList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]IPPool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolList. +func (in *IPPoolList) DeepCopy() *IPPoolList { + if in == nil { + return nil + } + out := new(IPPoolList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPPoolList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolReference) DeepCopyInto(out *IPPoolReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolReference. +func (in *IPPoolReference) DeepCopy() *IPPoolReference { + if in == nil { + return nil + } + out := new(IPPoolReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolSpec) DeepCopyInto(out *IPPoolSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolSpec. +func (in *IPPoolSpec) DeepCopy() *IPPoolSpec { + if in == nil { + return nil + } + out := new(IPPoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolStatus) DeepCopyInto(out *IPPoolStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]IPPoolCondition, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolStatus. +func (in *IPPoolStatus) DeepCopy() *IPPoolStatus { + if in == nil { + return nil + } + out := new(IPPoolStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfig) DeepCopyInto(out *LoadBalancerConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfig. +func (in *LoadBalancerConfig) DeepCopy() *LoadBalancerConfig { + if in == nil { + return nil + } + out := new(LoadBalancerConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LoadBalancerConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigCondition) DeepCopyInto(out *LoadBalancerConfigCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigCondition. +func (in *LoadBalancerConfigCondition) DeepCopy() *LoadBalancerConfigCondition { + if in == nil { + return nil + } + out := new(LoadBalancerConfigCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigList) DeepCopyInto(out *LoadBalancerConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LoadBalancerConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigList. +func (in *LoadBalancerConfigList) DeepCopy() *LoadBalancerConfigList { + if in == nil { + return nil + } + out := new(LoadBalancerConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LoadBalancerConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigProviderReference) DeepCopyInto(out *LoadBalancerConfigProviderReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigProviderReference. +func (in *LoadBalancerConfigProviderReference) DeepCopy() *LoadBalancerConfigProviderReference { + if in == nil { + return nil + } + out := new(LoadBalancerConfigProviderReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigSpec) DeepCopyInto(out *LoadBalancerConfigSpec) { + *out = *in + out.ProviderRef = in.ProviderRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigSpec. +func (in *LoadBalancerConfigSpec) DeepCopy() *LoadBalancerConfigSpec { + if in == nil { + return nil + } + out := new(LoadBalancerConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfigStatus) DeepCopyInto(out *LoadBalancerConfigStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]LoadBalancerConfigCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfigStatus. +func (in *LoadBalancerConfigStatus) DeepCopy() *LoadBalancerConfigStatus { + if in == nil { + return nil + } + out := new(LoadBalancerConfigStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Network) DeepCopyInto(out *Network) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Network. +func (in *Network) DeepCopy() *Network { + if in == nil { + return nil + } + out := new(Network) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Network) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterface) DeepCopyInto(out *NetworkInterface) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterface. +func (in *NetworkInterface) DeepCopy() *NetworkInterface { + if in == nil { + return nil + } + out := new(NetworkInterface) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInterface) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceCondition) DeepCopyInto(out *NetworkInterfaceCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceCondition. +func (in *NetworkInterfaceCondition) DeepCopy() *NetworkInterfaceCondition { + if in == nil { + return nil + } + out := new(NetworkInterfaceCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceList) DeepCopyInto(out *NetworkInterfaceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetworkInterface, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceList. +func (in *NetworkInterfaceList) DeepCopy() *NetworkInterfaceList { + if in == nil { + return nil + } + out := new(NetworkInterfaceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInterfaceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfacePortAllocation) DeepCopyInto(out *NetworkInterfacePortAllocation) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfacePortAllocation. +func (in *NetworkInterfacePortAllocation) DeepCopy() *NetworkInterfacePortAllocation { + if in == nil { + return nil + } + out := new(NetworkInterfacePortAllocation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceProviderReference) DeepCopyInto(out *NetworkInterfaceProviderReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceProviderReference. +func (in *NetworkInterfaceProviderReference) DeepCopy() *NetworkInterfaceProviderReference { + if in == nil { + return nil + } + out := new(NetworkInterfaceProviderReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceReference) DeepCopyInto(out *NetworkInterfaceReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceReference. +func (in *NetworkInterfaceReference) DeepCopy() *NetworkInterfaceReference { + if in == nil { + return nil + } + out := new(NetworkInterfaceReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceSpec) DeepCopyInto(out *NetworkInterfaceSpec) { + *out = *in + if in.ProviderRef != nil { + in, out := &in.ProviderRef, &out.ProviderRef + *out = new(NetworkInterfaceProviderReference) + **out = **in + } + if in.PortAllocation != nil { + in, out := &in.PortAllocation, &out.PortAllocation + *out = new(NetworkInterfacePortAllocation) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceSpec. +func (in *NetworkInterfaceSpec) DeepCopy() *NetworkInterfaceSpec { + if in == nil { + return nil + } + out := new(NetworkInterfaceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceStatus) DeepCopyInto(out *NetworkInterfaceStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]NetworkInterfaceCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.IPConfigs != nil { + in, out := &in.IPConfigs, &out.IPConfigs + *out = make([]IPConfig, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceStatus. +func (in *NetworkInterfaceStatus) DeepCopy() *NetworkInterfaceStatus { + if in == nil { + return nil + } + out := new(NetworkInterfaceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkList) DeepCopyInto(out *NetworkList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Network, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkList. +func (in *NetworkList) DeepCopy() *NetworkList { + if in == nil { + return nil + } + out := new(NetworkList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkProviderReference) DeepCopyInto(out *NetworkProviderReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkProviderReference. +func (in *NetworkProviderReference) DeepCopy() *NetworkProviderReference { + if in == nil { + return nil + } + out := new(NetworkProviderReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkReference) DeepCopyInto(out *NetworkReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkReference. +func (in *NetworkReference) DeepCopy() *NetworkReference { + if in == nil { + return nil + } + out := new(NetworkReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { + *out = *in + out.ProviderRef = in.ProviderRef + if in.DNS != nil { + in, out := &in.DNS, &out.DNS + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DNSSearchDomains != nil { + in, out := &in.DNSSearchDomains, &out.DNSSearchDomains + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.NTP != nil { + in, out := &in.NTP, &out.NTP + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkSpec. +func (in *NetworkSpec) DeepCopy() *NetworkSpec { + if in == nil { + return nil + } + out := new(NetworkSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkStatus) DeepCopyInto(out *NetworkStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkStatus. +func (in *NetworkStatus) DeepCopy() *NetworkStatus { + if in == nil { + return nil + } + out := new(NetworkStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SingleNodeAvailabilityMode) DeepCopyInto(out *SingleNodeAvailabilityMode) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SingleNodeAvailabilityMode. +func (in *SingleNodeAvailabilityMode) DeepCopy() *SingleNodeAvailabilityMode { + if in == nil { + return nil + } + out := new(SingleNodeAvailabilityMode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMXNET3NetworkInterface) DeepCopyInto(out *VMXNET3NetworkInterface) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMXNET3NetworkInterface. +func (in *VMXNET3NetworkInterface) DeepCopy() *VMXNET3NetworkInterface { + if in == nil { + return nil + } + out := new(VMXNET3NetworkInterface) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VMXNET3NetworkInterface) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMXNET3NetworkInterfaceList) DeepCopyInto(out *VMXNET3NetworkInterfaceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VMXNET3NetworkInterface, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMXNET3NetworkInterfaceList. +func (in *VMXNET3NetworkInterfaceList) DeepCopy() *VMXNET3NetworkInterfaceList { + if in == nil { + return nil + } + out := new(VMXNET3NetworkInterfaceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VMXNET3NetworkInterfaceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMXNET3NetworkInterfaceSpec) DeepCopyInto(out *VMXNET3NetworkInterfaceSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMXNET3NetworkInterfaceSpec. +func (in *VMXNET3NetworkInterfaceSpec) DeepCopy() *VMXNET3NetworkInterfaceSpec { + if in == nil { + return nil + } + out := new(VMXNET3NetworkInterfaceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMXNET3NetworkInterfaceStatus) DeepCopyInto(out *VMXNET3NetworkInterfaceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMXNET3NetworkInterfaceStatus. +func (in *VMXNET3NetworkInterfaceStatus) DeepCopy() *VMXNET3NetworkInterfaceStatus { + if in == nil { + return nil + } + out := new(VMXNET3NetworkInterfaceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VSphereDistributedNetwork) DeepCopyInto(out *VSphereDistributedNetwork) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereDistributedNetwork. +func (in *VSphereDistributedNetwork) DeepCopy() *VSphereDistributedNetwork { + if in == nil { + return nil + } + out := new(VSphereDistributedNetwork) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VSphereDistributedNetwork) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VSphereDistributedNetworkCondition) DeepCopyInto(out *VSphereDistributedNetworkCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereDistributedNetworkCondition. +func (in *VSphereDistributedNetworkCondition) DeepCopy() *VSphereDistributedNetworkCondition { + if in == nil { + return nil + } + out := new(VSphereDistributedNetworkCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VSphereDistributedNetworkList) DeepCopyInto(out *VSphereDistributedNetworkList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VSphereDistributedNetwork, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereDistributedNetworkList. +func (in *VSphereDistributedNetworkList) DeepCopy() *VSphereDistributedNetworkList { + if in == nil { + return nil + } + out := new(VSphereDistributedNetworkList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VSphereDistributedNetworkList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VSphereDistributedNetworkSpec) DeepCopyInto(out *VSphereDistributedNetworkSpec) { + *out = *in + if in.IPPools != nil { + in, out := &in.IPPools, &out.IPPools + *out = make([]IPPoolReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereDistributedNetworkSpec. +func (in *VSphereDistributedNetworkSpec) DeepCopy() *VSphereDistributedNetworkSpec { + if in == nil { + return nil + } + out := new(VSphereDistributedNetworkSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VSphereDistributedNetworkStatus) DeepCopyInto(out *VSphereDistributedNetworkStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]VSphereDistributedNetworkCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereDistributedNetworkStatus. +func (in *VSphereDistributedNetworkStatus) DeepCopy() *VSphereDistributedNetworkStatus { + if in == nil { + return nil + } + out := new(VSphereDistributedNetworkStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualIPPoolsUtilization) DeepCopyInto(out *VirtualIPPoolsUtilization) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualIPPoolsUtilization. +func (in *VirtualIPPoolsUtilization) DeepCopy() *VirtualIPPoolsUtilization { + if in == nil { + return nil + } + out := new(VirtualIPPoolsUtilization) + in.DeepCopyInto(out) + return out +} diff --git a/external/net-operator/go.mod b/external/net-operator/go.mod new file mode 100644 index 000000000..ec90833e1 --- /dev/null +++ b/external/net-operator/go.mod @@ -0,0 +1,28 @@ +module github.com/vmware-tanzu/vm-operator/external/net-operator + +go 1.26.1 + +require ( + k8s.io/api v0.35.3 + k8s.io/apimachinery v0.35.3 +) + +require ( + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/text v0.31.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect +) diff --git a/external/net-operator/go.sum b/external/net-operator/go.sum new file mode 100644 index 000000000..4dbbd404d --- /dev/null +++ b/external/net-operator/go.sum @@ -0,0 +1,66 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ= +k8s.io/api v0.35.3/go.mod h1:9Y9tkBcFwKNq2sxwZTQh1Njh9qHl81D0As56tu42GA4= +k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8= +k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/external/nsx-operator/api/vpc/v1alpha1/addressbinding_types.go b/external/nsx-operator/api/vpc/v1alpha1/addressbinding_types.go new file mode 100644 index 000000000..1a6a70b95 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/addressbinding_types.go @@ -0,0 +1,47 @@ +package v1alpha1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +type AddressBindingSpec struct { + // VMName contains the VM's name + VMName string `json:"vmName"` + // InterfaceName contains the interface name of the VM, if not set, the first interface of the VM will be used + InterfaceName string `json:"interfaceName,omitempty"` + // IPAddressAllocationName contains name of the external IPAddressAllocation. + // IP address will be allocated from an external IPBlock of the VPC when this field is not set. + IPAddressAllocationName string `json:"ipAddressAllocationName,omitempty"` +} + +type AddressBindingStatus struct { + // Conditions describes current state of AddressBinding. + Conditions []Condition `json:"conditions,omitempty"` + // IP Address for port binding. + IPAddress string `json:"ipAddress"` +} + +// +genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:storageversion + +// AddressBinding is used to manage 1:1 NAT for a VM/NetworkInterface. +type AddressBinding struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AddressBindingSpec `json:"spec"` + Status AddressBindingStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// AddressBindingList contains a list of AddressBinding. +type AddressBindingList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AddressBinding `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AddressBinding{}, &AddressBindingList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/condition_types.go b/external/nsx-operator/api/vpc/v1alpha1/condition_types.go new file mode 100644 index 000000000..56f38be30 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/condition_types.go @@ -0,0 +1,34 @@ +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ConditionType string + +const ( + Ready ConditionType = "Ready" + GatewayConnectionReady ConditionType = "GatewayConnectionReady" + ServiceClusterReady ConditionType = "ServiceClusterReady" + AutoSnatEnabled ConditionType = "AutoSnatEnabled" + ExternalIPBlocksConfigured ConditionType = "ExternalIPBlocksConfigured" + DeleteFailure ConditionType = "DeletionFailed" +) + +// Condition defines condition of custom resource. +type Condition struct { + // Type defines condition type. + Type ConditionType `json:"type"` + // Status of the condition, one of True, False, Unknown. + Status corev1.ConditionStatus `json:"status"` + // Last time the condition transitioned from one status to another. + // This should be when the underlying condition changed. If that is not known, then using the time when + // the API field changed is acceptable. + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + // Reason shows a brief reason of condition. + Reason string `json:"reason,omitempty"` + // Message shows a human-readable message about condition. + Message string `json:"message,omitempty"` +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/doc.go b/external/nsx-operator/api/vpc/v1alpha1/doc.go new file mode 100644 index 000000000..f452fd25d --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/doc.go @@ -0,0 +1,4 @@ +// +k8s:deepcopy-gen=package +// +groupName=crd.nsx.vmware.com + +package v1alpha1 diff --git a/external/nsx-operator/api/vpc/v1alpha1/groupversion_info.go b/external/nsx-operator/api/vpc/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..c4d8528f4 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/groupversion_info.go @@ -0,0 +1,22 @@ +/* Copyright © 2021 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +// +kubebuilder:object:generate=true +// +groupName=crd.nsx.vmware.com +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "crd.nsx.vmware.com", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/external/nsx-operator/api/vpc/v1alpha1/ipaddressallocation_types.go b/external/nsx-operator/api/vpc/v1alpha1/ipaddressallocation_types.go new file mode 100644 index 000000000..facbdd79a --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/ipaddressallocation_types.go @@ -0,0 +1,71 @@ +/* Copyright © 2024 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package v1alpha1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +type IPAddressVisibility string + +var ( + IPAddressVisibilityExternal IPAddressVisibility = "External" + IPAddressVisibilityPrivate IPAddressVisibility = "Private" + IPAddressVisibilityPrivateTGW IPAddressVisibility = "PrivateTGW" +) + +// +genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:storageversion + +// IPAddressAllocation is the Schema for the IP allocation API. +// +kubebuilder:printcolumn:name="IPAddressBlockVisibility",type=string,JSONPath=`.spec.ipAddressBlockVisibility`,description="IPAddressBlockVisibility of IPAddressAllocation" +// +kubebuilder:printcolumn:name="AllocationIPs",type=string,JSONPath=`.status.allocationIPs`, description="AllocationIPs for the IPAddressAllocation" +type IPAddressAllocation struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + + Spec IPAddressAllocationSpec `json:"spec"` + Status IPAddressAllocationStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// IPAddressAllocationList contains a list of IPAddressAllocation. +type IPAddressAllocationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []IPAddressAllocation `json:"items"` +} + +// IPAddressAllocationSpec defines the desired state of IPAddressAllocation. +// +kubebuilder:validation:XValidation:rule="!has(self.allocationSize) || !has(self.allocationIPs)", message="Only one of allocationSize or allocationIPs can be specified" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.allocationSize) || has(self.allocationSize)", message="allocationSize is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.allocationIPs) || has(self.allocationIPs)", message="allocationIPs is required once set" +type IPAddressAllocationSpec struct { + // IPAddressBlockVisibility specifies the visibility of the IPBlocks to allocate IP addresses. Can be External, Private or PrivateTGW. + // +kubebuilder:validation:Enum=External;Private;PrivateTGW + // +kubebuilder:default=Private + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + IPAddressBlockVisibility IPAddressVisibility `json:"ipAddressBlockVisibility,omitempty"` + // AllocationSize specifies the size of allocationIPs to be allocated. + // It should be a power of 2. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + // +kubebuilder:validation:Minimum:=1 + AllocationSize int `json:"allocationSize,omitempty"` + // AllocationIPs specifies the Allocated IP addresses in CIDR or single IP Address format. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + AllocationIPs string `json:"allocationIPs,omitempty"` +} + +// IPAddressAllocationStatus defines the observed state of IPAddressAllocation. +type IPAddressAllocationStatus struct { + // AllocationIPs is the allocated IP addresses + AllocationIPs string `json:"allocationIPs"` + Conditions []Condition `json:"conditions,omitempty"` +} + +func init() { + SchemeBuilder.Register(&IPAddressAllocation{}, &IPAddressAllocationList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/ipblocksinfo_types.go b/external/nsx-operator/api/vpc/v1alpha1/ipblocksinfo_types.go new file mode 100644 index 000000000..1f9a76d91 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/ipblocksinfo_types.go @@ -0,0 +1,54 @@ +/* Copyright © 2024 Broadcom, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +genclient +// +genclient:nonNamespaced +//+kubebuilder:object:root=true +//+kubebuilder:resource:scope="Cluster",path=ipblocksinfos + +// IPBlocksInfo is the Schema for the ipblocksinfo API +type IPBlocksInfo struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // ExternalIPCIDRs is a list of CIDR strings. Each CIDR is a contiguous IP address + // spaces represented by network address and prefix length. The visibility of the + // IPBlocks is External. + ExternalIPCIDRs []string `json:"externalIPCIDRs,omitempty"` + // PrivateTGWIPCIDRs is a list of CIDR strings. Each CIDR is a contiguous IP address + // spaces represented by network address and prefix length. The visibility of the + // IPBlocks is Private Transit Gateway. Only IPBlocks in default project will be included. + PrivateTGWIPCIDRs []string `json:"privateTGWIPCIDRs,omitempty"` + // ExternalIPRanges is an array of contiguous IP address space represented by start and end IPs. + // The visibility of the IPBlocks is External. + ExternalIPRanges []IPPoolRange `json:"externalIPRanges,omitempty"` + // PrivateTGWIPRanges is an array of contiguous IP address space represented by start and end IPs. + // The visibility of the IPBlocks is Private Transit Gateway. + PrivateTGWIPRanges []IPPoolRange `json:"privateTGWIPRanges,omitempty"` +} + +//+kubebuilder:object:root=true + +// IPBlocksInfoList contains a list of IPBlocksInfo +type IPBlocksInfoList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []IPBlocksInfo `json:"items"` +} + +type IPPoolRange struct { + // The start IP Address of the IP Range. + Start string `json:"start"` + // The end IP Address of the IP Range. + End string `json:"end"` +} + +func init() { + SchemeBuilder.Register(&IPBlocksInfo{}, &IPBlocksInfoList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/networkinfo_types.go b/external/nsx-operator/api/vpc/v1alpha1/networkinfo_types.go new file mode 100644 index 000000000..a6aaf3b28 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/networkinfo_types.go @@ -0,0 +1,46 @@ +/* Copyright © 2024 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:storageversion + +// NetworkInfo is used to report the network information for a namespace. +// +kubebuilder:resource:path=networkinfos +type NetworkInfo struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + VPCs []VPCState `json:"vpcs"` +} + +// +kubebuilder:object:root=true + +// NetworkInfoList contains a list of NetworkInfo. +type NetworkInfoList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetworkInfo `json:"items"` +} + +// VPCState defines information for VPC. +type VPCState struct { + // VPC name. + Name string `json:"name"` + // Default SNAT IP for Private Subnets. + DefaultSNATIP string `json:"defaultSNATIP"` + // LoadBalancerIPAddresses (AVI SE Subnet CIDR or NSX LB SNAT IPs). + LoadBalancerIPAddresses string `json:"loadBalancerIPAddresses,omitempty"` + // Private CIDRs used for the VPC. + PrivateIPs []string `json:"privateIPs,omitempty"` +} + +func init() { + SchemeBuilder.Register(&NetworkInfo{}, &NetworkInfoList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/register.go b/external/nsx-operator/api/vpc/v1alpha1/register.go new file mode 100644 index 000000000..2dfd631eb --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/register.go @@ -0,0 +1,12 @@ +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// SchemeGroupVersion is group version used to register these objects. +var SchemeGroupVersion = GroupVersion + +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/securitypolicy_types.go b/external/nsx-operator/api/vpc/v1alpha1/securitypolicy_types.go new file mode 100644 index 000000000..f8234678c --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/securitypolicy_types.go @@ -0,0 +1,142 @@ +/* Copyright © 2021 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +// +kubebuilder:object:generate=true +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// RuleAction describes the action to be applied on traffic matching a rule. +type RuleAction string + +const ( + // RuleActionAllow describes that the traffic matching the rule must be allowed. + RuleActionAllow RuleAction = "Allow" + // RuleActionDrop describes that the traffic matching the rule must be dropped. + RuleActionDrop RuleAction = "Drop" + // RuleActionReject indicates that the traffic matching the rule must be rejected and the + // client will receive a response. + RuleActionReject RuleAction = "Reject" +) + +// RuleDirection specifies the direction of traffic. +type RuleDirection string + +const ( + // RuleDirectionIn specifies that the direction of traffic must be ingress, equivalent to "Ingress". + RuleDirectionIn RuleDirection = "In" + // RuleDirectionIngress specifies that the direction of traffic must be ingress, equivalent to "In". + RuleDirectionIngress RuleDirection = "Ingress" + // RuleDirectionOut specifies that the direction of traffic must be egress, equivalent to "Egress". + RuleDirectionOut RuleDirection = "Out" + // RuleDirectionEgress specifies that the direction of traffic must be egress, equivalent to "Out". + RuleDirectionEgress RuleDirection = "Egress" +) + +// SecurityPolicySpec defines the desired state of SecurityPolicy. +type SecurityPolicySpec struct { + // Priority defines the order of policy enforcement. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=1000 + Priority int `json:"priority,omitempty"` + // AppliedTo is a list of policy targets to apply rules. + // Policy level 'Applied To' will take precedence over rule level. + AppliedTo []SecurityPolicyTarget `json:"appliedTo,omitempty"` + // Rules is a list of policy rules. + Rules []SecurityPolicyRule `json:"rules,omitempty"` +} + +// SecurityPolicyRule defines a rule of SecurityPolicy. +type SecurityPolicyRule struct { + // Action specifies the action to be applied on the rule. + Action *RuleAction `json:"action"` + // AppliedTo is a list of rule targets. + // Policy level 'Applied To' will take precedence over rule level. + AppliedTo []SecurityPolicyTarget `json:"appliedTo,omitempty"` + // Direction is the direction of the rule, including 'In' or 'Ingress', 'Out' or 'Egress'. + Direction *RuleDirection `json:"direction"` + // Sources defines the endpoints where the traffic is from. For ingress rule only. + Sources []SecurityPolicyPeer `json:"sources,omitempty"` + // Destinations defines the endpoints where the traffic is to. For egress rule only. + Destinations []SecurityPolicyPeer `json:"destinations,omitempty"` + // Ports is a list of ports to be matched. + Ports []SecurityPolicyPort `json:"ports,omitempty"` + // Name is the display name of this rule. + Name string `json:"name,omitempty"` +} + +// SecurityPolicyTarget defines the target endpoints to apply SecurityPolicy. +type SecurityPolicyTarget struct { + // VMSelector uses label selector to select VMs. + VMSelector *metav1.LabelSelector `json:"vmSelector,omitempty"` + // PodSelector uses label selector to select Pods. + PodSelector *metav1.LabelSelector `json:"podSelector,omitempty"` +} + +// SecurityPolicyPeer defines the source or destination of traffic. +type SecurityPolicyPeer struct { + // VMSelector uses label selector to select VMs. + VMSelector *metav1.LabelSelector `json:"vmSelector,omitempty"` + // PodSelector uses label selector to select Pods. + PodSelector *metav1.LabelSelector `json:"podSelector,omitempty"` + // NamespaceSelector uses label selector to select Namespaces. + NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` + // IPBlocks is a list of IP CIDRs. + IPBlocks []IPBlock `json:"ipBlocks,omitempty"` +} + +// IPBlock describes a particular CIDR that is allowed or denied to/from the workloads matched by an AppliedTo. +type IPBlock struct { + // CIDR is a string representing the IP Block. + // A valid example is "192.168.1.1/24". + CIDR string `json:"cidr"` +} + +// SecurityPolicyPort describes protocol and ports for traffic. +type SecurityPolicyPort struct { + // Protocol(TCP, UDP) is the protocol to match traffic. + // It is TCP by default. + // +kubebuilder:default=TCP + Protocol corev1.Protocol `json:"protocol,omitempty"` + // Port is the name or port number. + Port intstr.IntOrString `json:"port,omitempty"` + // EndPort defines the end of port range. + EndPort int `json:"endPort,omitempty"` +} + +// SecurityPolicyStatus defines the observed state of SecurityPolicy. +type SecurityPolicyStatus struct { + // Conditions describes current state of security policy. + Conditions []Condition `json:"conditions"` +} + +// +genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:storageversion + +// SecurityPolicy is the Schema for the securitypolicies API. +type SecurityPolicy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SecurityPolicySpec `json:"spec"` + Status SecurityPolicyStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// SecurityPolicyList contains a list of SecurityPolicy. +type SecurityPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SecurityPolicy `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SecurityPolicy{}, &SecurityPolicyList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/staticroute_types.go b/external/nsx-operator/api/vpc/v1alpha1/staticroute_types.go new file mode 100644 index 000000000..40b02cbd0 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/staticroute_types.go @@ -0,0 +1,64 @@ +/* Copyright © 2022-2023 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type StaticRouteStatusCondition string + +// StaticRouteCondition defines condition of StaticRoute. +type StaticRouteCondition Condition + +// StaticRouteSpec defines static routes configuration on VPC. +type StaticRouteSpec struct { + // Specify network address in CIDR format. + // +kubebuilder:validation:Format=cidr + Network string `json:"network"` + // Next hop gateway + // +kubebuilder:validation:MinItems=1 + NextHops []NextHop `json:"nextHops"` +} + +// NextHop defines next hop configuration for network. +type NextHop struct { + // Next hop gateway IP address. + // +kubebuilder:validation:Format=ip + IPAddress string `json:"ipAddress"` +} + +// StaticRouteStatus defines the observed state of StaticRoute. +type StaticRouteStatus struct { + Conditions []StaticRouteCondition `json:"conditions"` +} + +// +genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:storageversion + +// StaticRoute is the Schema for the staticroutes API. +// +kubebuilder:printcolumn:name="Network",type=string,JSONPath=`.spec.network`,description="Network in CIDR format" +// +kubebuilder:printcolumn:name="NextHops",type=string,JSONPath=`.spec.nextHops[*].ipAddress`,description="Next Hops" +type StaticRoute struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec StaticRouteSpec `json:"spec,omitempty"` + Status StaticRouteStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// StaticRouteList contains a list of StaticRoute. +type StaticRouteList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []StaticRoute `json:"items"` +} + +func init() { + SchemeBuilder.Register(&StaticRoute{}, &StaticRouteList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/subnet_types.go b/external/nsx-operator/api/vpc/v1alpha1/subnet_types.go new file mode 100644 index 000000000..13edbdae9 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/subnet_types.go @@ -0,0 +1,172 @@ +/* Copyright © 2022-2025 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type AccessMode string +type DHCPConfigMode string +type ConnectivityState string + +const ( + AccessModePublic string = "Public" + AccessModePrivate string = "Private" + AccessModeProject string = "PrivateTGW" + DHCPConfigModeDeactivated string = "DHCPDeactivated" + DHCPConfigModeServer string = "DHCPServer" + DHCPConfigModeRelay string = "DHCPRelay" + ConnectivityStateConnected ConnectivityState = "Connected" + ConnectivityStateDisconnected ConnectivityState = "Disconnected" +) + +// SubnetSpec defines the desired state of Subnet. +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.vpcName) || self.vpcName == oldSelf.vpcName",message="vpcName is immutable after set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv4SubnetSize) || has(self.ipv4SubnetSize)", message="ipv4SubnetSize is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.accessMode) || has(self.accessMode)", message="accessMode is required once set" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.advancedConfig) && has(oldSelf.advancedConfig.staticIPAllocation) && has(oldSelf.advancedConfig.staticIPAllocation.enabled) && (!has(self.advancedConfig.staticIPAllocation.enabled) || oldSelf.advancedConfig.staticIPAllocation.enabled != self.advancedConfig.staticIPAllocation.enabled))", message="staticIPAllocation enabled cannot be changed once set" +// +kubebuilder:validation:XValidation:rule="!(has(self.advancedConfig) && has(self.advancedConfig.staticIPAllocation) && has(self.advancedConfig.staticIPAllocation.enabled) && self.advancedConfig.staticIPAllocation.enabled==true && has(self.subnetDHCPConfig) && has(self.subnetDHCPConfig.mode) && (self.subnetDHCPConfig.mode=='DHCPServer' || self.subnetDHCPConfig.mode=='DHCPRelay'))", message="Static IP allocation and Subnet DHCP configuration cannot be enabled simultaneously on a Subnet" +// +kubebuilder:validation:XValidation:rule="!(has(self.advancedConfig) && has(self.advancedConfig.dhcpServerAddresses) && size(self.advancedConfig.dhcpServerAddresses)>0 && (!has(self.subnetDHCPConfig) || !has(self.subnetDHCPConfig.mode) || self.subnetDHCPConfig.mode!='DHCPServer'))", message="DHCPServerAddresses can only be set when DHCP mode is DHCPServer" +// +kubebuilder:validation:XValidation:rule="!has(self.ipAddresses) && !(has(self.subnetDHCPConfig) && has(self.subnetDHCPConfig.dhcpServerAdditionalConfig) && has(self.subnetDHCPConfig.dhcpServerAdditionalConfig.reservedIPRanges)) || has(self.ipAddresses)", message="ipAddresses is required to configure subnet reserved ip ranges." +// +kubebuilder:validation:XValidation:rule="!(has(self.advancedConfig) && has(self.advancedConfig.gatewayAddresses) && size(self.advancedConfig.gatewayAddresses)>0) || has(self.ipAddresses)", message="ipAddresses is required when custom gatewayAddresses are specified" +// +kubebuilder:validation:XValidation:rule="!(has(self.advancedConfig) && has(self.advancedConfig.dhcpServerAddresses) && size(self.advancedConfig.dhcpServerAddresses)>0) || has(self.ipAddresses)", message="ipAddresses is required when custom dhcpServerAddresses are specified" +type SubnetSpec struct { + // VPC name of the Subnet. + VPCName string `json:"vpcName,omitempty"` + // Size of Subnet based upon estimated workload count. + // +kubebuilder:validation:Maximum:=65536 + // +kubebuilder:validation:Minimum:=16 + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"` + // Access mode of Subnet, accessible only from within VPC or from outside VPC. + // +kubebuilder:validation:Enum=Private;Public;PrivateTGW + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + AccessMode AccessMode `json:"accessMode,omitempty"` + // Subnet CIDRS. + // +kubebuilder:validation:MinItems=0 + // +kubebuilder:validation:MaxItems=2 + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + IPAddresses []string `json:"ipAddresses,omitempty"` + + // DHCP mode of a Subnet can only switch between DHCPServer or DHCPRelay. + // If subnetDHCPConfig is not set, the DHCP mode is DHCPDeactivated by default. + // In order to enforce this rule, three XValidation rules are defined. + // The rule on SubnetSpec prevents the condition that subnetDHCPConfig is not set in + // old or current SubnetSpec while the current or old SubnetSpec specifies a Mode + // other than DHCPDeactivated. + // The rule on SubnetDHCPConfig prevents the condition that Mode is not set in old + // or current SubnetDHCPConfig while the current or old one specifies a Mode other + // than DHCPDeactivated. + // The rule on SubnetDHCPConfig.Mode prevents the Mode changing between DHCPDeactivated + // and DHCPServer or DHCPRelay. + + // DHCP configuration for Subnet. + SubnetDHCPConfig SubnetDHCPConfig `json:"subnetDHCPConfig,omitempty"` + // VPC Subnet advanced configuration. + AdvancedConfig SubnetAdvancedConfig `json:"advancedConfig,omitempty"` +} + +// SubnetStatus defines the observed state of Subnet. +type SubnetStatus struct { + // Network address of the Subnet. + NetworkAddresses []string `json:"networkAddresses,omitempty"` + // Gateway address of the Subnet. + GatewayAddresses []string `json:"gatewayAddresses,omitempty"` + // DHCP server IP address. + DHCPServerAddresses []string `json:"DHCPServerAddresses,omitempty"` + // VLAN extension configured for VPC Subnet. + VLANExtension VLANExtension `json:"vlanExtension,omitempty"` + // Whether this is a pre-created Subnet shared with the Namespace. + // +kubebuilder:default=false + Shared bool `json:"shared,omitempty"` + Conditions []Condition `json:"conditions,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion + +// Subnet is the Schema for the subnets API. +// +kubebuilder:printcolumn:name="AccessMode",type=string,JSONPath=`.spec.accessMode`,description="Access mode of Subnet" +// +kubebuilder:printcolumn:name="IPv4SubnetSize",type=string,JSONPath=`.spec.ipv4SubnetSize`,description="Size of Subnet" +// +kubebuilder:printcolumn:name="NetworkAddresses",type=string,JSONPath=`.status.networkAddresses[*]`,description="CIDRs for the Subnet" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec) || has(self.spec)", message="spec is required once set" +type Subnet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SubnetSpec `json:"spec,omitempty"` + Status SubnetStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// SubnetList contains a list of Subnet. +type SubnetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Subnet `json:"items"` +} + +type SubnetAdvancedConfig struct { + // Connectivity status of the Subnet from other Subnets of the VPC. + // The default value is "Connected". + // +kubebuilder:validation:Enum=Connected;Disconnected + // +kubebuilder:default=Connected + ConnectivityState ConnectivityState `json:"connectivityState,omitempty"` + // Whether this Subnet enabled VLAN extension. + // Default value is false. + // +kubebuilder:default=false + EnableVLANExtension bool `json:"enableVLANExtension,omitempty"` + // Static IP allocation for VPC Subnet Ports. + StaticIPAllocation StaticIPAllocation `json:"staticIPAllocation,omitempty"` + // GatewayAddresses specifies custom gateway IP addresses for the Subnet. + // +kubebuilder:validation:MaxItems=1 + GatewayAddresses []string `json:"gatewayAddresses,omitempty"` + // DHCPServerAddresses specifies custom DHCP server IP addresses for the Subnet. + // +kubebuilder:validation:MaxItems=1 + DHCPServerAddresses []string `json:"dhcpServerAddresses,omitempty"` +} + +type StaticIPAllocation struct { + // Activate or deactivate static IP allocation for VPC Subnet Ports. + // If the DHCP mode is DHCPDeactivated or not set, its default value is true. + // If the DHCP mode is DHCPServer or DHCPRelay, its default value is false. + // The value cannot be set to true when the DHCP mode is DHCPServer or DHCPRelay. + Enabled *bool `json:"enabled,omitempty"` +} + +// Additional DHCP server config for a VPC Subnet. +// The additional configuration must not be set when the Subnet has DHCP relay enabled or DHCP is deactivated. +type DHCPServerAdditionalConfig struct { + // Reserved IP ranges. + // Supported formats include: ["192.168.1.1", "192.168.1.3-192.168.1.100"] + // +kubebuilder:validation::MaxItems=10 + ReservedIPRanges []string `json:"reservedIPRanges,omitempty"` +} + +// SubnetDHCPConfig is a DHCP configuration for Subnet. +// +kubebuilder:validation:XValidation:rule="(!has(self.mode)|| self.mode=='DHCPDeactivated' || self.mode=='DHCPRelay' ) && (!has(self.dhcpServerAdditionalConfig) || !has(self.dhcpServerAdditionalConfig.reservedIPRanges) || size(self.dhcpServerAdditionalConfig.reservedIPRanges)==0) || has(self.mode) && self.mode=='DHCPServer'", message="DHCPServerAdditionalConfig must be cleared when Subnet has DHCP relay enabled or DHCP is deactivated." +type SubnetDHCPConfig struct { + // DHCP Mode. DHCPDeactivated will be used if it is not defined. + // It cannot switch from DHCPDeactivated to DHCPServer or DHCPRelay. + // +kubebuilder:validation:Enum=DHCPServer;DHCPRelay;DHCPDeactivated + Mode DHCPConfigMode `json:"mode,omitempty"` + // Additional DHCP server config for a VPC Subnet. + DHCPServerAdditionalConfig DHCPServerAdditionalConfig `json:"dhcpServerAdditionalConfig,omitempty"` +} + +// VLANExtension describes VLAN extension configuration for the VPC Subnet. +type VLANExtension struct { + // Flag to control whether the VLAN extension Subnet connects to the VPC gateway. + VPCGatewayConnectionEnable bool `json:"vpcGatewayConnectionEnable,omitempty"` + // VLAN ID of the VLAN extension Subnet. + VLANID int `json:"vlanId,omitempty"` +} + +func init() { + SchemeBuilder.Register(&Subnet{}, &SubnetList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/subnetconnectionbindingmap_types.go b/external/nsx-operator/api/vpc/v1alpha1/subnetconnectionbindingmap_types.go new file mode 100644 index 000000000..63ec553f0 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/subnetconnectionbindingmap_types.go @@ -0,0 +1,66 @@ +/* Copyright © 2024-2025 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package v1alpha1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +// +kubebuilder:validation:XValidation:rule="has(self.targetSubnetSetName) && !has(self.targetSubnetName) || !has(self.targetSubnetSetName) && has(self.targetSubnetName)",message="Only one of targetSubnetSetName or targetSubnetName can be specified" +// +kubebuilder:validation:XValidation:rule="!has(self.targetSubnetName) || (self.subnetName != self.targetSubnetName)",message="subnetName and targetSubnetName must be different" +type SubnetConnectionBindingMapSpec struct { + // SubnetName is the Subnet name which this SubnetConnectionBindingMap is associated. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="subnetName is immutable" + SubnetName string `json:"subnetName"` + // TargetSubnetSetName specifies the target SubnetSet which a Subnet is connected to. + // +kubebuilder:validation:Optional + TargetSubnetSetName string `json:"targetSubnetSetName,omitempty"` + // TargetSubnetName specifies the target Subnet which a Subnet is connected to. + // +kubebuilder:validation:Optional + TargetSubnetName string `json:"targetSubnetName,omitempty"` + // VLANTrafficTag is the VLAN tag configured in the binding. Note, the value of VLANTrafficTag should be + // unique on the target Subnet or SubnetSet. + // +kubebuilder:validation:Maximum:=4095 + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Required + VLANTrafficTag int64 `json:"vlanTrafficTag"` +} + +// SubnetConnectionBindingMapStatus defines the observed state of SubnetConnectionBindingMap. +type SubnetConnectionBindingMapStatus struct { + // Conditions described if the SubnetConnectionBindingMaps is configured on NSX or not. + // Condition type "" + Conditions []Condition `json:"conditions,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:resource:scope="Namespaced",path=subnetconnectionbindingmaps,shortName=subnetbinding;subnetbindings +// +kubebuilder:selectablefield:JSONPath=`.spec.subnetName` + +// SubnetConnectionBindingMap is the Schema for the SubnetConnectionBindingMap API. +// +kubebuilder:printcolumn:name="name",type=string,JSONPath=`.metadata.name`,description="The name of the SubnetConnectionBindingMap resource" +// +kubebuilder:printcolumn:name="subnet",type=string,JSONPath=`.spec.subnetName`,description="The Subnet which the SubnetConnectionBindingMap is associated" +// +kubebuilder:printcolumn:name="targetSubnet",type=string,JSONPath=`.spec.targetSubnetName`,description="The target Subnet which the SubnetConnectionBindingMap is connected to" +// +kubebuilder:printcolumn:name="targetSubnetSet",type=string,JSONPath=`.spec.targetSubnetSetName`,description="The target SubnetSet which the SubnetConnectionBindingMap is connected to" +// +kubebuilder:printcolumn:name="vlanTrafficTag",type=integer,JSONPath=`.spec.vlanTrafficTag`,description="Vlan used in the NSX SubnetConnectionBindingMap" +type SubnetConnectionBindingMap struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec SubnetConnectionBindingMapSpec `json:"spec,omitempty"` + Status SubnetConnectionBindingMapStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// SubnetConnectionBindingMapList contains a list of SubnetConnectionBindingMap. +type SubnetConnectionBindingMapList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SubnetConnectionBindingMap `json:"items,omitempty"` +} + +func init() { + SchemeBuilder.Register(&SubnetConnectionBindingMap{}, &SubnetConnectionBindingMapList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/subnetipreservation_types.go b/external/nsx-operator/api/vpc/v1alpha1/subnetipreservation_types.go new file mode 100644 index 000000000..709300704 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/subnetipreservation_types.go @@ -0,0 +1,65 @@ +/* Copyright © 2025 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SubnetIPReservationSpec defines the desired state of SubnetIPReservation +type SubnetIPReservationSpec struct { + // Subnet specifies the Subnet to reserve IPs from. + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Subnet is immutable" + Subnet string `json:"subnet"` + + // NumberOfIPs defines number of IPs requested to be reserved. + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="NumberOfIPs is immutable" + // +kubebuilder:validation:Maximum:=100 + // +kubebuilder:validation:Minimum:=1 + NumberOfIPs int `json:"numberOfIPs"` +} + +// SubnetIPReservationStatus defines the observed state of SubnetIPReservation +type SubnetIPReservationStatus struct { + // Conditions described if the SubnetIPReservation is configured on NSX or not. + // Condition type "" + Conditions []Condition `json:"conditions,omitempty"` + // List of reserved IPs. + // Supported formats include: ["192.168.1.1", "192.168.1.3-192.168.1.100"] + IPs []string `json:"ips,omitempty"` +} + +//+genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:storageversion +//+kubebuilder:selectablefield:JSONPath=.spec.subnet + +// SubnetIPReservation is the Schema for the subnetipreservations API +// +kubebuilder:printcolumn:name="Subnet",type=string,JSONPath=`.spec.subnet`,description="The parent Subnet name of the SubnetIPReservation." +// +kubebuilder:printcolumn:name="NumberOfIPs",type=string,JSONPath=`.spec.numberOfIPs`,description="Number of IPs requested to be reserved." +// +kubebuilder:printcolumn:name="IPs",type=string,JSONPath=`.status.ips[:]`,description="List of reserved IPs." +type SubnetIPReservation struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +kubebuilder:validation:Required + Spec SubnetIPReservationSpec `json:"spec"` + Status SubnetIPReservationStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// SubnetIPReservationList contains a list of SubnetIPReservation +type SubnetIPReservationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SubnetIPReservation `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SubnetIPReservation{}, &SubnetIPReservationList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/subnetport_types.go b/external/nsx-operator/api/vpc/v1alpha1/subnetport_types.go new file mode 100644 index 000000000..7004fddd1 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/subnetport_types.go @@ -0,0 +1,90 @@ +/* Copyright © 2022-2025 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +kubebuilder:validation:XValidation:rule="!has(self.subnetSet) || !has(self.subnet)",message="Only one of subnet or subnetSet can be specified or both set to empty in which case default SubnetSet for VM will be used" +// SubnetPortSpec defines the desired state of SubnetPort. +type SubnetPortSpec struct { + // Subnet defines the parent Subnet name of the SubnetPort. + Subnet string `json:"subnet,omitempty"` + // SubnetSet defines the parent SubnetSet name of the SubnetPort. + SubnetSet string `json:"subnetSet,omitempty"` + // AddressBindings defines static address bindings used for the SubnetPort. + AddressBindings []PortAddressBinding `json:"addressBindings,omitempty"` +} + +// PortAddressBinding defines static addresses for the Port. +type PortAddressBinding struct { + // The IP Address. + IPAddress string `json:"ipAddress,omitempty"` + // The MAC address. + MACAddress string `json:"macAddress,omitempty"` +} + +// SubnetPortStatus defines the observed state of SubnetPort. +type SubnetPortStatus struct { + // Conditions describes current state of SubnetPort. + Conditions []Condition `json:"conditions,omitempty"` + // SubnetPort attachment state. + Attachment PortAttachment `json:"attachment,omitempty"` + NetworkInterfaceConfig NetworkInterfaceConfig `json:"networkInterfaceConfig,omitempty"` +} + +// VIF attachment state of a SubnetPort. +type PortAttachment struct { + // ID of the SubnetPort VIF attachment. + ID string `json:"id,omitempty"` +} + +type NetworkInterfaceConfig struct { + // NSX Logical Switch UUID of the Subnet. + LogicalSwitchUUID string `json:"logicalSwitchUUID,omitempty"` + IPAddresses []NetworkInterfaceIPAddress `json:"ipAddresses,omitempty"` + // The MAC address. + MACAddress string `json:"macAddress,omitempty"` + // DHCPDeactivatedOnSubnet indicates whether DHCP is deactivated on the Subnet. + DHCPDeactivatedOnSubnet bool `json:"dhcpDeactivatedOnSubnet,omitempty"` +} + +type NetworkInterfaceIPAddress struct { + // IP address string with the prefix. + IPAddress string `json:"ipAddress,omitempty"` + // Gateway address of the Subnet. + Gateway string `json:"gateway,omitempty"` +} + +// +genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:storageversion +//+kubebuilder:selectablefield:JSONPath=`.spec.subnet` + +// SubnetPort is the Schema for the subnetports API. +// +kubebuilder:printcolumn:name="VIFID",type=string,JSONPath=`.status.attachment.id`,description="Attachment VIF ID owned by the SubnetPort." +// +kubebuilder:printcolumn:name="IPAddress",type=string,JSONPath=`.status.networkInterfaceConfig.ipAddresses[0].ipAddress`,description="IP address string with the prefix." +// +kubebuilder:printcolumn:name="MACAddress",type=string,JSONPath=`.status.networkInterfaceConfig.macAddress`,description="MAC Address of the SubnetPort." +type SubnetPort struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SubnetPortSpec `json:"spec,omitempty"` + Status SubnetPortStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// SubnetPortList contains a list of SubnetPort. +type SubnetPortList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SubnetPort `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SubnetPort{}, &SubnetPortList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/subnetset_types.go b/external/nsx-operator/api/vpc/v1alpha1/subnetset_types.go new file mode 100644 index 000000000..3c718d41c --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/subnetset_types.go @@ -0,0 +1,85 @@ +/* Copyright © 2022-2023 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SubnetSetSpec defines the desired state of SubnetSet. +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.accessMode) || has(self.accessMode)", message="accessMode is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv4SubnetSize) || has(self.ipv4SubnetSize)", message="ipv4SubnetSize is required once set" +// +kubebuilder:validation:XValidation:rule="!has(self.subnetDHCPConfig) || has(self.subnetDHCPConfig) && !has(self.subnetDHCPConfig.dhcpServerAdditionalConfig) || has(self.subnetDHCPConfig) && has(self.subnetDHCPConfig.dhcpServerAdditionalConfig) && !has(self.subnetDHCPConfig.dhcpServerAdditionalConfig.reservedIPRanges)", message="reservedIPRanges is not supported in SubnetSet" +type SubnetSetSpec struct { + // Size of Subnet based upon estimated workload count. + // +kubebuilder:validation:Maximum:=65536 + // +kubebuilder:validation:Minimum:=16 + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"` + // Access mode of Subnet, accessible only from within VPC or from outside VPC. + // +kubebuilder:validation:Enum=Private;Public;PrivateTGW + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + AccessMode AccessMode `json:"accessMode,omitempty"` + // DHCP mode of a Subnet can only switch between DHCPServer or DHCPRelay. + // If subnetDHCPConfig is not set, the DHCP mode is DHCPDeactivated by default. + // In order to enforce this rule, three XValidation rules are defined. + // The rule on SubnetSpec prevents the condition that subnetDHCPConfig is not set in + // old or current SubnetSpec while the current or old SubnetSpec specifies a Mode + // other than DHCPDeactivated. + // The rule on SubnetDHCPConfig prevents the condition that Mode is not set in old + // or current SubnetDHCPConfig while the current or old one specifies a Mode other + // than DHCPDeactivated. + // The rule on SubnetDHCPConfig.Mode prevents the Mode changing between DHCPDeactivated + // and DHCPServer or DHCPRelay. + + // Subnet DHCP configuration. + SubnetDHCPConfig SubnetDHCPConfig `json:"subnetDHCPConfig,omitempty"` +} + +// SubnetInfo defines the observed state of a single Subnet of a SubnetSet. +type SubnetInfo struct { + // Network address of the Subnet. + NetworkAddresses []string `json:"networkAddresses,omitempty"` + // Gateway address of the Subnet. + GatewayAddresses []string `json:"gatewayAddresses,omitempty"` + // Dhcp server IP address. + DHCPServerAddresses []string `json:"DHCPServerAddresses,omitempty"` +} + +// SubnetSetStatus defines the observed state of SubnetSet. +type SubnetSetStatus struct { + Conditions []Condition `json:"conditions,omitempty"` + Subnets []SubnetInfo `json:"subnets,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion + +// SubnetSet is the Schema for the subnetsets API. +// +kubebuilder:printcolumn:name="AccessMode",type=string,JSONPath=`.spec.accessMode`,description="Access mode of Subnet" +// +kubebuilder:printcolumn:name="IPv4SubnetSize",type=string,JSONPath=`.spec.ipv4SubnetSize`,description="Size of Subnet" +// +kubebuilder:printcolumn:name="NetworkAddresses",type=string,JSONPath=`.status.subnets[*].networkAddresses[*]`,description="CIDRs for the SubnetSet" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec) || has(self.spec)", message="spec is required once set" +type SubnetSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SubnetSetSpec `json:"spec,omitempty"` + Status SubnetSetStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// SubnetSetList contains a list of SubnetSet. +type SubnetSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SubnetSet `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SubnetSet{}, &SubnetSetList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/vpcnetworkconfiguration_types.go b/external/nsx-operator/api/vpc/v1alpha1/vpcnetworkconfiguration_types.go new file mode 100644 index 000000000..5dab052a5 --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/vpcnetworkconfiguration_types.go @@ -0,0 +1,92 @@ +/* Copyright © 2022-2025 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +// +kubebuilder:object:generate=true +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VPCNetworkConfigurationSpec defines the desired state of VPCNetworkConfiguration. +// There is a default VPCNetworkConfiguration that applies to Namespaces +// do not have a VPCNetworkConfiguration assigned. When a field is not set +// in a Namespace's VPCNetworkConfiguration, the Namespace will use the value +// in the default VPCNetworkConfiguration. +type VPCNetworkConfigurationSpec struct { + // NSX path of the VPC the Namespace is associated with. + // If vpc is set, only defaultSubnetSize takes effect, other fields are ignored. + // +optional + VPC string `json:"vpc,omitempty"` + + // NSX path of the shared Subnets the Namespace is associated with. + // +optional + Subnets []string `json:"subnets,omitempty"` + + // NSX Project the Namespace is associated with. + NSXProject string `json:"nsxProject,omitempty"` + + // VPCConnectivityProfile Path. This profile has configuration related to creating VPC transit gateway attachment. + VPCConnectivityProfile string `json:"vpcConnectivityProfile,omitempty"` + + // Private IPs. + PrivateIPs []string `json:"privateIPs,omitempty"` + + // Default size of Subnets. + // Defaults to 32. + // +kubebuilder:default=32 + // +kubebuilder:validation:Maximum:=65536 + // +kubebuilder:validation:Minimum:=16 + DefaultSubnetSize int `json:"defaultSubnetSize,omitempty"` +} + +// VPCNetworkConfigurationStatus defines the observed state of VPCNetworkConfiguration +type VPCNetworkConfigurationStatus struct { + // VPCs describes VPC info, now it includes Load Balancer Subnet info which are needed + // for the Avi Kubernetes Operator (AKO). + VPCs []VPCInfo `json:"vpcs,omitempty"` + // Conditions describe current state of VPCNetworkConfiguration. + Conditions []Condition `json:"conditions,omitempty"` +} + +// VPCInfo defines VPC info needed by tenant admin. +type VPCInfo struct { + // VPC name. + Name string `json:"name"` + // AVISESubnetPath is the NSX Policy Path for the AVI SE Subnet. + AVISESubnetPath string `json:"lbSubnetPath,omitempty"` + // NSXLoadBalancerPath is the NSX Policy path for the NSX Load Balancer. + NSXLoadBalancerPath string `json:"nsxLoadBalancerPath,omitempty"` + // NSX Policy path for VPC. + VPCPath string `json:"vpcPath"` +} + +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion + +// VPCNetworkConfiguration is the Schema for the vpcnetworkconfigurations API. +// +kubebuilder:resource:scope="Cluster" +// +kubebuilder:printcolumn:name="VPCPath",type=string,JSONPath=`.status.vpcs[0].vpcPath`,description="NSX VPC path the Namespace is associated with" +type VPCNetworkConfiguration struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec VPCNetworkConfigurationSpec `json:"spec,omitempty"` + Status VPCNetworkConfigurationStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// VPCNetworkConfigurationList contains a list of VPCNetworkConfiguration. +type VPCNetworkConfigurationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []VPCNetworkConfiguration `json:"items"` +} + +func init() { + SchemeBuilder.Register(&VPCNetworkConfiguration{}, &VPCNetworkConfigurationList{}) +} diff --git a/external/nsx-operator/api/vpc/v1alpha1/zz_generated.deepcopy.go b/external/nsx-operator/api/vpc/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..78593789c --- /dev/null +++ b/external/nsx-operator/api/vpc/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,1622 @@ +//go:build !ignore_autogenerated + +/* Copyright © 2024 VMware, Inc. All Rights Reserved. + SPDX-License-Identifier: Apache-2.0 */ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddressBinding) DeepCopyInto(out *AddressBinding) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddressBinding. +func (in *AddressBinding) DeepCopy() *AddressBinding { + if in == nil { + return nil + } + out := new(AddressBinding) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AddressBinding) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddressBindingList) DeepCopyInto(out *AddressBindingList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AddressBinding, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddressBindingList. +func (in *AddressBindingList) DeepCopy() *AddressBindingList { + if in == nil { + return nil + } + out := new(AddressBindingList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AddressBindingList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddressBindingSpec) DeepCopyInto(out *AddressBindingSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddressBindingSpec. +func (in *AddressBindingSpec) DeepCopy() *AddressBindingSpec { + if in == nil { + return nil + } + out := new(AddressBindingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddressBindingStatus) DeepCopyInto(out *AddressBindingStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddressBindingStatus. +func (in *AddressBindingStatus) DeepCopy() *AddressBindingStatus { + if in == nil { + return nil + } + out := new(AddressBindingStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Condition) DeepCopyInto(out *Condition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Condition. +func (in *Condition) DeepCopy() *Condition { + if in == nil { + return nil + } + out := new(Condition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DHCPServerAdditionalConfig) DeepCopyInto(out *DHCPServerAdditionalConfig) { + *out = *in + if in.ReservedIPRanges != nil { + in, out := &in.ReservedIPRanges, &out.ReservedIPRanges + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DHCPServerAdditionalConfig. +func (in *DHCPServerAdditionalConfig) DeepCopy() *DHCPServerAdditionalConfig { + if in == nil { + return nil + } + out := new(DHCPServerAdditionalConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocation) DeepCopyInto(out *IPAddressAllocation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocation. +func (in *IPAddressAllocation) DeepCopy() *IPAddressAllocation { + if in == nil { + return nil + } + out := new(IPAddressAllocation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPAddressAllocation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationList) DeepCopyInto(out *IPAddressAllocationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]IPAddressAllocation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationList. +func (in *IPAddressAllocationList) DeepCopy() *IPAddressAllocationList { + if in == nil { + return nil + } + out := new(IPAddressAllocationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPAddressAllocationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationSpec) DeepCopyInto(out *IPAddressAllocationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationSpec. +func (in *IPAddressAllocationSpec) DeepCopy() *IPAddressAllocationSpec { + if in == nil { + return nil + } + out := new(IPAddressAllocationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAddressAllocationStatus) DeepCopyInto(out *IPAddressAllocationStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAddressAllocationStatus. +func (in *IPAddressAllocationStatus) DeepCopy() *IPAddressAllocationStatus { + if in == nil { + return nil + } + out := new(IPAddressAllocationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPBlock) DeepCopyInto(out *IPBlock) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPBlock. +func (in *IPBlock) DeepCopy() *IPBlock { + if in == nil { + return nil + } + out := new(IPBlock) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPBlocksInfo) DeepCopyInto(out *IPBlocksInfo) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.ExternalIPCIDRs != nil { + in, out := &in.ExternalIPCIDRs, &out.ExternalIPCIDRs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PrivateTGWIPCIDRs != nil { + in, out := &in.PrivateTGWIPCIDRs, &out.PrivateTGWIPCIDRs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalIPRanges != nil { + in, out := &in.ExternalIPRanges, &out.ExternalIPRanges + *out = make([]IPPoolRange, len(*in)) + copy(*out, *in) + } + if in.PrivateTGWIPRanges != nil { + in, out := &in.PrivateTGWIPRanges, &out.PrivateTGWIPRanges + *out = make([]IPPoolRange, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPBlocksInfo. +func (in *IPBlocksInfo) DeepCopy() *IPBlocksInfo { + if in == nil { + return nil + } + out := new(IPBlocksInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPBlocksInfo) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPBlocksInfoList) DeepCopyInto(out *IPBlocksInfoList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]IPBlocksInfo, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPBlocksInfoList. +func (in *IPBlocksInfoList) DeepCopy() *IPBlocksInfoList { + if in == nil { + return nil + } + out := new(IPBlocksInfoList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPBlocksInfoList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolRange) DeepCopyInto(out *IPPoolRange) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolRange. +func (in *IPPoolRange) DeepCopy() *IPPoolRange { + if in == nil { + return nil + } + out := new(IPPoolRange) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInfo) DeepCopyInto(out *NetworkInfo) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.VPCs != nil { + in, out := &in.VPCs, &out.VPCs + *out = make([]VPCState, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInfo. +func (in *NetworkInfo) DeepCopy() *NetworkInfo { + if in == nil { + return nil + } + out := new(NetworkInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInfo) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInfoList) DeepCopyInto(out *NetworkInfoList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetworkInfo, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInfoList. +func (in *NetworkInfoList) DeepCopy() *NetworkInfoList { + if in == nil { + return nil + } + out := new(NetworkInfoList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInfoList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceConfig) DeepCopyInto(out *NetworkInterfaceConfig) { + *out = *in + if in.IPAddresses != nil { + in, out := &in.IPAddresses, &out.IPAddresses + *out = make([]NetworkInterfaceIPAddress, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceConfig. +func (in *NetworkInterfaceConfig) DeepCopy() *NetworkInterfaceConfig { + if in == nil { + return nil + } + out := new(NetworkInterfaceConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceIPAddress) DeepCopyInto(out *NetworkInterfaceIPAddress) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceIPAddress. +func (in *NetworkInterfaceIPAddress) DeepCopy() *NetworkInterfaceIPAddress { + if in == nil { + return nil + } + out := new(NetworkInterfaceIPAddress) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NextHop) DeepCopyInto(out *NextHop) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NextHop. +func (in *NextHop) DeepCopy() *NextHop { + if in == nil { + return nil + } + out := new(NextHop) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortAddressBinding) DeepCopyInto(out *PortAddressBinding) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortAddressBinding. +func (in *PortAddressBinding) DeepCopy() *PortAddressBinding { + if in == nil { + return nil + } + out := new(PortAddressBinding) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortAttachment) DeepCopyInto(out *PortAttachment) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortAttachment. +func (in *PortAttachment) DeepCopy() *PortAttachment { + if in == nil { + return nil + } + out := new(PortAttachment) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityPolicy) DeepCopyInto(out *SecurityPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityPolicy. +func (in *SecurityPolicy) DeepCopy() *SecurityPolicy { + if in == nil { + return nil + } + out := new(SecurityPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SecurityPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityPolicyList) DeepCopyInto(out *SecurityPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SecurityPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityPolicyList. +func (in *SecurityPolicyList) DeepCopy() *SecurityPolicyList { + if in == nil { + return nil + } + out := new(SecurityPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SecurityPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityPolicyPeer) DeepCopyInto(out *SecurityPolicyPeer) { + *out = *in + if in.VMSelector != nil { + in, out := &in.VMSelector, &out.VMSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.PodSelector != nil { + in, out := &in.PodSelector, &out.PodSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.NamespaceSelector != nil { + in, out := &in.NamespaceSelector, &out.NamespaceSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.IPBlocks != nil { + in, out := &in.IPBlocks, &out.IPBlocks + *out = make([]IPBlock, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityPolicyPeer. +func (in *SecurityPolicyPeer) DeepCopy() *SecurityPolicyPeer { + if in == nil { + return nil + } + out := new(SecurityPolicyPeer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityPolicyPort) DeepCopyInto(out *SecurityPolicyPort) { + *out = *in + out.Port = in.Port +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityPolicyPort. +func (in *SecurityPolicyPort) DeepCopy() *SecurityPolicyPort { + if in == nil { + return nil + } + out := new(SecurityPolicyPort) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityPolicyRule) DeepCopyInto(out *SecurityPolicyRule) { + *out = *in + if in.Action != nil { + in, out := &in.Action, &out.Action + *out = new(RuleAction) + **out = **in + } + if in.AppliedTo != nil { + in, out := &in.AppliedTo, &out.AppliedTo + *out = make([]SecurityPolicyTarget, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Direction != nil { + in, out := &in.Direction, &out.Direction + *out = new(RuleDirection) + **out = **in + } + if in.Sources != nil { + in, out := &in.Sources, &out.Sources + *out = make([]SecurityPolicyPeer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Destinations != nil { + in, out := &in.Destinations, &out.Destinations + *out = make([]SecurityPolicyPeer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]SecurityPolicyPort, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityPolicyRule. +func (in *SecurityPolicyRule) DeepCopy() *SecurityPolicyRule { + if in == nil { + return nil + } + out := new(SecurityPolicyRule) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityPolicySpec) DeepCopyInto(out *SecurityPolicySpec) { + *out = *in + if in.AppliedTo != nil { + in, out := &in.AppliedTo, &out.AppliedTo + *out = make([]SecurityPolicyTarget, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]SecurityPolicyRule, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityPolicySpec. +func (in *SecurityPolicySpec) DeepCopy() *SecurityPolicySpec { + if in == nil { + return nil + } + out := new(SecurityPolicySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityPolicyStatus) DeepCopyInto(out *SecurityPolicyStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityPolicyStatus. +func (in *SecurityPolicyStatus) DeepCopy() *SecurityPolicyStatus { + if in == nil { + return nil + } + out := new(SecurityPolicyStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityPolicyTarget) DeepCopyInto(out *SecurityPolicyTarget) { + *out = *in + if in.VMSelector != nil { + in, out := &in.VMSelector, &out.VMSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.PodSelector != nil { + in, out := &in.PodSelector, &out.PodSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityPolicyTarget. +func (in *SecurityPolicyTarget) DeepCopy() *SecurityPolicyTarget { + if in == nil { + return nil + } + out := new(SecurityPolicyTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticIPAllocation) DeepCopyInto(out *StaticIPAllocation) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticIPAllocation. +func (in *StaticIPAllocation) DeepCopy() *StaticIPAllocation { + if in == nil { + return nil + } + out := new(StaticIPAllocation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticRoute) DeepCopyInto(out *StaticRoute) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticRoute. +func (in *StaticRoute) DeepCopy() *StaticRoute { + if in == nil { + return nil + } + out := new(StaticRoute) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StaticRoute) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticRouteCondition) DeepCopyInto(out *StaticRouteCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticRouteCondition. +func (in *StaticRouteCondition) DeepCopy() *StaticRouteCondition { + if in == nil { + return nil + } + out := new(StaticRouteCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticRouteList) DeepCopyInto(out *StaticRouteList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]StaticRoute, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticRouteList. +func (in *StaticRouteList) DeepCopy() *StaticRouteList { + if in == nil { + return nil + } + out := new(StaticRouteList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StaticRouteList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticRouteSpec) DeepCopyInto(out *StaticRouteSpec) { + *out = *in + if in.NextHops != nil { + in, out := &in.NextHops, &out.NextHops + *out = make([]NextHop, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticRouteSpec. +func (in *StaticRouteSpec) DeepCopy() *StaticRouteSpec { + if in == nil { + return nil + } + out := new(StaticRouteSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticRouteStatus) DeepCopyInto(out *StaticRouteStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]StaticRouteCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticRouteStatus. +func (in *StaticRouteStatus) DeepCopy() *StaticRouteStatus { + if in == nil { + return nil + } + out := new(StaticRouteStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Subnet) DeepCopyInto(out *Subnet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subnet. +func (in *Subnet) DeepCopy() *Subnet { + if in == nil { + return nil + } + out := new(Subnet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Subnet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetAdvancedConfig) DeepCopyInto(out *SubnetAdvancedConfig) { + *out = *in + in.StaticIPAllocation.DeepCopyInto(&out.StaticIPAllocation) + if in.GatewayAddresses != nil { + in, out := &in.GatewayAddresses, &out.GatewayAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DHCPServerAddresses != nil { + in, out := &in.DHCPServerAddresses, &out.DHCPServerAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetAdvancedConfig. +func (in *SubnetAdvancedConfig) DeepCopy() *SubnetAdvancedConfig { + if in == nil { + return nil + } + out := new(SubnetAdvancedConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetConnectionBindingMap) DeepCopyInto(out *SubnetConnectionBindingMap) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetConnectionBindingMap. +func (in *SubnetConnectionBindingMap) DeepCopy() *SubnetConnectionBindingMap { + if in == nil { + return nil + } + out := new(SubnetConnectionBindingMap) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SubnetConnectionBindingMap) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetConnectionBindingMapList) DeepCopyInto(out *SubnetConnectionBindingMapList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SubnetConnectionBindingMap, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetConnectionBindingMapList. +func (in *SubnetConnectionBindingMapList) DeepCopy() *SubnetConnectionBindingMapList { + if in == nil { + return nil + } + out := new(SubnetConnectionBindingMapList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SubnetConnectionBindingMapList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetConnectionBindingMapSpec) DeepCopyInto(out *SubnetConnectionBindingMapSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetConnectionBindingMapSpec. +func (in *SubnetConnectionBindingMapSpec) DeepCopy() *SubnetConnectionBindingMapSpec { + if in == nil { + return nil + } + out := new(SubnetConnectionBindingMapSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetConnectionBindingMapStatus) DeepCopyInto(out *SubnetConnectionBindingMapStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetConnectionBindingMapStatus. +func (in *SubnetConnectionBindingMapStatus) DeepCopy() *SubnetConnectionBindingMapStatus { + if in == nil { + return nil + } + out := new(SubnetConnectionBindingMapStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetDHCPConfig) DeepCopyInto(out *SubnetDHCPConfig) { + *out = *in + in.DHCPServerAdditionalConfig.DeepCopyInto(&out.DHCPServerAdditionalConfig) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetDHCPConfig. +func (in *SubnetDHCPConfig) DeepCopy() *SubnetDHCPConfig { + if in == nil { + return nil + } + out := new(SubnetDHCPConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetIPReservation) DeepCopyInto(out *SubnetIPReservation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetIPReservation. +func (in *SubnetIPReservation) DeepCopy() *SubnetIPReservation { + if in == nil { + return nil + } + out := new(SubnetIPReservation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SubnetIPReservation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetIPReservationList) DeepCopyInto(out *SubnetIPReservationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SubnetIPReservation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetIPReservationList. +func (in *SubnetIPReservationList) DeepCopy() *SubnetIPReservationList { + if in == nil { + return nil + } + out := new(SubnetIPReservationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SubnetIPReservationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetIPReservationSpec) DeepCopyInto(out *SubnetIPReservationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetIPReservationSpec. +func (in *SubnetIPReservationSpec) DeepCopy() *SubnetIPReservationSpec { + if in == nil { + return nil + } + out := new(SubnetIPReservationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetIPReservationStatus) DeepCopyInto(out *SubnetIPReservationStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.IPs != nil { + in, out := &in.IPs, &out.IPs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetIPReservationStatus. +func (in *SubnetIPReservationStatus) DeepCopy() *SubnetIPReservationStatus { + if in == nil { + return nil + } + out := new(SubnetIPReservationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetInfo) DeepCopyInto(out *SubnetInfo) { + *out = *in + if in.NetworkAddresses != nil { + in, out := &in.NetworkAddresses, &out.NetworkAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.GatewayAddresses != nil { + in, out := &in.GatewayAddresses, &out.GatewayAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DHCPServerAddresses != nil { + in, out := &in.DHCPServerAddresses, &out.DHCPServerAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetInfo. +func (in *SubnetInfo) DeepCopy() *SubnetInfo { + if in == nil { + return nil + } + out := new(SubnetInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetList) DeepCopyInto(out *SubnetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Subnet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetList. +func (in *SubnetList) DeepCopy() *SubnetList { + if in == nil { + return nil + } + out := new(SubnetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SubnetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetPort) DeepCopyInto(out *SubnetPort) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetPort. +func (in *SubnetPort) DeepCopy() *SubnetPort { + if in == nil { + return nil + } + out := new(SubnetPort) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SubnetPort) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetPortList) DeepCopyInto(out *SubnetPortList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SubnetPort, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetPortList. +func (in *SubnetPortList) DeepCopy() *SubnetPortList { + if in == nil { + return nil + } + out := new(SubnetPortList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SubnetPortList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetPortSpec) DeepCopyInto(out *SubnetPortSpec) { + *out = *in + if in.AddressBindings != nil { + in, out := &in.AddressBindings, &out.AddressBindings + *out = make([]PortAddressBinding, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetPortSpec. +func (in *SubnetPortSpec) DeepCopy() *SubnetPortSpec { + if in == nil { + return nil + } + out := new(SubnetPortSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetPortStatus) DeepCopyInto(out *SubnetPortStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.Attachment = in.Attachment + in.NetworkInterfaceConfig.DeepCopyInto(&out.NetworkInterfaceConfig) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetPortStatus. +func (in *SubnetPortStatus) DeepCopy() *SubnetPortStatus { + if in == nil { + return nil + } + out := new(SubnetPortStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetSet) DeepCopyInto(out *SubnetSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetSet. +func (in *SubnetSet) DeepCopy() *SubnetSet { + if in == nil { + return nil + } + out := new(SubnetSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SubnetSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetSetList) DeepCopyInto(out *SubnetSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SubnetSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetSetList. +func (in *SubnetSetList) DeepCopy() *SubnetSetList { + if in == nil { + return nil + } + out := new(SubnetSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SubnetSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetSetSpec) DeepCopyInto(out *SubnetSetSpec) { + *out = *in + in.SubnetDHCPConfig.DeepCopyInto(&out.SubnetDHCPConfig) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetSetSpec. +func (in *SubnetSetSpec) DeepCopy() *SubnetSetSpec { + if in == nil { + return nil + } + out := new(SubnetSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetSetStatus) DeepCopyInto(out *SubnetSetStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]SubnetInfo, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetSetStatus. +func (in *SubnetSetStatus) DeepCopy() *SubnetSetStatus { + if in == nil { + return nil + } + out := new(SubnetSetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetSpec) DeepCopyInto(out *SubnetSpec) { + *out = *in + if in.IPAddresses != nil { + in, out := &in.IPAddresses, &out.IPAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.SubnetDHCPConfig.DeepCopyInto(&out.SubnetDHCPConfig) + in.AdvancedConfig.DeepCopyInto(&out.AdvancedConfig) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetSpec. +func (in *SubnetSpec) DeepCopy() *SubnetSpec { + if in == nil { + return nil + } + out := new(SubnetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetStatus) DeepCopyInto(out *SubnetStatus) { + *out = *in + if in.NetworkAddresses != nil { + in, out := &in.NetworkAddresses, &out.NetworkAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.GatewayAddresses != nil { + in, out := &in.GatewayAddresses, &out.GatewayAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DHCPServerAddresses != nil { + in, out := &in.DHCPServerAddresses, &out.DHCPServerAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.VLANExtension = in.VLANExtension + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetStatus. +func (in *SubnetStatus) DeepCopy() *SubnetStatus { + if in == nil { + return nil + } + out := new(SubnetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VLANExtension) DeepCopyInto(out *VLANExtension) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VLANExtension. +func (in *VLANExtension) DeepCopy() *VLANExtension { + if in == nil { + return nil + } + out := new(VLANExtension) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPCInfo) DeepCopyInto(out *VPCInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCInfo. +func (in *VPCInfo) DeepCopy() *VPCInfo { + if in == nil { + return nil + } + out := new(VPCInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPCNetworkConfiguration) DeepCopyInto(out *VPCNetworkConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCNetworkConfiguration. +func (in *VPCNetworkConfiguration) DeepCopy() *VPCNetworkConfiguration { + if in == nil { + return nil + } + out := new(VPCNetworkConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VPCNetworkConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPCNetworkConfigurationList) DeepCopyInto(out *VPCNetworkConfigurationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VPCNetworkConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCNetworkConfigurationList. +func (in *VPCNetworkConfigurationList) DeepCopy() *VPCNetworkConfigurationList { + if in == nil { + return nil + } + out := new(VPCNetworkConfigurationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VPCNetworkConfigurationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPCNetworkConfigurationSpec) DeepCopyInto(out *VPCNetworkConfigurationSpec) { + *out = *in + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PrivateIPs != nil { + in, out := &in.PrivateIPs, &out.PrivateIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCNetworkConfigurationSpec. +func (in *VPCNetworkConfigurationSpec) DeepCopy() *VPCNetworkConfigurationSpec { + if in == nil { + return nil + } + out := new(VPCNetworkConfigurationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPCNetworkConfigurationStatus) DeepCopyInto(out *VPCNetworkConfigurationStatus) { + *out = *in + if in.VPCs != nil { + in, out := &in.VPCs, &out.VPCs + *out = make([]VPCInfo, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCNetworkConfigurationStatus. +func (in *VPCNetworkConfigurationStatus) DeepCopy() *VPCNetworkConfigurationStatus { + if in == nil { + return nil + } + out := new(VPCNetworkConfigurationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPCState) DeepCopyInto(out *VPCState) { + *out = *in + if in.PrivateIPs != nil { + in, out := &in.PrivateIPs, &out.PrivateIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCState. +func (in *VPCState) DeepCopy() *VPCState { + if in == nil { + return nil + } + out := new(VPCState) + in.DeepCopyInto(out) + return out +} diff --git a/external/nsx-operator/go.mod b/external/nsx-operator/go.mod new file mode 100644 index 000000000..cf73fb11e --- /dev/null +++ b/external/nsx-operator/go.mod @@ -0,0 +1,29 @@ +module github.com/vmware-tanzu/vm-operator/external/nsx-operator + +go 1.26.1 + +require ( + k8s.io/api v0.35.3 + k8s.io/apimachinery v0.35.3 + sigs.k8s.io/controller-runtime v0.23.3 +) + +require ( + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/text v0.31.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect +) diff --git a/external/nsx-operator/go.sum b/external/nsx-operator/go.sum new file mode 100644 index 000000000..a5d672dff --- /dev/null +++ b/external/nsx-operator/go.sum @@ -0,0 +1,88 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ= +k8s.io/api v0.35.3/go.mod h1:9Y9tkBcFwKNq2sxwZTQh1Njh9qHl81D0As56tu42GA4= +k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8= +k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80= +sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1/cnsunregistervolume_types.go b/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1/cnsunregistervolume_types.go new file mode 100644 index 000000000..6dae0ad93 --- /dev/null +++ b/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1/cnsunregistervolume_types.go @@ -0,0 +1,77 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CnsUnregisterVolumeSpec defines the desired state of CnsUnregisterVolume +// +k8s:openapi-gen=true +type CnsUnregisterVolumeSpec struct { + // VolumeID indicates the volume handle of CNS volume to be unregistered + VolumeID string `json:"volumeID,omitempty"` + + // PVCName indicates the name of the PVC to be unregistered. + PVCName string `json:"pvcName,omitempty"` + + // RetainFCD indicates if the volume should be retained as an FCD. + // If set to false or not specified, the volume will be retained as a VMDK. + RetainFCD bool `json:"retainFCD,omitempty"` + + // ForceUnregister indicates if the volume should be forcefully unregistered. + // If set to true, the volume will be unregistered even if it is still in use by any VM. + // This should be used with caution as it may lead to data loss. + ForceUnregister bool `json:"forceUnregister,omitempty"` +} + +// CnsUnregisterVolumeStatus defines the observed state of CnsUnregisterVolume +// +k8s:openapi-gen=true +type CnsUnregisterVolumeStatus struct { + // Indicates the volume is successfully unregistered. + // This field must only be set by the entity completing the unregister + // operation, i.e. the CNS Operator. + Unregistered bool `json:"unregistered"` + + // The last error encountered during export operation, if any. + // This field must only be set by the entity completing the export + // operation, i.e. the CNS Operator. + Error string `json:"error,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CnsUnregisterVolume is the Schema for the cnsunregistervolumes API +// +k8s:openapi-gen=true +// +kubebuilder:subresource:status +type CnsUnregisterVolume struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CnsUnregisterVolumeSpec `json:"spec,omitempty"` + Status CnsUnregisterVolumeStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// CnsUnregisterVolumeList contains a list of CnsUnregisterVolume +type CnsUnregisterVolumeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CnsUnregisterVolume `json:"items"` +} + diff --git a/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1/groupversion_info.go b/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..cf6654a99 --- /dev/null +++ b/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1/groupversion_info.go @@ -0,0 +1,49 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName specifies the group name used to register the objects. +const GroupName = "cns.vmware.com" + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} + + // schemeBuilder is used to add go types to the GroupVersionKind scheme. + schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = schemeBuilder.AddToScheme + + objectTypes = []runtime.Object{ + &CnsUnregisterVolume{}, + &CnsUnregisterVolumeList{}, + } +) + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(GroupVersion, objectTypes...) + metav1.AddToGroupVersion(scheme, GroupVersion) + return nil +} + diff --git a/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1/zz_generated.deepcopy.go b/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..ec9e4b127 --- /dev/null +++ b/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,72 @@ +//go:build !ignore_autogenerated + +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CnsUnregisterVolume) DeepCopyInto(out *CnsUnregisterVolume) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CnsUnregisterVolume. +func (in *CnsUnregisterVolume) DeepCopy() *CnsUnregisterVolume { + if in == nil { + return nil + } + out := new(CnsUnregisterVolume) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CnsUnregisterVolume) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CnsUnregisterVolumeList) DeepCopyInto(out *CnsUnregisterVolumeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CnsUnregisterVolume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CnsUnregisterVolumeList. +func (in *CnsUnregisterVolumeList) DeepCopy() *CnsUnregisterVolumeList { + if in == nil { + return nil + } + out := new(CnsUnregisterVolumeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CnsUnregisterVolumeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/go.mod b/go.mod index 4253792dd..b7bee1ddf 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/vmware-tanzu/vm-operator -go 1.26.1 +go 1.26.2 replace ( github.com/vmware-tanzu/vm-operator/api => ./api @@ -35,7 +35,7 @@ require ( // The version of Ginkgo must match the version in hack/tools/go.mod and api/go.mod. // If updating one, please update the others. -require github.com/onsi/ginkgo/v2 v2.23.4 +require github.com/onsi/ginkgo/v2 v2.27.2 require ( github.com/cespare/xxhash/v2 v2.3.0 @@ -44,10 +44,10 @@ require ( github.com/go-pkgz/expirable-cache/v3 v3.1.0 github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 - github.com/onsi/gomega v1.36.3 - github.com/prometheus/client_golang v1.22.0 + github.com/onsi/gomega v1.38.2 + github.com/prometheus/client_golang v1.23.2 github.com/vmware-tanzu/image-registry-operator-api v0.0.0-20250624211456-dfc90459c658 - github.com/vmware-tanzu/net-operator-api v0.0.0-20250826165015-90a4bb21727b + github.com/vmware-tanzu/net-operator-api v0.0.0-20260408184803-2370a4eb950f github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20250813103855-288a237381b5 github.com/vmware/govmomi v0.53.0-alpha.0.0.20260330184955-83b4909d9b9b golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 @@ -55,19 +55,20 @@ require ( // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/24 golang.org/x/text v0.34.0 golang.org/x/tools v0.41.0 - k8s.io/api v0.34.1 - k8s.io/apiextensions-apiserver v0.34.1 - k8s.io/apimachinery v0.34.1 - k8s.io/client-go v0.34.1 - k8s.io/component-base v0.34.1 - k8s.io/component-helpers v0.33.0 + k8s.io/api v0.35.4 + k8s.io/apiextensions-apiserver v0.35.4 + k8s.io/apimachinery v0.35.4 + k8s.io/client-go v0.35.4 + k8s.io/component-base v0.35.4 + k8s.io/component-helpers v0.35.4 k8s.io/klog/v2 v2.130.1 - sigs.k8s.io/controller-runtime v0.22.3 + sigs.k8s.io/controller-runtime v0.23.1 sigs.k8s.io/yaml v1.6.0 ) require ( cel.dev/expr v0.25.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect @@ -84,7 +85,6 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/cel-go v0.26.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect @@ -97,29 +97,28 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/spf13/cobra v1.9.1 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/spf13/cobra v1.10.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect - go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect - go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect - go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.32.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.40.0 // indirect @@ -130,14 +129,14 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/grpc v1.79.3 // indirect google.golang.org/protobuf v1.36.10 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.34.1 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + k8s.io/apiserver v0.35.4 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect ) diff --git a/go.sum b/go.sum index 59170919e..90085a1d7 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -28,6 +30,12 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -47,8 +55,8 @@ github.com/go-pkgz/expirable-cache/v3 v3.1.0 h1:s05P851/O6QJ6Mc+7o2bh9aGtD3romB1 github.com/go-pkgz/expirable-cache/v3 v3.1.0/go.mod h1:6pVgNleydKPj0J2/mzrI02/RDo4ivKx5v2XlNmIjhjo= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= @@ -74,10 +82,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -91,6 +99,10 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -99,31 +111,30 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= -github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= -github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= -github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.10.0 h1:a5/WeUlSDCvV5a45ljW2ZFtV0bTDpkfSAj3uqB6Sc+0= +github.com/spf13/cobra v1.10.0/go.mod h1:9dhySC7dnTtEiqzmqfkLj47BslqLCUPMXjG2lj/NgoE= +github.com/spf13/pflag v1.0.8/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -137,93 +148,74 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/vmware-tanzu/image-registry-operator-api v0.0.0-20250624211456-dfc90459c658 h1:JJg5zTkKLyCQDcKJpuOGiZM2aqQ7NWe5VJT+H9lpQrE= github.com/vmware-tanzu/image-registry-operator-api v0.0.0-20250624211456-dfc90459c658/go.mod h1:sh4NJb1tCbzNRJ+ajRuu3thDovFN10Hic2wYmyklG/M= -github.com/vmware-tanzu/net-operator-api v0.0.0-20250826165015-90a4bb21727b h1:4LXcpS7olGK7vDtzpkSoGMvkFYm0HNdzMqJxnTiv0sY= -github.com/vmware-tanzu/net-operator-api v0.0.0-20250826165015-90a4bb21727b/go.mod h1:w6QJGm3crIA16ZIz1FVQXD2NVeJhOgGXxW05RbVTSTo= +github.com/vmware-tanzu/net-operator-api v0.0.0-20260408184803-2370a4eb950f h1:YT8YK7YiYFNqyLkxApxbspe3EnCB7/JJ09xS86ybuCs= +github.com/vmware-tanzu/net-operator-api v0.0.0-20260408184803-2370a4eb950f/go.mod h1:w6QJGm3crIA16ZIz1FVQXD2NVeJhOgGXxW05RbVTSTo= github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20250813103855-288a237381b5 h1:OUPe+BjC/XWqHRWYHDCtTa/lZpEz6U8YmZcZi1Rp5BU= github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20250813103855-288a237381b5/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk= github.com/vmware/govmomi v0.53.0-alpha.0.0.20260330184955-83b4909d9b9b h1:iPvfXmFW/VHcJUk7nvbJEpXArZMSKgGySHvVH/CmJBQ= github.com/vmware/govmomi v0.53.0-alpha.0.0.20260330184955-83b4909d9b9b/go.mod h1:0F3hChqXDrSQQnjfSiCqRE5lPD4aZlbOtKG4uroq2a4= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= -go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= @@ -239,42 +231,42 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= -k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= -k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= -k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= -k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= -k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= -k8s.io/apiserver v0.34.1 h1:U3JBGdgANK3dfFcyknWde1G6X1F4bg7PXuvlqt8lITA= -k8s.io/apiserver v0.34.1/go.mod h1:eOOc9nrVqlBI1AFCvVzsob0OxtPZUCPiUJL45JOTBG0= -k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= -k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= -k8s.io/component-base v0.34.1 h1:v7xFgG+ONhytZNFpIz5/kecwD+sUhVE6HU7qQUiRM4A= -k8s.io/component-base v0.34.1/go.mod h1:mknCpLlTSKHzAQJJnnHVKqjxR7gBeHRv0rPXA7gdtQ0= -k8s.io/component-helpers v0.33.0 h1:0AdW0A0mIgljLgtG0hJDdJl52PPqTrtMgOgtm/9i/Ys= -k8s.io/component-helpers v0.33.0/go.mod h1:9SRiXfLldPw9lEEuSsapMtvT8j/h1JyFFapbtybwKvU= +k8s.io/api v0.35.4 h1:P7nFYKl5vo9AGUp1Z+Pmd3p2tA7bX2wbFWCvDeRv988= +k8s.io/api v0.35.4/go.mod h1:yl4lqySWOgYJJf9RERXKUwE9g2y+CkuwG+xmcOK8wXU= +k8s.io/apiextensions-apiserver v0.35.4 h1:HeP+Upp7ItdvnyGmub0yoix+2z5+ev4M5cE5TCgtOUU= +k8s.io/apiextensions-apiserver v0.35.4/go.mod h1:ogQlk+stIE8mnoRthSYCwlOS12fVqgWFiErMwPaXA7c= +k8s.io/apimachinery v0.35.4 h1:xtdom9RG7e+yDp71uoXoJDWEE2eOiHgeO4GdBzwWpds= +k8s.io/apimachinery v0.35.4/go.mod h1:NNi1taPOpep0jOj+oRha3mBJPqvi0hGdaV8TCqGQ+cc= +k8s.io/apiserver v0.35.4 h1:vtuFqNFmF9bPRdHDL2lpK6qCTPWDreZJL4LRPwVM6ho= +k8s.io/apiserver v0.35.4/go.mod h1:JnBcb+J8kFXKpZkgcbcUnPBBHi4qgBii1I7dLxFY/oo= +k8s.io/client-go v0.35.4 h1:DN6fyaGuzK64UvnKO5fOA6ymSjvfGAnCAHAR0C66kD8= +k8s.io/client-go v0.35.4/go.mod h1:2Pg9WpsS4NeOpoYTfHHfMxBG8zFMSAUi4O/qoiJC3nY= +k8s.io/component-base v0.35.4 h1:6n1tNJ87johN0Hif0Fs8K2GMthsaUwMqCebUDLYyv7U= +k8s.io/component-base v0.35.4/go.mod h1:qaDJgz5c1KYKla9occFmlJEfPpkuA55s90G509R+PeY= +k8s.io/component-helpers v0.35.4 h1:WJM/+fAeeJTAqxPDxgH0aB0q7t8DP+AbV5WkRkOoxYA= +k8s.io/component-helpers v0.35.4/go.mod h1:mE7X9mnMQEX6IbZejdMlWvCx3EPVt1/9PhH/FW0XHDI= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.22.3 h1:I7mfqz/a/WdmDCEnXmSPm8/b/yRTy6JsKKENTijTq8Y= -sigs.k8s.io/controller-runtime v0.22.3/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/controller-runtime v0.23.1 h1:TjJSM80Nf43Mg21+RCy3J70aj/W6KyvDtOlpKf+PupE= +sigs.k8s.io/controller-runtime v0.23.1/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/deploy-wcp.sh b/hack/deploy-wcp.sh index eae403558..96940f465 100755 --- a/hack/deploy-wcp.sh +++ b/hack/deploy-wcp.sh @@ -208,7 +208,8 @@ function sv_get_vip_and_password() { fatal "No Supervisor cluster found" fi - SV_VIP=$(echo "$sv" | awk '/^IP:/ {print $2}') + # In dual stack, the both IPs are comma separated. Take the first one. + SV_VIP=$(echo "$sv" | awk '/^IP:/ {print $2}' | cut -d',' -f1) SV_PASSWORD=$(echo "$sv" | awk '/^PWD:/ {print $2}') log "Supervisor VIP: $SV_VIP Password: $SV_PASSWORD" diff --git a/hack/new-schema-version.py b/hack/new-schema-version.py index 0a1e9ec13..6faa420e9 100755 --- a/hack/new-schema-version.py +++ b/hack/new-schema-version.py @@ -336,8 +336,7 @@ def _generate_conversion_webhooks_go( for t in root_types: lines.extend( [ - f"\tif err := ctrl.NewWebhookManagedBy(mgr).", - f"\t\tFor(&{import_alias}.{t}{{}}).", + f"\tif err := ctrl.NewWebhookManagedBy(mgr, &{import_alias}.{t}{{}}).", "\t\tComplete(); err != nil {", "", "\t\treturn err", diff --git a/hack/setup-testbed-env.sh b/hack/setup-testbed-env.sh new file mode 100755 index 000000000..c04097c95 --- /dev/null +++ b/hack/setup-testbed-env.sh @@ -0,0 +1,280 @@ +#!/usr/bin/env bash +# +# Setup testbed environment for vm-operator E2E tests +# +# This script fetches testbed information from local files or URLs and prepares the environment +# for running vm-operator E2E tests by setting up kubeconfig and environment variables. +# +# # Local file: +# source ./hack/setup-testbed-env.sh ./testbedinfo.json +# +# # Remote URL: +# source ./hack/setup-testbed-env.sh https://example.com/testbed.json +# +# # Enable E2E kubeconfig setup by copying kubeconfig to ~/.kube/wcp-config +# # which is used by the E2E test to access the Supervisor cluster. +# source ./hack/setup-testbed-env.sh ./testbedinfo.json --e2e +# + +set -u + +SCRIPT_SOURCED=true +# Function to handle exits gracefully (return if sourced, exit if executed) +script_exit() { + local exit_code=${1:-1} + if [ "$SCRIPT_SOURCED" = "true" ]; then + return "$exit_code" + else + exit "$exit_code" + fi +} + +# Parse command line arguments +TESTBED_SOURCE="" +ENABLE_E2E_KUBECONFIG="" + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --e2e) + ENABLE_E2E_KUBECONFIG="true" + shift + ;; + --help|-h) + echo "Usage: $0 TESTBED_SOURCE [OPTIONS]" + echo "" + echo "Arguments:" + echo " TESTBED_SOURCE Path to local testbedinfo.json or HTTP(S) URL (required)" + echo "" + echo "Options:" + echo " --e2e Enable E2E kubeconfig setup (copy kubeconfig to wcp-config)" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 ./testbedinfo.json --e2e" + echo " $0 https://example.com/testbed.json" + echo " source $0 ./testbedinfo.json --e2e" + echo " source $0 https://example.com/testbed.json" + shift + script_exit 0 + ;; + -*) + echo "Unknown option: $1" + echo "Use --help for usage information" + script_exit 1 + ;; + *) + TESTBED_SOURCE="$1" + shift + ;; + esac + done +} + +# Parse arguments (works for both sourced and executed) +parse_args "$@" + +echo "=== VM Operator E2E Testbed Environment Setup ===" + +# Check that testbed source is provided +if [ -z "$TESTBED_SOURCE" ]; then + echo "Error: Testbed source is required" + echo "Usage: $0 TESTBED_SOURCE [--e2e]" + echo "Examples:" + echo " $0 ./testbedinfo.json --e2e" + echo " $0 https://example.com/testbed.json" + script_exit 1 +fi + +# Determine data source and fetch testbed data +if [[ "$TESTBED_SOURCE" =~ ^https?:// ]]; then + echo "Fetching testbed data from URL: ${TESTBED_SOURCE}" + + if ! testbed_data=$(curl -fsSL "$TESTBED_SOURCE" 2>/dev/null); then + echo "Error: Failed to fetch testbed data from ${TESTBED_SOURCE}" + script_exit 1 + fi + + echo "Successfully fetched testbed data from URL" +else + echo "Using local testbed file: ${TESTBED_SOURCE}" + + if [ ! -f "$TESTBED_SOURCE" ]; then + echo "Error: Local testbed file not found: ${TESTBED_SOURCE}" + script_exit 1 + fi + + if ! testbed_data=$(cat "$TESTBED_SOURCE" 2>/dev/null); then + echo "Error: Failed to read local testbed file: ${TESTBED_SOURCE}" + script_exit 1 + fi + + echo "Successfully loaded local testbed data" +fi + +# Parse the testbed data - handle both UTS format and raw vCenter format +# UTS format has deliverable_blob, raw format is direct JSON +if echo "$testbed_data" | jq -e '.deliverable_blob' >/dev/null 2>&1; then + echo "Detected UTS deliverable_blob format, extracting data..." + testbed_info=$(echo "$testbed_data" | jq -r '.deliverable_blob') +else + echo "Detected direct testbed format" + testbed_info="$testbed_data" +fi + +# Extract vCenter connection details - try multiple JSON path patterns +# Pattern 1: WCP testbed format with nested vc array +if echo "$testbed_info" | jq -e '.vc[0]' >/dev/null 2>&1; then + echo "Using vc[0] format for vCenter details" + export VC_URL=$(echo "$testbed_info" | jq -r '.vc[0].ip4 // .vc[0].ip // .vc[0].vcenter_ip') + export VC_ROOT_PASSWORD=$(echo "$testbed_info" | jq -r '.vc[0].password // .vc[0].vcenter_password') + export VC_ROOT_USERNAME=$(echo "$testbed_info" | jq -r '.vc[0].username // .vc[0].vcenter_username // "administrator@vsphere.local"') +# Pattern 2: Direct format with vcenter_ prefixed fields +elif echo "$testbed_info" | jq -e '.vcenter_ip' >/dev/null 2>&1; then + echo "Using direct vcenter_ format for vCenter details" + export VC_URL=$(echo "$testbed_info" | jq -r '.vcenter_ip') + export VC_ROOT_PASSWORD=$(echo "$testbed_info" | jq -r '.vcenter_password') + export VC_ROOT_USERNAME=$(echo "$testbed_info" | jq -r '.vcenter_username // "administrator@vsphere.local"') +# Pattern 3: Simple format (for testing/fallback) +else + echo "Using simple format for vCenter details" + export VC_URL=$(echo "$testbed_info" | jq -r '.vc_ip // .vcenter_ip // .testbed_ip') + export VC_ROOT_PASSWORD=$(echo "$testbed_info" | jq -r '.vc_password // .vcenter_password // .password') + export VC_ROOT_USERNAME=$(echo "$testbed_info" | jq -r '.vc_username // .vcenter_username // .username // "administrator@vsphere.local"') +fi + +# Validate required variables are set and not null +if [ -z "${VC_URL}" ] || [ "${VC_URL}" = "null" ]; then + echo "Error: Could not extract VC_URL from testbed data" + echo "Available fields:" + echo "$testbed_info" | jq -r 'keys[]' 2>/dev/null || echo "Invalid JSON format" + script_exit 1 +fi + +if [ -z "${VC_ROOT_PASSWORD}" ] || [ "${VC_ROOT_PASSWORD}" = "null" ]; then + echo "Error: Could not extract VC_ROOT_PASSWORD from testbed data" + script_exit 1 +fi + +if [ -z "${VC_ROOT_USERNAME}" ] || [ "${VC_ROOT_USERNAME}" = "null" ]; then + export VC_ROOT_USERNAME="administrator@vsphere.local" +fi + +echo "Configuration loaded successfully:" +echo " VC_URL: ${VC_URL}" +echo " VC_ROOT_USERNAME: ${VC_ROOT_USERNAME}" + +# Set up common environment variables for vm-operator E2E tests +export VCSA_IP="${VC_URL}" +export SSH_PASSWORD="${VC_ROOT_PASSWORD}" +export SSH_USERNAME="${VC_ROOT_USERNAME}" +export GOVC_PASSWORD="${SSH_PASSWORD}" +export VCSA_PASSWORD="${SSH_PASSWORD}" + +# Detect networking type from testbed data +NETWORKING_TYPE=$(echo "$testbed_info" | jq -r '.networking // "vds"') +export NETWORK="${NETWORKING_TYPE}" + +# Set test configuration defaults +export TEST_SKIP="" # Tests to skip +export TEST_TAGS="vmservice" # Test tags to run +export TEST_FOCUS="${TEST_FOCUS:-}" # Test focus pattern + +echo "Standard environment variables set for vm-operator E2E tests" + +# Get WCP/Supervisor cluster credentials +echo "Retrieving WCP/Supervisor cluster credentials..." + +# Install sshpass if not available (in case the container doesn't have it) +if ! command -v sshpass >/dev/null 2>&1; then + echo "Installing sshpass..." + if command -v apt-get >/dev/null 2>&1; then + apt-get update -qq && apt-get install -y -qq sshpass + elif command -v yum >/dev/null 2>&1; then + yum install -y -q sshpass + else + echo "Warning: sshpass not found and package manager not detected" + echo "WCP credential extraction may fail" + fi +fi + +# Extract WCP credentials from vCenter +if ! wcp_info=$(sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 "${SSH_USERNAME}@${VCSA_IP}" "/usr/lib/vmware-wcp/decryptK8Pwd.py" 2>/dev/null); then + echo "Warning: Failed to extract WCP credentials via SSH" + echo "This may be expected if WCP is not fully enabled yet" + # Don't exit - allow tests to run without kubeconfig if needed + WCP_PASSWORD="" + WCP_IP="" +fi + +if [ -n "$wcp_info" ]; then + WCP_PASSWORD="$(echo "$wcp_info" | grep "PWD:" | sed -E "s/.*PWD: ([^ ]*).*/\1/")" + WCP_IP="$(echo "$wcp_info" | grep "IP:" | sed -E "s/.*IP: ([^ ]*).*/\1/")" + + echo "WCP Supervisor IP: ${WCP_IP}, Password: [REDACTED]" + + # Set up kubeconfig for supervisor cluster access + echo "Setting up kubeconfig for Supervisor cluster..." + + # Create .kube directory if it doesn't exist + mkdir -p "$HOME/.kube" + wcp_kubeconfig_destination="$HOME/.kube/wcp-config" + kubeconfig_destination="$HOME/.kube/${VCSA_IP}.kubeconfig" + + echo "Downloading kubeconfig from supervisor cluster to ${kubeconfig_destination}" + + # Copy kubeconfig from supervisor cluster + if sshpass -p "${WCP_PASSWORD}" scp -o StrictHostKeyChecking=no -o ConnectTimeout=30 "root@${WCP_IP}:~/.kube/config" "${kubeconfig_destination}" 2>/dev/null; then + export KUBECONFIG="${kubeconfig_destination}" + echo "Kubeconfig exported to: ${kubeconfig_destination}" + + # Fix the server URL in kubeconfig (replace 127.0.0.1 with actual IP) + if command -v sed >/dev/null 2>&1; then + case "${OSTYPE:-$(uname -s)}" in + darwin*|Darwin) + # macOS sed + sed -i "" "s/127.0.0.1/${WCP_IP}/g" "${kubeconfig_destination}" + ;; + *) + # Linux sed + sed -i "s/127.0.0.1/${WCP_IP}/g" "${kubeconfig_destination}" + ;; + esac + echo "Kubeconfig server URL updated to use ${WCP_IP}" + fi + + # Only copy kubeconfig to wcp-config if --e2e flag is enabled + if [ "$ENABLE_E2E_KUBECONFIG" = "true" ]; then + echo "Copying kubeconfig to ${wcp_kubeconfig_destination} (--e2e flag enabled)" + cp "${kubeconfig_destination}" "${wcp_kubeconfig_destination}" + else + echo "Skipping kubeconfig copy to ${wcp_kubeconfig_destination} (use --e2e flag to enable)" + fi + + else + echo "Warning: Failed to copy kubeconfig from supervisor cluster" + echo "Tests requiring kubectl access may fail" + fi +else + echo "Warning: No WCP credentials found - supervisor cluster may not be ready" +fi + +# Export additional variables that vm-operator tests might expect +export SUPERVISOR_CLUSTER_IP="${WCP_IP:-}" +export SUPERVISOR_CLUSTER_PASSWORD="${WCP_PASSWORD:-}" + +echo "" +echo "=== Environment Setup Complete ===" +echo "Data source: ${TESTBED_SOURCE}" +echo "E2E kubeconfig: $(if [ "$ENABLE_E2E_KUBECONFIG" = "true" ]; then echo "Enabled"; else echo "Disabled (use --e2e to enable)"; fi)" +echo "" +echo "The following variables are now available:" +echo " VC_URL/VCSA_IP: ${VC_URL}" +echo " VC_ROOT_USERNAME/SSH_USERNAME: ${VC_ROOT_USERNAME}" +echo " NETWORK: ${NETWORK}" +echo " KUBECONFIG: ${KUBECONFIG:-not set}" +echo " SUPERVISOR_CLUSTER_IP: ${SUPERVISOR_CLUSTER_IP:-not set}" +echo "" +echo "You can now run vm-operator E2E tests with these credentials." +echo "Example: make e2e-smoke" +echo "" \ No newline at end of file diff --git a/hack/tools/go.mod b/hack/tools/go.mod index 01a2ac076..0a7ab066d 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -1,16 +1,16 @@ module github.com/vmware-tanzu/vm-operator/hack/tools -go 1.26.1 +go 1.26.2 // The version of Ginkgo must match the one from ../../go.mod. -require github.com/onsi/ginkgo/v2 v2.23.4 +require github.com/onsi/ginkgo/v2 v2.27.2 require ( github.com/AlekSi/gocov-xml v1.1.0 github.com/axw/gocov v1.1.0 github.com/elastic/crd-ref-docs v0.0.12 github.com/golangci/golangci-lint/v2 v2.1.6 - golang.org/x/tools v0.32.0 + golang.org/x/tools v0.36.0 golang.org/x/vuln v1.1.1 k8s.io/code-generator v0.31.0 sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20240820183333-e6c3d139d2b6 @@ -33,7 +33,7 @@ require ( github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect - github.com/Masterminds/semver/v3 v3.3.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect github.com/alecthomas/chroma/v2 v2.17.2 // indirect @@ -84,7 +84,7 @@ require ( github.com/ghostiam/protogetter v0.3.15 // indirect github.com/go-critic/go-critic v0.13.0 // indirect github.com/go-errors/errors v1.4.2 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -101,7 +101,7 @@ require ( github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect github.com/gobuffalo/flect v1.0.2 // indirect github.com/gobwas/glob v0.2.3 // indirect - github.com/goccy/go-yaml v1.11.3 // indirect + github.com/goccy/go-yaml v1.18.0 // indirect github.com/gofrs/flock v0.12.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -242,15 +242,16 @@ require ( go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.37.0 // indirect + golang.org/x/crypto v0.41.0 // indirect golang.org/x/exp/typeparams v0.0.0-20250210185358-939b2ce775ac // indirect - golang.org/x/mod v0.24.0 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/sync v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/mod v0.27.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/tools/go/expect v0.1.1-deprecated // indirect + golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect + google.golang.org/protobuf v1.36.7 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/hack/tools/go.sum b/hack/tools/go.sum index 3455fb65d..20aa1cb55 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -24,8 +24,8 @@ github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJ github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= -github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= @@ -141,12 +141,18 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.15 h1:1KF5sXel0HE48zh1/vn0Loiw25A9ApyseLzQuif1mLY= github.com/ghostiam/protogetter v0.3.15/go.mod h1:WZ0nw9pfzsgxuRsPOFQomgDVSWtDLJRfQJEhsGbmQMA= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-critic/go-critic v0.13.0 h1:kJzM7wzltQasSUXtYyTl6UaPVySO6GkaR1thFnJ6afY= github.com/go-critic/go-critic v0.13.0/go.mod h1:M/YeuJ3vOCQDnP2SU+ZhjgRzwzcBW87JqLpMJLrZDLI= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -156,12 +162,6 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= @@ -193,8 +193,8 @@ github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I= -github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -287,6 +287,8 @@ github.com/jjti/go-spancheck v0.6.4 h1:Tl7gQpYf4/TMU7AT84MN83/6PutY21Nb9fuQjFTpR github.com/jjti/go-spancheck v0.6.4/go.mod h1:yAEYdKJ2lRkDA8g7X+oKUHXOWVAXSBJRv04OhF+QUjk= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julz/importas v0.2.0 h1:y+MJN/UdL63QbFJHws9BVC5RpA2iq0kpjrFajTGivjQ= @@ -322,8 +324,6 @@ github.com/ldez/tagliatelle v0.7.1 h1:bTgKjjc2sQcsgPiT902+aadvMjCeMHrY7ly2XKFORI github.com/ldez/tagliatelle v0.7.1/go.mod h1:3zjxUpsNB2aEZScWiZTHrAXOl1x25t3cRmzfK1mlo2I= github.com/ldez/usetesting v0.4.3 h1:pJpN0x3fMupdTf/IapYjnkhiY1nSTN+pox1/GyBRw3k= github.com/ldez/usetesting v0.4.3/go.mod h1:eEs46T3PpQ+9RgN9VjpY6qWdiw2/QmfiDeWmdZdrjIQ= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leonklingele/grouper v1.1.2 h1:o1ARBDLOmmasUaNDesWqWCIFH3u7hoFlM84YrjT3mIY= github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -340,6 +340,8 @@ github.com/maratori/testableexamples v1.0.0 h1:dU5alXRrD8WKSjOUnmJZuzdxWOEQ57+7s github.com/maratori/testableexamples v1.0.0/go.mod h1:4rhjL1n20TUTT4vdh3RDqSizKLyXp7K2u6HgraZCGzE= github.com/maratori/testpackage v1.1.1 h1:S58XVV5AD7HADMmD0fNnziNHqKvSdDuEKdPD1rNTU04= github.com/maratori/testpackage v1.1.1/go.mod h1:s4gRK/ym6AMrqpOa/kEbQTV4Q4jb7WeLZzVhVVVOQMc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/godox v1.1.0 h1:W5mqwbyWrwZv6OQ5Z1a/DHGMOvXYCBP3+Ht7KMoJhq4= github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= @@ -351,6 +353,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/mgechev/revive v1.9.0 h1:8LaA62XIKrb8lM6VsBSQ92slt/o92z5+hTw3CmrvSrM= github.com/mgechev/revive v1.9.0/go.mod h1:LAPq3+MgOf7GcL5PlWIkHb0PT7XH4NuC2LdWymhb9Mo= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -388,10 +392,10 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= -github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= -github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= -github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= @@ -508,6 +512,14 @@ github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpR github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/tetafro/godot v1.5.1 h1:PZnjCol4+FqaEzvZg5+O8IY2P3hfY9JzRBNPv1pEDS4= github.com/tetafro/godot v1.5.1/go.mod h1:cCdPtEndkmqqrhiCfkmxDodMQJ/f3L1BCNskCUZdTwk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 h1:9LPGD+jzxMlnk5r6+hJnar67cgpDIz/iyD+rfl5r2Vk= github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= github.com/timonwong/loggercheck v0.11.0 h1:jdaMpYBl+Uq9mWPXv1r8jc5fC3gyXx4/WGwTnnNKn4M= @@ -565,14 +577,16 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= @@ -590,8 +604,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -608,8 +622,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -619,8 +633,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -643,8 +657,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -663,8 +677,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -686,18 +700,20 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= -golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/vuln v1.1.1 h1:4nYQg4OSr7uYQMtjuuYqLAEVuTjY4k/CPMYqvv5OPcI= golang.org/x/vuln v1.1.1/go.mod h1:hNgE+SKMSp2wHVUpW0Ow2ejgKpNJePdML+4YjxrVxik= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/main.go b/main.go index 4c27f3d01..0ea1a729c 100644 --- a/main.go +++ b/main.go @@ -292,6 +292,11 @@ func initFlags() { "webhook-secret-volume-mount-path", defaultConfig.WebhookSecretVolumeMountPath, "The filesystem path to which the webhook secret is mounted.") + flag.StringVar( + &managerOpts.WebhookBindAddress, + "webhook-bind-address", + pkgmgr.DefaultWebhookBindAddress, + "The IP address the webhook server binds to.") flag.BoolVar( &managerOpts.ContainerNode, "container-node", diff --git a/pkg/builder/builder_suite_test.go b/pkg/builder/builder_suite_test.go index 6e5ca3c72..1c32d8ffd 100644 --- a/pkg/builder/builder_suite_test.go +++ b/pkg/builder/builder_suite_test.go @@ -16,7 +16,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" "k8s.io/klog/v2" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -39,7 +39,7 @@ type fakeManager struct { scheme *runtime.Scheme } -func (f fakeManager) GetEventRecorderFor(name string) record.EventRecorder { +func (f fakeManager) GetEventRecorder(name string) events.EventRecorder { return nil } diff --git a/pkg/builder/mutating_webhook.go b/pkg/builder/mutating_webhook.go index f2aa3b040..1af7156cf 100644 --- a/pkg/builder/mutating_webhook.go +++ b/pkg/builder/mutating_webhook.go @@ -63,8 +63,8 @@ func NewMutatingWebhook( } webhookNameShort := generateMutateName(webhookName, mutator.For()) - webhookPath := "/" + webhookNameShort webhookNameLong := fmt.Sprintf("%s/%s/%s", ctx.Namespace, ctx.Name, webhookNameShort) + webhookPath := "/" + webhookNameShort // Build the WebhookContext. webhookContext := &pkgctx.WebhookContext{ @@ -72,7 +72,7 @@ func NewMutatingWebhook( Name: webhookNameShort, Namespace: ctx.Namespace, ServiceAccountName: ctx.ServiceAccountName, - Recorder: record.New(mgr.GetEventRecorderFor(webhookNameLong)), + Recorder: record.New(mgr.GetEventRecorder(webhookNameLong)), Logger: ctx.Logger.WithName(webhookNameShort), EnableWebhookClientVerification: ctx.EnableWebhookClientVerification, } diff --git a/pkg/builder/validating_webhook.go b/pkg/builder/validating_webhook.go index 0d1bd93b3..4a3e40d3c 100644 --- a/pkg/builder/validating_webhook.go +++ b/pkg/builder/validating_webhook.go @@ -80,7 +80,7 @@ func NewValidatingWebhook( Name: webhookNameShort, Namespace: ctx.Namespace, ServiceAccountName: ctx.ServiceAccountName, - Recorder: record.New(mgr.GetEventRecorderFor(webhookNameLong)), + Recorder: record.New(mgr.GetEventRecorder(webhookNameLong)), Logger: ctx.Logger.WithName(webhookNameShort), EnableWebhookClientVerification: ctx.EnableWebhookClientVerification, } diff --git a/pkg/config/capabilities/capabilities.go b/pkg/config/capabilities/capabilities.go index 9cb2413e7..29be47033 100644 --- a/pkg/config/capabilities/capabilities.go +++ b/pkg/config/capabilities/capabilities.go @@ -102,11 +102,15 @@ const ( // for mutability of Storage Policy via Volume Attributes Class (VAC). CapabilityKeyStoragePolicyMutability = "supports_VM_PVC_storage_policy_mutability" - // CapabilityKeyVMExtraConfig is the name of the capability key defined + // CapabilityKeyVlanSubinterface is the name of capability key defined in the + // Supervisor capabilities CRD for the VM service supports VLAN Sub-Interfaces. + CapabilityKeyVlanSubinterface = "supports_vm_service_vlan_subinterface" + + // CapabilityKeyTelcoVMServiceAPI is the name of the capability key defined // in the Supervisor capabilities CRD for the VM Service's support for - // VM extraConfig configuration: spec.advanced first-class fields, extraConfig + // Telco VM advanced properties: spec.advanced first-class fields, extraConfig // fallback, and per-NIC VMXNet3 tuning properties. - CapabilityKeyVMExtraConfig = "supports_vm_service_vm_extra_config" + CapabilityKeyTelcoVMServiceAPI = "supports_telco_vm_service_api" ) var ( @@ -271,8 +275,10 @@ func updateCapabilitiesFeaturesFromCRD( fs.VMAffinityDuringExecution = capStatus.Activated case CapabilityKeyStoragePolicyMutability: fs.StoragePolicyMutability = capStatus.Activated - case CapabilityKeyVMExtraConfig: - fs.VMExtraConfig = capStatus.Activated + case CapabilityKeyVlanSubinterface: + fs.VMVlanSubinterface = capStatus.Activated + case CapabilityKeyTelcoVMServiceAPI: + fs.TelcoVMServiceAPI = capStatus.Activated } } diff --git a/pkg/config/capabilities/capabilities_test.go b/pkg/config/capabilities/capabilities_test.go index 47672723a..80e97eae7 100644 --- a/pkg/config/capabilities/capabilities_test.go +++ b/pkg/config/capabilities/capabilities_test.go @@ -177,6 +177,9 @@ var _ = Describe("UpdateCapabilities", func() { capabilities.CapabilityKeyStoragePolicyMutability: { Activated: true, }, + capabilities.CapabilityKeyVlanSubinterface: { + Activated: true, + }, } Expect(client.Status().Patch(ctx, &obj, objPatch)).To(Succeed()) }) @@ -198,6 +201,7 @@ var _ = Describe("UpdateCapabilities", func() { config.Features.VSpherePolicies = true config.Features.VMAffinityDuringExecution = true config.Features.StoragePolicyMutability = true + config.Features.VMVlanSubinterface = true }) }) Specify("capabilities did not change", func() { @@ -248,6 +252,9 @@ var _ = Describe("UpdateCapabilities", func() { Specify(capabilities.CapabilityKeyStoragePolicyMutability, func() { Expect(pkgcfg.FromContext(ctx).Features.StoragePolicyMutability).To(BeTrue()) }) + Specify(capabilities.CapabilityKeyVlanSubinterface, func() { + Expect(pkgcfg.FromContext(ctx).Features.VMVlanSubinterface).To(BeTrue()) + }) }) When("the capabilities are different", func() { @@ -299,6 +306,9 @@ var _ = Describe("UpdateCapabilities", func() { Specify(capabilities.CapabilityKeyStoragePolicyMutability, func() { Expect(pkgcfg.FromContext(ctx).Features.StoragePolicyMutability).To(BeTrue()) }) + Specify(capabilities.CapabilityKeyVlanSubinterface, func() { + Expect(pkgcfg.FromContext(ctx).Features.VMVlanSubinterface).To(BeTrue()) + }) }) }) @@ -358,6 +368,9 @@ var _ = Describe("UpdateCapabilities", func() { capabilities.CapabilityKeyStoragePolicyMutability: { Activated: false, }, + capabilities.CapabilityKeyVlanSubinterface: { + Activated: false, + }, } Expect(client.Status().Patch(ctx, &obj, objPatch)).To(Succeed()) }) @@ -410,6 +423,9 @@ var _ = Describe("UpdateCapabilities", func() { Specify(capabilities.CapabilityKeyStoragePolicyMutability, func() { Expect(pkgcfg.FromContext(ctx).Features.StoragePolicyMutability).To(BeFalse()) }) + Specify(capabilities.CapabilityKeyVlanSubinterface, func() { + Expect(pkgcfg.FromContext(ctx).Features.VMVlanSubinterface).To(BeFalse()) + }) }) When("the capabilities are different", func() { @@ -424,6 +440,7 @@ var _ = Describe("UpdateCapabilities", func() { config.Features.VMSharedDisks = true config.Features.GuestCustomizationVCDParity = true config.Features.StoragePolicyMutability = true + config.Features.VMVlanSubinterface = true }) }) Specify("capabilities changed", func() { @@ -474,6 +491,9 @@ var _ = Describe("UpdateCapabilities", func() { Specify(capabilities.CapabilityKeyStoragePolicyMutability, func() { Expect(pkgcfg.FromContext(ctx).Features.StoragePolicyMutability).To(BeFalse()) }) + Specify(capabilities.CapabilityKeyVlanSubinterface, func() { + Expect(pkgcfg.FromContext(ctx).Features.VMVlanSubinterface).To(BeFalse()) + }) }) }) }) @@ -781,6 +801,19 @@ var _ = Describe("UpdateCapabilitiesFeatures", func() { Expect(pkgcfg.FromContext(ctx).Features.StoragePolicyMutability).To(BeTrue()) }) }) + Context(capabilities.CapabilityKeyVlanSubinterface, func() { + BeforeEach(func() { + Expect(pkgcfg.FromContext(ctx).Features.VMVlanSubinterface).To(BeFalse()) + obj.Status.Supervisor[capabilities.CapabilityKeyVlanSubinterface] = capv1.CapabilityStatus{ + Activated: true, + } + }) + Specify("Enabled", func() { + Expect(ok).To(BeTrue()) + Expect(diff).To(Equal("VMVlanSubinterface=true")) + Expect(pkgcfg.FromContext(ctx).Features.VMVlanSubinterface).To(BeTrue()) + }) + }) }) }) @@ -842,6 +875,9 @@ var _ = Describe("WouldUpdateCapabilitiesFeatures", func() { capabilities.CapabilityKeyStoragePolicyMutability: { Activated: true, }, + capabilities.CapabilityKeyVlanSubinterface: { + Activated: true, + }, } ok, diff = false, "" @@ -870,6 +906,7 @@ var _ = Describe("WouldUpdateCapabilitiesFeatures", func() { config.Features.VSpherePolicies = true config.Features.VMAffinityDuringExecution = true config.Features.StoragePolicyMutability = true + config.Features.VMVlanSubinterface = true }) }) Specify("capabilities did not change", func() { @@ -921,6 +958,9 @@ var _ = Describe("WouldUpdateCapabilitiesFeatures", func() { Specify(capabilities.CapabilityKeyStoragePolicyMutability, func() { Expect(pkgcfg.FromContext(ctx).Features.StoragePolicyMutability).To(BeTrue()) }) + Specify(capabilities.CapabilityKeyVlanSubinterface, func() { + Expect(pkgcfg.FromContext(ctx).Features.VMVlanSubinterface).To(BeTrue()) + }) }) When("the capabilities are different", func() { @@ -939,11 +979,12 @@ var _ = Describe("WouldUpdateCapabilitiesFeatures", func() { config.Features.VSpherePolicies = false config.Features.VMAffinityDuringExecution = false config.Features.StoragePolicyMutability = false + config.Features.VMVlanSubinterface = false }) }) Specify("capabilities changed", func() { Expect(ok).To(BeTrue()) - Expect(diff).To(Equal("BringYourOwnEncryptionKey=true,GuestCustomizationVCDParity=true,ImmutableClasses=true,InventoryContentLibrary=true,MutableNetworks=true,StoragePolicyMutability=true,TKGMultipleCL=true,VMAffinityDuringExecution=true,VMGroups=true,VMPlacementPolicies=true,VMSharedDisks=true,VMSnapshots=true,VMWaitForFirstConsumerPVC=true,VSpherePolicies=true,WorkloadDomainIsolation=true")) + Expect(diff).To(Equal("BringYourOwnEncryptionKey=true,GuestCustomizationVCDParity=true,ImmutableClasses=true,InventoryContentLibrary=true,MutableNetworks=true,StoragePolicyMutability=true,TKGMultipleCL=true,VMAffinityDuringExecution=true,VMGroups=true,VMPlacementPolicies=true,VMSharedDisks=true,VMSnapshots=true,VMVlanSubinterface=true,VMWaitForFirstConsumerPVC=true,VSpherePolicies=true,WorkloadDomainIsolation=true")) }) Specify(capabilities.CapabilityKeyBringYourOwnKeyProvider, func() { Expect(pkgcfg.FromContext(ctx).Features.BringYourOwnEncryptionKey).To(BeFalse()) @@ -990,6 +1031,9 @@ var _ = Describe("WouldUpdateCapabilitiesFeatures", func() { Specify(capabilities.CapabilityKeyStoragePolicyMutability, func() { Expect(pkgcfg.FromContext(ctx).Features.StoragePolicyMutability).To(BeFalse()) }) + Specify(capabilities.CapabilityKeyVlanSubinterface, func() { + Expect(pkgcfg.FromContext(ctx).Features.VMVlanSubinterface).To(BeFalse()) + }) }) }) }) diff --git a/pkg/config/config.go b/pkg/config/config.go index 97ac7f84a..c91c0aeb9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -201,7 +201,8 @@ type FeatureStates struct { AllDisksArePVCs bool VMAffinityDuringExecution bool StoragePolicyMutability bool - VMExtraConfig bool + VMVlanSubinterface bool + TelcoVMServiceAPI bool } type InstanceStorage struct { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 24c16cd6f..6c3d16aa4 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -12,6 +12,12 @@ import ( ) var _ = Describe("Config", func() { + Describe("Default", func() { + It("Should bind the profiler to loopback only", func() { + Expect(pkgcfg.Default().ProfilerAddr).To(Equal("127.0.0.1:8073")) + }) + }) + Describe("GetMaxDeployThreadsOnProvider", func() { When("MaxDeployThreadsOnProvider == 0", func() { It("Should return MaxDeployThreadsOnProvider", func() { diff --git a/pkg/config/default.go b/pkg/config/default.go index 534df360a..5bccff881 100644 --- a/pkg/config/default.go +++ b/pkg/config/default.go @@ -52,7 +52,7 @@ func Default() Config { PodName: defaultPrefix + "controller-manager", PodNamespace: defaultPrefix + "system", PodServiceAccountName: defaultPrefix + "service-account", - ProfilerAddr: ":8073", + ProfilerAddr: "127.0.0.1:8073", RateLimitBurst: 1000, RateLimitQPS: 500, SyncPeriod: 30 * time.Minute, diff --git a/pkg/context/fake/fake_controller_manager_context.go b/pkg/context/fake/fake_controller_manager_context.go index b047f0930..097893504 100644 --- a/pkg/context/fake/fake_controller_manager_context.go +++ b/pkg/context/fake/fake_controller_manager_context.go @@ -5,7 +5,7 @@ package fake import ( - clientrecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config" @@ -29,7 +29,7 @@ func NewControllerManagerContext() *pkgctx.ControllerManagerContext { ServiceAccountName: ServiceAccountName, LeaderElectionNamespace: LeaderElectionNamespace, LeaderElectionID: LeaderElectionID, - Recorder: record.New(clientrecord.NewFakeRecorder(1024)), + Recorder: record.New(events.NewFakeRecorder(1024)), VMProvider: providerfake.NewVMProvider(), } } diff --git a/pkg/context/fake/fake_webhook_context.go b/pkg/context/fake/fake_webhook_context.go index 7f9b58330..1fb1b4f30 100644 --- a/pkg/context/fake/fake_webhook_context.go +++ b/pkg/context/fake/fake_webhook_context.go @@ -5,7 +5,7 @@ package fake import ( - clientrecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/record" @@ -19,6 +19,6 @@ func NewWebhookContext(ctx *pkgctx.ControllerManagerContext) *pkgctx.WebhookCont Name: WebhookName, Namespace: ctx.Namespace, Logger: ctx.Logger.WithName(WebhookName), - Recorder: record.New(clientrecord.NewFakeRecorder(1024)), + Recorder: record.New(events.NewFakeRecorder(1024)), } } diff --git a/pkg/crd/crd.go b/pkg/crd/crd.go index 3feab56a0..fbad9d934 100644 --- a/pkg/crd/crd.go +++ b/pkg/crd/crd.go @@ -317,7 +317,7 @@ func Install( //nolint:gocyclo } } - if !features.VMExtraConfig { + if !features.TelcoVMServiceAPI { if err := removeFields( ctx, k, diff --git a/pkg/crd/crd_test.go b/pkg/crd/crd_test.go index 3e39eb852..b81448552 100644 --- a/pkg/crd/crd_test.go +++ b/pkg/crd/crd_test.go @@ -411,7 +411,7 @@ var _ = Describe("Install", func() { When("VM extra config capability is enabled", func() { BeforeEach(func() { pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { - config.Features.VMExtraConfig = true + config.Features.TelcoVMServiceAPI = true }) }) It("should get the expected crds", func() { @@ -554,7 +554,7 @@ var _ = Describe("Install", func() { config.Features.VSpherePolicies = true config.Features.BringYourOwnEncryptionKey = true config.Features.GuestCustomizationVCDParity = true - config.Features.VMExtraConfig = true + config.Features.TelcoVMServiceAPI = true }) }) It("should get the expected crds", func() { diff --git a/pkg/manager/constants.go b/pkg/manager/constants.go index d6731284f..666379317 100644 --- a/pkg/manager/constants.go +++ b/pkg/manager/constants.go @@ -59,6 +59,10 @@ const ( //nolint:gosec DefaultWebhookSecretVolumeMountPath = "/etc/vmware/wcp/webhook-certs" + // DefaultWebhookBindAddress is the default IP address the webhook server + // binds to. + DefaultWebhookBindAddress = "" + // DefaultContainerNode is the default value for the eponymous manager option. DefaultContainerNode = false diff --git a/pkg/manager/init/init_providers.go b/pkg/manager/init/init_providers.go index 2d5cfc128..ea96d9ac9 100644 --- a/pkg/manager/init/init_providers.go +++ b/pkg/manager/init/init_providers.go @@ -17,9 +17,8 @@ import ( func InitializeProviders( ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) error { - vmProviderName := fmt.Sprintf("%s/%s/vmProvider", ctx.Namespace, ctx.Name) - recorder := record.New(mgr.GetEventRecorderFor(vmProviderName)) + recorder := record.New(mgr.GetEventRecorder(vmProviderName)) ctx.VMProvider = vsphere.NewVSphereVMProviderFromClient(ctx, mgr.GetClient(), recorder) return nil } diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 9c40ebf8b..98d561a02 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -133,6 +133,7 @@ func New(ctx context.Context, opts Options) (Manager, error) { WebhookServer: webhook.NewServer(webhook.Options{ CertDir: opts.WebhookSecretVolumeMountPath, Port: opts.WebhookServiceContainerPort, + Host: opts.WebhookBindAddress, }), HealthProbeBindAddress: opts.HealthProbeBindAddress, PprofBindAddress: opts.PprofBindAddress, @@ -159,7 +160,7 @@ func New(ctx context.Context, opts Options) (Manager, error) { MaxConcurrentReconciles: opts.MaxConcurrentReconciles, ControllerMaxConcurrentReconciles: opts.ControllerMaxConcurrentReconciles, Logger: logger, - Recorder: record.New(mgr.GetEventRecorderFor(fmt.Sprintf("%s/%s", opts.PodNamespace, opts.PodName))), + Recorder: record.New(mgr.GetEventRecorder(fmt.Sprintf("%s/%s", opts.PodNamespace, opts.PodName))), ContainerNode: opts.ContainerNode, SyncPeriod: opts.SyncPeriod, EnableWebhookClientVerification: opts.EnableWebhookClientVerification, diff --git a/pkg/manager/options.go b/pkg/manager/options.go index e888cc69b..fecb71e50 100644 --- a/pkg/manager/options.go +++ b/pkg/manager/options.go @@ -135,6 +135,9 @@ type Options struct { // Defaults to the eponymous constant in this package. WebhookSecretVolumeMountPath string + // WebhookBindAddress is the IP address the webhook server binds to. + WebhookBindAddress string + // EnableWebhookClientVerification determines whether to use client certificate // verification for authentication to webhook requests. // @@ -227,6 +230,10 @@ func (o *Options) defaults() { o.WebhookSecretVolumeMountPath = DefaultWebhookSecretVolumeMountPath } + if o.WebhookBindAddress == "" { + o.WebhookBindAddress = DefaultWebhookBindAddress + } + if o.MetricsTLSOpts == nil { o.MetricsTLSOpts = []func(*tls.Config){ func(cfg *tls.Config) { diff --git a/pkg/prober/prober_manager.go b/pkg/prober/prober_manager.go index 44d86bca5..41b97cb41 100644 --- a/pkg/prober/prober_manager.go +++ b/pkg/prober/prober_manager.go @@ -91,7 +91,7 @@ func AddToManager( mgr ctrlmgr.Manager, vmProvider providers.VirtualMachineProviderInterface) (Manager, error) { - probeRecorder := vmoprecord.New(mgr.GetEventRecorderFor(proberManagerName)) + probeRecorder := vmoprecord.New(mgr.GetEventRecorder(proberManagerName)) // Add the probe manager explicitly as runnable in order to receive a Start() event. m := NewManager(ctx, mgr.GetClient(), probeRecorder, vmProvider) diff --git a/pkg/prober/prober_manager_test.go b/pkg/prober/prober_manager_test.go index 570eb9179..7c66b18ab 100644 --- a/pkg/prober/prober_manager_test.go +++ b/pkg/prober/prober_manager_test.go @@ -16,7 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" - clientgorecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" @@ -77,7 +77,7 @@ var _ = Describe("VirtualMachine probes", func() { JustBeforeEach(func() { fakeClient = builder.NewFakeClient(initObjects...) - eventRecorder := clientgorecord.NewFakeRecorder(1024) + eventRecorder := events.NewFakeRecorder(1024) fakeRecorder = record.New(eventRecorder) fakeVMProvider = &fake.VMProvider{} fakeCtrlManager = &fakeManager{client: fakeClient} @@ -310,7 +310,7 @@ type fakeManager struct { client client.Client } -func (f fakeManager) GetEventRecorderFor(name string) clientgorecord.EventRecorder { +func (f fakeManager) GetEventRecorder(name string) events.EventRecorder { return nil } diff --git a/pkg/prober/worker/readiness_worker_test.go b/pkg/prober/worker/readiness_worker_test.go index 2e97f9679..58ff91d94 100644 --- a/pkg/prober/worker/readiness_worker_test.go +++ b/pkg/prober/worker/readiness_worker_test.go @@ -14,7 +14,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - clientgorecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" @@ -58,7 +58,7 @@ var _ = Describe("VirtualMachine readiness probes", func() { vmKey = client.ObjectKey{Name: vm.Name, Namespace: vm.Namespace} fakeClient = builder.NewFakeClient() - eventRecorder := clientgorecord.NewFakeRecorder(1024) + eventRecorder := events.NewFakeRecorder(1024) fakeRecorder = record.New(eventRecorder) fakeEvents = eventRecorder.Events diff --git a/pkg/providers/vsphere/constants/constants.go b/pkg/providers/vsphere/constants/constants.go index 90ae60a39..d80f6c3c9 100644 --- a/pkg/providers/vsphere/constants/constants.go +++ b/pkg/providers/vsphere/constants/constants.go @@ -57,6 +57,10 @@ const ( PCIPassthruMMIOSizeExtraConfigKey = "pciPassthru.64bitMMIOSizeGB" //nolint:gosec PCIPassthruMMIOSizeDefault = "512" + // ExtraConfig reserved prefixes — vm-operator controls these and users must not set them directly. + ExtraConfigReservedPrefixVMService = "vmservice." + ExtraConfigReservedKeyVMXRebootPowerCycle = "vmx.reboot.powerCycle" + // FirmwareOverrideAnnotation is the annotation key used for firmware override. FirmwareOverrideAnnotation = pkg.VMOperatorKey + "/firmware" diff --git a/pkg/providers/vsphere/network/netop_assignment_mode_test.go b/pkg/providers/vsphere/network/netop_assignment_mode_test.go new file mode 100644 index 000000000..dac497208 --- /dev/null +++ b/pkg/providers/vsphere/network/netop_assignment_mode_test.go @@ -0,0 +1,140 @@ +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + + netopv1alpha1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" + "github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels" + "github.com/vmware-tanzu/vm-operator/pkg/providers/vsphere/network" +) + +var _ = Describe("NetOP assignment mode helpers", + Label(testlabels.API), + func() { + DescribeTable("EffectiveNetOPIPv4AssignmentMode", + func(st netopv1alpha1.NetworkInterfaceStatus, want netopv1alpha1.NetworkInterfaceIPAssignmentMode) { + Expect(network.EffectiveNetOPIPv4AssignmentMode(st)).To(Equal(want)) + }, + Entry("explicit DHCP", + netopv1alpha1.NetworkInterfaceStatus{ + IPAssignmentMode: netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP, + }, + netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP), + Entry("explicit static pool", + netopv1alpha1.NetworkInterfaceStatus{ + IPAssignmentMode: netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool, + }, + netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool), + Entry("unset with IPv4 IPConfig implies static pool", + netopv1alpha1.NetworkInterfaceStatus{ + IPConfigs: []netopv1alpha1.IPConfig{ + {IP: "10.0.0.1", IPFamily: corev1.IPv4Protocol}, + }, + }, + netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool), + Entry("unset without IPv4 implies DHCP", + netopv1alpha1.NetworkInterfaceStatus{ + IPConfigs: []netopv1alpha1.IPConfig{ + {IP: "2001:db8::1", IPFamily: corev1.IPv6Protocol}, + }, + }, + netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP), + ) + + DescribeTable("EffectiveNetOPIPv6AssignmentMode", + func(st netopv1alpha1.NetworkInterfaceStatus, want netopv1alpha1.NetworkInterfaceIPAssignmentMode) { + Expect(network.EffectiveNetOPIPv6AssignmentMode(st)).To(Equal(want)) + }, + Entry("explicit DHCP", + netopv1alpha1.NetworkInterfaceStatus{ + IPv6AssignmentMode: netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP, + }, + netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP), + Entry("explicit static pool", + netopv1alpha1.NetworkInterfaceStatus{ + IPv6AssignmentMode: netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool, + }, + netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool), + Entry("explicit none", + netopv1alpha1.NetworkInterfaceStatus{ + IPv6AssignmentMode: netopv1alpha1.NetworkInterfaceIPAssignmentModeNone, + }, + netopv1alpha1.NetworkInterfaceIPAssignmentModeNone), + Entry("unset defaults to none even if IPv6 appears in IPConfigs", + netopv1alpha1.NetworkInterfaceStatus{ + IPConfigs: []netopv1alpha1.IPConfig{ + {IP: "2001:db8::1", IPFamily: corev1.IPv6Protocol}, + }, + }, + netopv1alpha1.NetworkInterfaceIPAssignmentModeNone), + Entry("unset without IPv6 implies none", + netopv1alpha1.NetworkInterfaceStatus{ + IPConfigs: []netopv1alpha1.IPConfig{ + {IP: "10.0.0.1", IPFamily: corev1.IPv4Protocol}, + }, + }, + netopv1alpha1.NetworkInterfaceIPAssignmentModeNone), + ) + + DescribeTable("NetOPInterfaceIPFamilyPolicyFromIPAMModes", + func(modes []corev1.IPFamily, want netopv1alpha1.NetworkInterfaceIPFamilyPolicy) { + Expect(network.NetOPInterfaceIPFamilyPolicyFromIPAMModes(modes)).To(Equal(want)) + }, + Entry("nil slice defaults to IPv4Only", []corev1.IPFamily(nil), netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv4Only), + Entry("empty slice defaults to IPv4Only", []corev1.IPFamily{}, netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv4Only), + Entry("IPv4 only", []corev1.IPFamily{corev1.IPv4Protocol}, netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv4Only), + Entry("IPv6 only", []corev1.IPFamily{corev1.IPv6Protocol}, netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv6Only), + Entry("dual stack order v4 v6", + []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}, + netopv1alpha1.NetworkInterfaceIPFamilyPolicyDualStack), + Entry("dual stack order v6 v4", + []corev1.IPFamily{corev1.IPv6Protocol, corev1.IPv4Protocol}, + netopv1alpha1.NetworkInterfaceIPFamilyPolicyDualStack), + ) + + Describe("SyncNetOPIPFamilyPolicyFromIPAMModes", func() { + It("clears NetOP IPFamilyPolicy when VM IPAMModes is nil and NetOP had a policy", func() { + netIf := &netopv1alpha1.NetworkInterface{ + Spec: netopv1alpha1.NetworkInterfaceSpec{ + IPFamilyPolicy: netopv1alpha1.NetworkInterfaceIPFamilyPolicyDualStack, + }, + } + iface := &vmopv1.VirtualMachineNetworkInterfaceSpec{Name: "eth0"} + network.SyncNetOPIPFamilyPolicyFromIPAMModes(iface, netIf) + Expect(netIf.Spec.IPFamilyPolicy).To(Equal(netopv1alpha1.NetworkInterfaceIPFamilyPolicy(""))) + }) + + It("clears NetOP IPFamilyPolicy when VM IPAMModes is empty slice", func() { + netIf := &netopv1alpha1.NetworkInterface{ + Spec: netopv1alpha1.NetworkInterfaceSpec{ + IPFamilyPolicy: netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv6Only, + }, + } + iface := &vmopv1.VirtualMachineNetworkInterfaceSpec{ + Name: "eth0", + IPAMModes: []corev1.IPFamily{}, + } + network.SyncNetOPIPFamilyPolicyFromIPAMModes(iface, netIf) + Expect(netIf.Spec.IPFamilyPolicy).To(Equal(netopv1alpha1.NetworkInterfaceIPFamilyPolicy(""))) + }) + + It("sets NetOP IPFamilyPolicy when VM IPAMModes is non-empty", func() { + netIf := &netopv1alpha1.NetworkInterface{} + iface := &vmopv1.VirtualMachineNetworkInterfaceSpec{ + Name: "eth0", + IPAMModes: []corev1.IPFamily{corev1.IPv6Protocol}, + } + network.SyncNetOPIPFamilyPolicyFromIPAMModes(iface, netIf) + Expect(netIf.Spec.IPFamilyPolicy).To(Equal(netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv6Only)) + }) + }) + }) diff --git a/pkg/providers/vsphere/network/network.go b/pkg/providers/vsphere/network/network.go index 5196bf5be..7a12444db 100644 --- a/pkg/providers/vsphere/network/network.go +++ b/pkg/providers/vsphere/network/network.go @@ -357,6 +357,16 @@ func NetOPCRName(vmName, networkName, interfaceName string, isV1A1 bool) string return name } +// SyncNetOPIPFamilyPolicyFromIPAMModes sets or clears the NetOP IPFamilyPolicy from the VM IPAMModes. +// When IPAMModes is empty, the policy is cleared so NetOP can apply its default. +func SyncNetOPIPFamilyPolicyFromIPAMModes(interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec, netIf *netopv1alpha1.NetworkInterface) { + if len(interfaceSpec.IPAMModes) > 0 { + netIf.Spec.IPFamilyPolicy = NetOPInterfaceIPFamilyPolicyFromIPAMModes(interfaceSpec.IPAMModes) + } else { + netIf.Spec.IPFamilyPolicy = "" + } +} + func createNetOPNetworkInterface( vmCtx pkgctx.VirtualMachineContext, client ctrlclient.Client, @@ -423,6 +433,9 @@ func createNetOPNetworkInterface( } // NetOP only defines a VMXNet3 type, but it doesn't really matter for our purposes. netIf.Spec.Type = netopv1alpha1.NetworkInterfaceTypeVMXNet3 + // Set or clear IPFamilyPolicy from VM IPAMModes. When IPAMModes is empty, clear + // the policy so NetOP can apply its default (and updates remove a prior policy). + SyncNetOPIPFamilyPolicyFromIPAMModes(interfaceSpec, netIf) return nil }) @@ -446,6 +459,54 @@ func createNetOPNetworkInterface( return netOpNetIfToResult(vimClient, netIf), nil } +// EffectiveNetOPIPv4AssignmentMode returns how IPv4 is assigned according to NetworkInterface status. +// When IPAssignmentMode is unset, NetOP assumes static pool if any IPv4 address is present in IPConfigs, +// otherwise DHCP. +func EffectiveNetOPIPv4AssignmentMode(st netopv1alpha1.NetworkInterfaceStatus) netopv1alpha1.NetworkInterfaceIPAssignmentMode { + if st.IPAssignmentMode != "" { + return st.IPAssignmentMode + } + for i := range st.IPConfigs { + ip := &st.IPConfigs[i] + if ip.IPFamily == corev1.IPv4Protocol && ip.IP != "" { + return netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool + } + } + return netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP +} + +// EffectiveNetOPIPv6AssignmentMode returns how IPv6 is assigned according to NetworkInterface status. +// When IPv6AssignmentMode is unset, NetOP defaults to none (no IPv6 assignment). +func EffectiveNetOPIPv6AssignmentMode(st netopv1alpha1.NetworkInterfaceStatus) netopv1alpha1.NetworkInterfaceIPAssignmentMode { + if st.IPv6AssignmentMode != "" { + return st.IPv6AssignmentMode + } + return netopv1alpha1.NetworkInterfaceIPAssignmentModeNone +} + +// NetOPInterfaceIPFamilyPolicyFromIPAMModes maps spec IPAMModes (Kubernetes IP families) to +// NetOP NetworkInterfaceIPFamilyPolicy. With both families present it returns DualStack; +// IPv6 alone returns IPv6-only; otherwise IPv4-only (including empty input). +func NetOPInterfaceIPFamilyPolicyFromIPAMModes(ipamModes []corev1.IPFamily) netopv1alpha1.NetworkInterfaceIPFamilyPolicy { + hasV4, hasV6 := false, false + for _, f := range ipamModes { + if f == corev1.IPv4Protocol { + hasV4 = true + } + if f == corev1.IPv6Protocol { + hasV6 = true + } + } + switch { + case hasV4 && hasV6: + return netopv1alpha1.NetworkInterfaceIPFamilyPolicyDualStack + case hasV6: + return netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv6Only + default: + return netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv4Only + } +} + func netOpNetIfToResult( vimClient *vim25.Client, netIf *netopv1alpha1.NetworkInterface) *NetworkInterfaceResult { @@ -463,29 +524,37 @@ func netOpNetIfToResult( Backing: object.NewDistributedVirtualPortgroup(vimClient, pgObjRef), } - switch netIf.Status.IPAssignmentMode { - case netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP: - // When NetOP indicates DHCP, IPConfigs will be empty. - // Since NetOP doesn't distinguish between IPv4 and IPv6, set both. - // User's interface spec can override either or both flags. - result.DHCP4 = true - result.DHCP6 = true - case netopv1alpha1.NetworkInterfaceIPAssignmentModeNone: + v4Mode := EffectiveNetOPIPv4AssignmentMode(netIf.Status) + v6Mode := EffectiveNetOPIPv6AssignmentMode(netIf.Status) + + if v4Mode == netopv1alpha1.NetworkInterfaceIPAssignmentModeNone && + v6Mode == netopv1alpha1.NetworkInterfaceIPAssignmentModeNone { result.NoIPAM = true - default: // netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool - // Process all IPConfigs (both IPv4 and IPv6). This correctly handles: - // - IPv4-only scenarios (only IPv4 IPConfigs) - // - IPv6-only scenarios (only IPv6 IPConfigs) - // - Dual-stack scenarios (both IPv4 and IPv6 IPConfigs) - // DHCP4/DHCP6 are not set in StaticPool mode, only IPConfigs are populated. - for _, ip := range netIf.Status.IPConfigs { - ipConfig := NetworkInterfaceIPConfig{ - IPCIDR: ipCIDRNotation(ip.IP, ip.SubnetMask, ip.IPFamily == corev1.IPv4Protocol), - IsIPv4: ip.IPFamily == corev1.IPv4Protocol, - Gateway: ip.Gateway, + return result + } + + result.DHCP4 = v4Mode == netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP + result.DHCP6 = v6Mode == netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP + + for _, ip := range netIf.Status.IPConfigs { + switch ip.IPFamily { + case corev1.IPv4Protocol: + if v4Mode != netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool { + continue } - result.IPConfigs = append(result.IPConfigs, ipConfig) + case corev1.IPv6Protocol: + if v6Mode != netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool { + continue + } + default: + continue } + ipConfig := NetworkInterfaceIPConfig{ + IPCIDR: ipCIDRFromNetOPIPConfig(ip), + IsIPv4: ip.IPFamily == corev1.IPv4Protocol, + Gateway: ip.Gateway, + } + result.IPConfigs = append(result.IPConfigs, ipConfig) } return result @@ -963,6 +1032,24 @@ func waitForReadyNCPNetworkInterface( return vnetIf, nil } +// ipCIDRFromNetOPIPConfig builds the CIDR string for a NetOP IPConfig, preferring +// the Prefix field over the deprecated SubnetMask field. +func ipCIDRFromNetOPIPConfig(ip netopv1alpha1.IPConfig) string { + isIPv4 := ip.IPFamily == corev1.IPv4Protocol + mask := ip.SubnetMask + // Prefix is an integer (e.g. 24, 64) while ipCIDRNotation expects a subnet mask + // string (e.g. "255.255.255.0", "ffff:ffff:ffff:ffff::"). Convert via net.CIDRMask + // so we can reuse ipCIDRNotation for both paths. + if ip.Prefix != nil { + bits := 32 + if !isIPv4 { + bits = 128 + } + mask = net.IP(net.CIDRMask(int(*ip.Prefix), bits)).String() + } + return ipCIDRNotation(ip.IP, mask, isIPv4) +} + // ipCIDRNotation takes the IP and subnet mask and returns the IP in CIDR notation. // TODO: Better error checking. Nail down exactly how we want handle IPv4inV6 addresses. func ipCIDRNotation(ip string, mask string, isIPv4 bool) string { diff --git a/pkg/providers/vsphere/network/network_test.go b/pkg/providers/vsphere/network/network_test.go index 60fc6a9c9..f83538e9b 100644 --- a/pkg/providers/vsphere/network/network_test.go +++ b/pkg/providers/vsphere/network/network_test.go @@ -12,6 +12,7 @@ import ( vimtypes "github.com/vmware/govmomi/vim25/types" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -331,6 +332,8 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", Label(testlabels.VCSim), f netInterface.Status.ExternalID = externalID netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value netInterface.Status.MacAddress = "" // NetOP doesn't set this. + netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool + netInterface.Status.IPv6AssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ { IP: "192.168.1.110", @@ -463,12 +466,161 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", Label(testlabels.VCSim), f Expect(results.Results).To(HaveLen(1)) result := results.Results[0] Expect(result.DHCP4).To(BeTrue()) - Expect(result.DHCP6).To(BeTrue()) // Both should be set when NetOP indicates DHCP + Expect(result.DHCP6).To(BeFalse()) + Expect(result.NoIPAM).To(BeFalse()) + Expect(result.IPConfigs).To(BeEmpty()) + }) + }) + + When("DHCP is enabled for both IPv4 and IPv6", func() { + It("returns success with both DHCP flags set", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate successful NetOP reconcile with explicit IPv6 DHCP", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP + netInterface.Status.IPv6AssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + networkSpec) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.DHCP4).To(BeTrue()) + Expect(result.DHCP6).To(BeTrue()) Expect(result.NoIPAM).To(BeFalse()) Expect(result.IPConfigs).To(BeEmpty()) }) }) + When("IPv4 uses DHCP and IPv6 uses static pool", func() { + It("returns DHCP4 with IPv6 IPConfigs only", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate NetOP mixed assignment modes", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP + netInterface.Status.IPv6AssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool + netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ + { + IP: "2001:db8::100", + IPFamily: corev1.IPv6Protocol, + Gateway: "2001:db8::1", + SubnetMask: "ffff:ffff:ffff:ffff::", + }, + } + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + networkSpec) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.DHCP4).To(BeTrue()) + Expect(result.DHCP6).To(BeFalse()) + Expect(result.NoIPAM).To(BeFalse()) + Expect(result.IPConfigs).To(HaveLen(1)) + Expect(result.IPConfigs[0].IsIPv4).To(BeFalse()) + Expect(result.IPConfigs[0].IPCIDR).To(Equal("2001:db8::100/64")) + }) + }) + + When("IPv4 uses static pool and IPv6 uses DHCP", func() { + It("returns DHCP6 with IPv4 IPConfigs only", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate NetOP mixed assignment modes", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool + netInterface.Status.IPv6AssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeDHCP + netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ + { + IP: "192.168.1.100", + IPFamily: corev1.IPv4Protocol, + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + } + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + networkSpec) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.DHCP4).To(BeFalse()) + Expect(result.DHCP6).To(BeTrue()) + Expect(result.NoIPAM).To(BeFalse()) + Expect(result.IPConfigs).To(HaveLen(1)) + Expect(result.IPConfigs[0].IsIPv4).To(BeTrue()) + Expect(result.IPConfigs[0].IPCIDR).To(Equal("192.168.1.100/24")) + }) + }) + When("No IPAM is enabled", func() { It("returns success", func() { Expect(err).To(HaveOccurred()) @@ -550,6 +702,8 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", Label(testlabels.VCSim), f netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value netInterface.Status.MacAddress = "" // NetOP doesn't set this. + netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool + netInterface.Status.IPv6AssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ { IP: "192.168.1.110", @@ -631,18 +785,19 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", Label(testlabels.VCSim), f Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool + netInterface.Status.IPv6AssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ { - IP: "2001:db8::100", - IPFamily: corev1.IPv6Protocol, - Gateway: "2001:db8::1", - SubnetMask: "ffff:ffff:ffff:ffff::", + IP: "2001:db8::100", + IPFamily: corev1.IPv6Protocol, + Gateway: "2001:db8::1", + Prefix: ptr.To(int32(64)), }, { - IP: "2001:db8::101", - IPFamily: corev1.IPv6Protocol, - Gateway: "2001:db8::1", - SubnetMask: "ffff:ffff:ffff:ffff::", + IP: "2001:db8::101", + IPFamily: corev1.IPv6Protocol, + Gateway: "2001:db8::1", + Prefix: ptr.To(int32(64)), }, } netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ @@ -699,16 +854,16 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", Label(testlabels.VCSim), f netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ { - IP: "192.168.1.100", - IPFamily: corev1.IPv4Protocol, - Gateway: "192.168.1.1", - SubnetMask: "255.255.255.0", + IP: "192.168.1.100", + IPFamily: corev1.IPv4Protocol, + Gateway: "192.168.1.1", + Prefix: ptr.To(int32(24)), }, { - IP: "192.168.1.101", - IPFamily: corev1.IPv4Protocol, - Gateway: "192.168.1.1", - SubnetMask: "255.255.255.0", + IP: "192.168.1.101", + IPFamily: corev1.IPv4Protocol, + Gateway: "192.168.1.1", + Prefix: ptr.To(int32(24)), }, } netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ @@ -834,18 +989,21 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", Label(testlabels.VCSim), f Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool + netInterface.Status.IPv6AssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ { IP: "192.168.1.100", IPFamily: corev1.IPv4Protocol, Gateway: "192.168.1.1", SubnetMask: "255.255.255.0", + Prefix: ptr.To(int32(24)), }, { IP: "2001:db8::100", IPFamily: corev1.IPv6Protocol, Gateway: "2001:db8::1", SubnetMask: "ffff:ffff:ffff:ffff::", + Prefix: ptr.To(int32(64)), }, } netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ @@ -902,6 +1060,7 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", Label(testlabels.VCSim), f Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool + netInterface.Status.IPv6AssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ { IP: "192.168.1.100", @@ -969,6 +1128,7 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", Label(testlabels.VCSim), f Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool + netInterface.Status.IPv6AssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ { IP: "192.168.1.100", @@ -1107,6 +1267,7 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", Label(testlabels.VCSim), f Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value netInterface.Status.IPAssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool + netInterface.Status.IPv6AssignmentMode = netopv1alpha1.NetworkInterfaceIPAssignmentModeStaticPool netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ { IP: "192.168.1.100", @@ -1145,6 +1306,207 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", Label(testlabels.VCSim), f Expect(result.IPConfigs[1].Gateway).To(Equal("2001:db8::2")) }) }) + + Context("IPAMModes", func() { + Context("IPv4Only policy", func() { + BeforeEach(func() { + networkSpec.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: &common.PartialObjectRef{Name: networkName}, + IPAMModes: []corev1.IPFamily{corev1.IPv4Protocol}, + }, + } + }) + + It("creates NetworkInterface CR with IPv4Only policy", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + By("verify NetworkInterface CR has IPv4Only policy", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.IPFamilyPolicy).To(Equal(netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv4Only)) + }) + }) + }) + + Context("IPv6Only policy", func() { + BeforeEach(func() { + networkSpec.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: &common.PartialObjectRef{Name: networkName}, + IPAMModes: []corev1.IPFamily{corev1.IPv6Protocol}, + }, + } + }) + + It("creates NetworkInterface CR with IPv6Only policy", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + By("verify NetworkInterface CR has IPv6Only policy", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.IPFamilyPolicy).To(Equal(netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv6Only)) + }) + }) + }) + + Context("DualStack policy", func() { + BeforeEach(func() { + networkSpec.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: &common.PartialObjectRef{Name: networkName}, + IPAMModes: []corev1.IPFamily{ + corev1.IPv4Protocol, + corev1.IPv6Protocol, + }, + }, + } + }) + + It("creates NetworkInterface CR with DualStack policy", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + By("verify NetworkInterface CR has DualStack policy", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.IPFamilyPolicy).To(Equal(netopv1alpha1.NetworkInterfaceIPFamilyPolicyDualStack)) + }) + }) + }) + + Context("optional field not specified", func() { + BeforeEach(func() { + networkSpec.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: &common.PartialObjectRef{Name: networkName}, + // IPAMModes not set + }, + } + }) + + It("creates NetworkInterface CR without IPFamilyPolicy set", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + By("verify NetworkInterface CR does not have NetOP IPFamilyPolicy set", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + // IPFamilyPolicy should be empty string when not specified + Expect(netInterface.Spec.IPFamilyPolicy).To(Equal(netopv1alpha1.NetworkInterfaceIPFamilyPolicy(""))) + }) + }) + }) + + Context("multiple interfaces with different policies", func() { + BeforeEach(func() { + networkSpec.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: &common.PartialObjectRef{Name: networkName}, + IPAMModes: []corev1.IPFamily{corev1.IPv4Protocol}, + }, + { + Name: "eth1", + Network: &common.PartialObjectRef{Name: networkName}, + IPAMModes: []corev1.IPFamily{corev1.IPv6Protocol}, + }, + { + Name: "eth2", + Network: &common.PartialObjectRef{Name: networkName}, + IPAMModes: []corev1.IPFamily{ + corev1.IPv4Protocol, + corev1.IPv6Protocol, + }, + }, + } + }) + + It("creates NetworkInterface CRs with correct policies for each interface", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + expectedPolicies := map[string]netopv1alpha1.NetworkInterfaceIPFamilyPolicy{ + "eth0": netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv4Only, + "eth1": netopv1alpha1.NetworkInterfaceIPFamilyPolicyIPv6Only, + "eth2": netopv1alpha1.NetworkInterfaceIPFamilyPolicyDualStack, + } + + By("simulate successful NetOP reconcile for each interface", func() { + for ifName := range expectedPolicies { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, ifName, false), + Namespace: vm.Namespace, + }, + } + err := ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface) + if apierrors.IsNotFound(err) { + // First call bailed before creating this one; create it now. + netInterface.Spec.NetworkName = networkName + Expect(ctx.Client.Create(ctx, netInterface)).To(Succeed()) + } else { + Expect(err).ToNot(HaveOccurred()) + } + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + } + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + networkSpec) + Expect(err).ToNot(HaveOccurred()) + Expect(results.Results).To(HaveLen(3)) + + for ifName, expectedPolicy := range expectedPolicies { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, ifName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.IPFamilyPolicy).To(Equal(expectedPolicy), "interface %q", ifName) + } + }) + }) + }) }) Context("NCP", func() { diff --git a/pkg/providers/vsphere/placement/cluster_placement.go b/pkg/providers/vsphere/placement/cluster_placement.go index 09fae6db8..ed039e919 100644 --- a/pkg/providers/vsphere/placement/cluster_placement.go +++ b/pkg/providers/vsphere/placement/cluster_placement.go @@ -18,6 +18,7 @@ import ( pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context" pkglog "github.com/vmware-tanzu/vm-operator/pkg/log" pkgutil "github.com/vmware-tanzu/vm-operator/pkg/util" + faultutil "github.com/vmware-tanzu/vm-operator/pkg/util/vsphere/fault" ) // Recommendation is the info about a placement recommendation. @@ -224,13 +225,14 @@ func getClusterPlacementRecommendations( finder *find.Finder, resourcePoolsMoRefs []vimtypes.ManagedObjectReference, configSpecs []vimtypes.VirtualMachineConfigSpec, - needDatastorePlacement bool) (map[string]Recommendation, error) { + needHostPlacement, needDatastorePlacement bool) (map[string]Recommendation, error) { logger := pkglog.FromContextOrDefault(ctx) placementSpec := vimtypes.PlaceVmsXClusterSpec{ PlacementType: string(vimtypes.PlaceVmsXClusterSpecPlacementTypeCreateAndPowerOn), ResourcePools: resourcePoolsMoRefs, VmPlacementSpecs: make([]vimtypes.PlaceVmsXClusterSpecVmPlacementSpec, len(configSpecs)), + HostRecommRequired: &needHostPlacement, DatastoreRecommRequired: &needDatastorePlacement, } @@ -250,16 +252,17 @@ func getClusterPlacementRecommendations( logger.V(6).Info("PlaceVmsXCluster response", "results", vimtypes.ToString(results)) if len(results.Faults) != 0 { - var faultMgs []string - for _, f := range results.Faults { - msgs := make([]string, 0, len(f.Faults)) - for _, ff := range f.Faults { - msgs = append(msgs, ff.LocalizedMessage) + var faultMsgs []string + for _, pf := range results.Faults { + + leafMessages := faultutil.LocalizedMessagesFromFaults(pf.Faults) + if len(leafMessages) > 0 { + faultMsgs = append(faultMsgs, + fmt.Sprintf("Resource pool (%s) has faults: %s", pf.ResourcePool.Value, strings.Join(leafMessages, " "))) } - faultMgs = append(faultMgs, - fmt.Sprintf("ResourcePool %s faults: %s", f.ResourcePool.Value, strings.Join(msgs, ", "))) + } - return nil, fmt.Errorf("faults: %v", faultMgs) + return nil, fmt.Errorf("%v", faultMsgs) } recommendations := make(map[string]Recommendation, len(results.PlacementInfos)) diff --git a/pkg/providers/vsphere/placement/group_placement.go b/pkg/providers/vsphere/placement/group_placement.go index bf41f9ca6..1f2e464d8 100644 --- a/pkg/providers/vsphere/placement/group_placement.go +++ b/pkg/providers/vsphere/placement/group_placement.go @@ -107,5 +107,6 @@ func getGroupPlacementRecommendations( finder, candidateRPMoRefs, configSpecs, + true, needDatastorePlacement) } diff --git a/pkg/providers/vsphere/placement/zone_placement.go b/pkg/providers/vsphere/placement/zone_placement.go index c52e4d45c..68c8359c4 100644 --- a/pkg/providers/vsphere/placement/zone_placement.go +++ b/pkg/providers/vsphere/placement/zone_placement.go @@ -351,6 +351,7 @@ func getPlacementRecommendation( finder, candidateRPMoRefs, []vimtypes.VirtualMachineConfigSpec{configSpec}, + false, needDatastorePlacement) if err != nil { return Recommendation{}, fmt.Errorf("PlaceVmsXCluster failed: %w", err) diff --git a/pkg/providers/vsphere/placement/zone_placement_test.go b/pkg/providers/vsphere/placement/zone_placement_test.go index 30067b48b..a1d3942d4 100644 --- a/pkg/providers/vsphere/placement/zone_placement_test.go +++ b/pkg/providers/vsphere/placement/zone_placement_test.go @@ -313,6 +313,27 @@ func vcSimPlacement() { }) }) }) + + Context("when PlaceVmsXCluster returns NoCompatibleHost faults", func() { + AfterEach(func() { + ctx.SimulatorService().ClearFaultRules() + }) + + It("returns an error containing the leaf root cause (UnsupportedGuest)", func() { + ctx.SimulatorService().AddFaultRule(&simulator.FaultInjectionRule{ + MethodName: "PlaceVmsXCluster", + ObjectType: "Folder", + ObjectName: "*", + FaultType: simulator.FaultTypeCustom, + Fault: &vimtypes.NoCompatibleHost{}, + Probability: 1.0, + Enabled: true, + }) + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Finder, configSpec, constraints) + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + }) + }) }) }) diff --git a/pkg/providers/vsphere/upgrade/virtualmachine/vm_schema_upgrade.go b/pkg/providers/vsphere/upgrade/virtualmachine/vm_schema_upgrade.go index bfedc2816..c3f57657b 100644 --- a/pkg/providers/vsphere/upgrade/virtualmachine/vm_schema_upgrade.go +++ b/pkg/providers/vsphere/upgrade/virtualmachine/vm_schema_upgrade.go @@ -166,8 +166,8 @@ func ReconcileSchemaUpgrade( } } - if features.VMExtraConfig { - if f := vmopv1util.FeatureVersionNetExtraConfig; !vmFeatureVersion.Has(f) { + if features.TelcoVMServiceAPI { + if f := vmopv1util.FeatureVersionTelcoVMServiceAPI; !vmFeatureVersion.Has(f) { if _, err := virtualmachine.FillEmptyNetworkInterfaceTypesFromClass( ctx, k8sClient, vm); err != nil { diff --git a/pkg/providers/vsphere/upgrade/virtualmachine/vm_schema_upgrade_test.go b/pkg/providers/vsphere/upgrade/virtualmachine/vm_schema_upgrade_test.go index 9f00065ae..3c1e1aeb7 100644 --- a/pkg/providers/vsphere/upgrade/virtualmachine/vm_schema_upgrade_test.go +++ b/pkg/providers/vsphere/upgrade/virtualmachine/vm_schema_upgrade_test.go @@ -323,10 +323,10 @@ var _ = Describe("ReconcileSchemaUpgrade", func() { }) }) - When("VMExtraConfig feature is enabled", func() { + When("TelcoVMServiceAPI feature is enabled", func() { BeforeEach(func() { pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { - config.Features.VMExtraConfig = true + config.Features.TelcoVMServiceAPI = true }) vm.Spec.ClassName = "schema-up-class" vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ diff --git a/pkg/providers/vsphere/virtualmachine/affinity.go b/pkg/providers/vsphere/virtualmachine/affinity.go index 0f2b703f4..172dfcf0f 100644 --- a/pkg/providers/vsphere/virtualmachine/affinity.go +++ b/pkg/providers/vsphere/virtualmachine/affinity.go @@ -13,6 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" + pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config" pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context" kubeutil "github.com/vmware-tanzu/vm-operator/pkg/util/kube" ) @@ -79,7 +80,6 @@ func genConfigSpecAffinityPolicies( // processVMAffinity returns placement policies for VM affinity rules. // VM affinity is bidirectional, so we only need to send in the label specified // in the VM affinity policy. Not additional labels. -// Note: only zone topology is supported for VM affinity. func processVMAffinity( vmCtx pkgctx.VirtualMachineContext, affinity *vmopv1.VMAffinitySpec) []vimtypes.BaseVmPlacementPolicy { @@ -99,6 +99,22 @@ func processVMAffinity( }) } + // Process host-topology affinity terms only when VMAffinityDuringExecution is enabled. + if pkgcfg.FromContext(vmCtx).Features.VMAffinityDuringExecution { + // Process required affinity terms associated with host topology. + requiredHostTagIDs := buildTagIDsFromHostTopology( + vmCtx, + affinity.RequiredDuringSchedulingPreferredDuringExecution, + ) + for _, tagID := range requiredHostTagIDs { + placementPols = append(placementPols, &vimtypes.VmVmAffinity{ + AffinedVmsTag: tagID, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }) + } + } + // Process preferred affinity terms associated with zone topology. preferredZoneTagIDs := buildTagIDsFromZoneTopology( vmCtx, @@ -112,13 +128,28 @@ func processVMAffinity( }) } + // Process host-topology affinity terms only when VMAffinityDuringExecution is enabled. + if pkgcfg.FromContext(vmCtx).Features.VMAffinityDuringExecution { + // Process preferred affinity terms associated with host topology. + preferredHostTagIDs := buildTagIDsFromHostTopology( + vmCtx, + affinity.PreferredDuringSchedulingPreferredDuringExecution, + ) + for _, tagID := range preferredHostTagIDs { + placementPols = append(placementPols, &vimtypes.VmVmAffinity{ + AffinedVmsTag: tagID, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessPreferredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }) + } + } + return placementPols } // processVMAntiAffinity returns placement policies from VM anti-affinity rules. -// Use a single VmToVmGroupsAntiAffinity policy if the labels are non-empty. -// Note: only zone topology is processed for VM anti-affinity; host topology -// will be handled by ClusterModules. +// Use a single VmToVmGroupsAntiAffinity policy per topology/strictness +// combination if the labels are non-empty. func processVMAntiAffinity( vmCtx pkgctx.VirtualMachineContext, antiAffinity *vmopv1.VMAntiAffinitySpec) []vimtypes.BaseVmPlacementPolicy { @@ -138,6 +169,21 @@ func processVMAntiAffinity( }) } + // Process host-topology anti-affinity terms only when VMAffinityDuringExecution is enabled. + if pkgcfg.FromContext(vmCtx).Features.VMAffinityDuringExecution { + requiredHostTagIDs := buildTagIDsFromHostTopology( + vmCtx, + antiAffinity.RequiredDuringSchedulingPreferredDuringExecution, + ) + for _, tagID := range requiredHostTagIDs { + placementPols = append(placementPols, &vimtypes.VmVmAntiAffinity{ + AntiAffinedVmsTag: tagID, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }) + } + } + // Process preferred anti-affinity terms associated with zone topology. preferredZoneTagIDs := buildTagIDsFromZoneTopology( vmCtx, @@ -151,20 +197,36 @@ func processVMAntiAffinity( }) } + // Process host-topology anti-affinity terms only when VMAffinityDuringExecution is enabled. + if pkgcfg.FromContext(vmCtx).Features.VMAffinityDuringExecution { + preferredHostTagIDs := buildTagIDsFromHostTopology( + vmCtx, + antiAffinity.PreferredDuringSchedulingPreferredDuringExecution, + ) + for _, tagID := range preferredHostTagIDs { + placementPols = append(placementPols, &vimtypes.VmVmAntiAffinity{ + AntiAffinedVmsTag: tagID, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessPreferredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }) + } + } + return placementPols } -// buildTagIDsFromZoneTopology returns a list of TagIds built from the given -// affinity/anti-affinity terms that have zone topology. +// buildTagIDsFromTopology returns a list of TagIds built from the given +// affinity/anti-affinity terms that match the specified topology key. // Terms with other topology types are ignored. -func buildTagIDsFromZoneTopology( +func buildTagIDsFromTopology( vmCtx pkgctx.VirtualMachineContext, - terms []vmopv1.VMAffinityTerm) []vimtypes.TagId { + terms []vmopv1.VMAffinityTerm, + topologyKey string) []vimtypes.TagId { var tagIDs []vimtypes.TagId for _, term := range terms { - if term.TopologyKey != corev1.LabelTopologyZone { + if term.TopologyKey != topologyKey { continue } @@ -188,6 +250,24 @@ func buildTagIDsFromZoneTopology( return tagIDs } +// buildTagIDsFromZoneTopology returns a list of TagIds built from the given +// affinity/anti-affinity terms that have zone topology. +func buildTagIDsFromZoneTopology( + vmCtx pkgctx.VirtualMachineContext, + terms []vmopv1.VMAffinityTerm) []vimtypes.TagId { + + return buildTagIDsFromTopology(vmCtx, terms, corev1.LabelTopologyZone) +} + +// buildTagIDsFromHostTopology returns a list of TagIds built from the given +// affinity/anti-affinity terms that have host topology. +func buildTagIDsFromHostTopology( + vmCtx pkgctx.VirtualMachineContext, + terms []vmopv1.VMAffinityTerm) []vimtypes.TagId { + + return buildTagIDsFromTopology(vmCtx, terms, corev1.LabelHostname) +} + // extractLabelsFromSelector extracts all labels from a LabelSelector, handling both // MatchLabels and MatchExpressions with "In" operator (supporting multiple values). // Returns a slice of formatted labels in "key:value" format. diff --git a/pkg/providers/vsphere/virtualmachine/configspec_test.go b/pkg/providers/vsphere/virtualmachine/configspec_test.go index b3b3b10b6..a5e3544de 100644 --- a/pkg/providers/vsphere/virtualmachine/configspec_test.go +++ b/pkg/providers/vsphere/virtualmachine/configspec_test.go @@ -1025,69 +1025,468 @@ var _ = Describe("CreateConfigSpecForPlacement", func() { }) }) - Context("with unsupported topology key (host with affinity)", func() { + When("VMAffinityDuringExecution feature is enabled", func() { BeforeEach(func() { - vmCtx.VM.Spec.Affinity = &vmopv1.AffinitySpec{ - VMAffinity: &vmopv1.VMAffinitySpec{ - RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "component": "web", + pkgcfg.SetContext(vmCtx, func(config *pkgcfg.Config) { + config.Features.VMAffinityDuringExecution = true + }) + }) + + Context("required affinity policy with host topology key", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Affinity = &vmopv1.AffinitySpec{ + VMAffinity: &vmopv1.VMAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": "web", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "tier", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"frontend", "backend"}, + }, + }, }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "tier", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"frontend", "backend"}, + TopologyKey: corev1.LabelHostname, + }, + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment": "prod", }, }, + TopologyKey: corev1.LabelHostname, }, - TopologyKey: corev1.LabelHostname, }, - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "environment": "prod", + }, + } + }) + + It("config spec should have the expected host-level affinity policies", func() { + Expect(configSpec.VmPlacementPolicies).To(HaveLen(4)) + + pols := []vimtypes.BaseVmPlacementPolicy{ + &vimtypes.VmVmAffinity{ + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + AffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "component:web", + Category: vmCtx.VM.Namespace, + }, + }, + }, + &vimtypes.VmVmAffinity{ + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + AffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "tier:frontend", + Category: vmCtx.VM.Namespace, + }, + }, + }, + &vimtypes.VmVmAffinity{ + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + AffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "tier:backend", + Category: vmCtx.VM.Namespace, + }, + }, + }, + &vimtypes.VmVmAffinity{ + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + AffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "environment:prod", + Category: vmCtx.VM.Namespace, + }, + }, + }, + } + + Expect(configSpec.VmPlacementPolicies).To(HaveLen(4)) + Expect(configSpec.VmPlacementPolicies).To(ConsistOf(pols)) + assertVMTags(configSpec, []string{"vm-label1:vm-value1", "vm-label2:vm-value2"}, vmCtx.VM.Namespace) + }) + }) + + Context("preferred affinity policy with host topology key", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Affinity = &vmopv1.AffinitySpec{ + VMAffinity: &vmopv1.VMAffinitySpec{ + PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": "web", + }, + }, + TopologyKey: corev1.LabelHostname, + }, + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment": "prod", + }, }, + TopologyKey: corev1.LabelHostname, }, - TopologyKey: corev1.LabelHostname, }, }, - PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "component": "web", + } + }) + + It("config spec should have the expected host-level affinity policies", func() { + Expect(configSpec.VmPlacementPolicies).To(HaveLen(2)) + + pols := []vimtypes.BaseVmPlacementPolicy{ + &vimtypes.VmVmAffinity{ + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessPreferredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + AffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "component:web", + Category: vmCtx.VM.Namespace, + }, + }, + }, + &vimtypes.VmVmAffinity{ + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessPreferredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + AffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "environment:prod", + Category: vmCtx.VM.Namespace, + }, + }, + }, + } + + Expect(configSpec.VmPlacementPolicies).To(HaveLen(2)) + Expect(configSpec.VmPlacementPolicies).To(ConsistOf(pols)) + assertVMTags(configSpec, []string{"vm-label1:vm-value1", "vm-label2:vm-value2"}, vmCtx.VM.Namespace) + }) + }) + + Context("required anti-affinity policy with host topology key", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Affinity = &vmopv1.AffinitySpec{ + VMAntiAffinity: &vmopv1.VMAntiAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": "web", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "tier", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"frontend", "backend"}, + }, + }, }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "tier", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"frontend", "backend"}, + TopologyKey: corev1.LabelHostname, + }, + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment": "prod", }, }, + TopologyKey: corev1.LabelHostname, }, - TopologyKey: corev1.LabelHostname, }, - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "environment": "prod", + }, + } + }) + + It("creates multiple VmVmAntiAffinity policies with host topology", func() { + Expect(configSpec.VmPlacementPolicies).To(HaveLen(4)) + + expectedPolicies := []vimtypes.BaseVmPlacementPolicy{ + &vimtypes.VmVmAntiAffinity{ + AntiAffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "component:web", + Category: vmCtx.VM.Namespace, + }, + }, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }, + &vimtypes.VmVmAntiAffinity{ + AntiAffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "tier:frontend", + Category: vmCtx.VM.Namespace, + }, + }, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }, + &vimtypes.VmVmAntiAffinity{ + AntiAffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "tier:backend", + Category: vmCtx.VM.Namespace, + }, + }, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }, + &vimtypes.VmVmAntiAffinity{ + AntiAffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "environment:prod", + Category: vmCtx.VM.Namespace, + }, + }, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }, + } + + Expect(configSpec.VmPlacementPolicies).To(ConsistOf(expectedPolicies)) + assertVMTags(configSpec, []string{"vm-label1:vm-value1", "vm-label2:vm-value2"}, vmCtx.VM.Namespace) + }) + }) + + Context("preferred anti-affinity policy with host topology key", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Affinity = &vmopv1.AffinitySpec{ + VMAntiAffinity: &vmopv1.VMAntiAffinitySpec{ + PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": "web", + }, + }, + TopologyKey: corev1.LabelHostname, + }, + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment": "prod", + }, }, + TopologyKey: corev1.LabelHostname, }, - TopologyKey: corev1.LabelHostname, }, }, - }, - } + } + }) + + It("creates multiple VmVmAntiAffinity policies with host topology", func() { + Expect(configSpec.VmPlacementPolicies).To(HaveLen(2)) + + expectedPolicies := []vimtypes.BaseVmPlacementPolicy{ + &vimtypes.VmVmAntiAffinity{ + AntiAffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "component:web", + Category: vmCtx.VM.Namespace, + }, + }, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessPreferredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }, + &vimtypes.VmVmAntiAffinity{ + AntiAffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "environment:prod", + Category: vmCtx.VM.Namespace, + }, + }, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessPreferredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }, + } + + Expect(configSpec.VmPlacementPolicies).To(ConsistOf(expectedPolicies)) + assertVMTags(configSpec, []string{"vm-label1:vm-value1", "vm-label2:vm-value2"}, vmCtx.VM.Namespace) + }) }) - It("creates a minimal policy with VM tags but no anti-affinity rules", func() { - Expect(configSpec.VmPlacementPolicies).To(BeEmpty()) + Context("mixed zone and host topology anti-affinity terms", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Affinity = &vmopv1.AffinitySpec{ + VMAntiAffinity: &vmopv1.VMAntiAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "zone-label": "zone-value", + }, + }, + TopologyKey: corev1.LabelTopologyZone, + }, + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "host-label": "host-value", + }, + }, + TopologyKey: corev1.LabelHostname, + }, + }, + }, + } + }) - // VM tags should still be attached (without VM Operator managed labels) even though anti-affinity policy failed. - assertVMTags(configSpec, []string{"vm-label1:vm-value1", "vm-label2:vm-value2"}, vmCtx.VM.Namespace) + It("creates separate policies for zone and host topologies", func() { + Expect(configSpec.VmPlacementPolicies).To(HaveLen(2)) + + pols := []vimtypes.BaseVmPlacementPolicy{ + &vimtypes.VmToVmGroupsAntiAffinity{ + AntiAffinedVmGroupTags: []vimtypes.TagId{ + { + NameId: &vimtypes.TagIdNameId{ + Tag: "zone-label:zone-value", + Category: vmCtx.VM.Namespace, + }, + }, + }, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyVSphereZone), + }, + &vimtypes.VmVmAntiAffinity{ + AntiAffinedVmsTag: vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "host-label:host-value", + Category: vmCtx.VM.Namespace, + }, + }, + PolicyStrictness: string(vimtypes.VmPlacementPolicyVmPlacementPolicyStrictnessRequiredDuringPlacementPreferredDuringExecution), + PolicyTopology: string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyHost), + }, + } + + Expect(configSpec.VmPlacementPolicies).To(HaveLen(2)) + Expect(configSpec.VmPlacementPolicies).To(ConsistOf(pols)) + assertVMTags(configSpec, []string{"vm-label1:vm-value1", "vm-label2:vm-value2"}, vmCtx.VM.Namespace) + }) + }) + }) + + When("VMAffinityDuringExecution feature is disabled", func() { + Context("host topology affinity terms are ignored", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Affinity = &vmopv1.AffinitySpec{ + VMAffinity: &vmopv1.VMAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": "web", + }, + }, + TopologyKey: corev1.LabelHostname, + }, + }, + PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment": "prod", + }, + }, + TopologyKey: corev1.LabelHostname, + }, + }, + }, + } + }) + + It("produces no placement policies", func() { + Expect(configSpec.VmPlacementPolicies).To(BeEmpty()) + assertVMTags(configSpec, []string{"vm-label1:vm-value1", "vm-label2:vm-value2"}, vmCtx.VM.Namespace) + }) + }) + + Context("host topology anti-affinity terms are ignored", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Affinity = &vmopv1.AffinitySpec{ + VMAntiAffinity: &vmopv1.VMAntiAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": "web", + }, + }, + TopologyKey: corev1.LabelHostname, + }, + }, + PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment": "prod", + }, + }, + TopologyKey: corev1.LabelHostname, + }, + }, + }, + } + }) + + It("produces no placement policies", func() { + Expect(configSpec.VmPlacementPolicies).To(BeEmpty()) + assertVMTags(configSpec, []string{"vm-label1:vm-value1", "vm-label2:vm-value2"}, vmCtx.VM.Namespace) + }) + }) + + Context("mixed zone and host topology terms only produce zone policies", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Affinity = &vmopv1.AffinitySpec{ + VMAntiAffinity: &vmopv1.VMAntiAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "zone-label": "zone-value", + }, + }, + TopologyKey: corev1.LabelTopologyZone, + }, + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "host-label": "host-value", + }, + }, + TopologyKey: corev1.LabelHostname, + }, + }, + }, + } + }) + + It("creates only the zone topology policy", func() { + Expect(configSpec.VmPlacementPolicies).To(HaveLen(1)) + + policy, ok := configSpec.VmPlacementPolicies[0].(*vimtypes.VmToVmGroupsAntiAffinity) + Expect(ok).To(BeTrue()) + Expect(policy.PolicyTopology).To(Equal(string(vimtypes.VmPlacementPolicyVmPlacementPolicyTopologyVSphereZone))) + Expect(policy.AntiAffinedVmGroupTags).To(ConsistOf(vimtypes.TagId{ + NameId: &vimtypes.TagIdNameId{ + Tag: "zone-label:zone-value", + Category: vmCtx.VM.Namespace, + }, + })) + + assertVMTags(configSpec, []string{"vm-label1:vm-value1", "vm-label2:vm-value2"}, vmCtx.VM.Namespace) + }) }) }) diff --git a/pkg/providers/vsphere/vmlifecycle/update_status.go b/pkg/providers/vsphere/vmlifecycle/update_status.go index 3e181ad23..2d58bd774 100644 --- a/pkg/providers/vsphere/vmlifecycle/update_status.go +++ b/pkg/providers/vsphere/vmlifecycle/update_status.go @@ -375,14 +375,18 @@ func reconcileStatusZone( var errs []error - zoneName := vmCtx.VM.Labels[corev1.LabelTopologyZone] - if zoneName == "" { + zoneLabel := vmCtx.VM.Labels[corev1.LabelTopologyZone] + zoneStatus := vmCtx.VM.Status.Zone + + // The label value is protected by the VM validation webhook for non-privileged users, + // but in case the value was accidentally changed by like kube-admin relook the zone. + if zoneLabel == "" || zoneLabel != zoneStatus { clusterMoRef, err := vcenter.GetResourcePoolOwnerMoRef( vmCtx, vcVM.Client(), vmCtx.MoVM.ResourcePool.Value) if err != nil { errs = append(errs, err) } else { - zoneName, err = topology.LookupZoneForClusterMoID( + zoneName, err := topology.LookupZoneForClusterMoID( vmCtx, k8sClient, clusterMoRef.Value) if err != nil { errs = append(errs, err) @@ -391,14 +395,11 @@ func reconcileStatusZone( vmCtx.VM.Labels = map[string]string{} } vmCtx.VM.Labels[corev1.LabelTopologyZone] = zoneName + vmCtx.VM.Status.Zone = zoneName } } } - if zoneName != "" { - vmCtx.VM.Status.Zone = zoneName - } - return errs } @@ -569,6 +570,55 @@ func getRuntimeHostHostname( return "", nil } +// extractIPFromAddress extracts the IP address from a string that may contain CIDR notation. +func extractIPFromAddress(address string) string { + ip, _, err := pkgutil.ParseIP(address) + if err != nil || ip == nil { + return "" + } + return ip.String() +} + +// findInterfaceContainingIP finds the index of the interface that contains the given IP. +func findInterfaceContainingIP(ip string, ifaces []vmopv1.VirtualMachineNetworkInterfaceStatus) int { + // Normalize the input IP for comparison + normalizedIP := extractIPFromAddress(ip) + if normalizedIP == "" { + return -1 + } + for i, iface := range ifaces { + if iface.IP == nil { + continue + } + for _, addr := range iface.IP.Addresses { + if extractIPFromAddress(addr.Address) == normalizedIP { + return i + } + } + } + return -1 +} + +// extractIPsFromInterface extracts IP addresses of the specified family from an interface. +func extractIPsFromInterface(iface vmopv1.VirtualMachineNetworkInterfaceStatus, isIPv4Required bool, validatePrimaryIP func(string) net.IP) []string { + var result []string + if iface.IP == nil { + return result + } + for _, addr := range iface.IP.Addresses { + ipStr := extractIPFromAddress(addr.Address) + if ipStr == "" { + continue + } + if a := validatePrimaryIP(ipStr); a != nil { + if (isIPv4Required && a.To4() != nil) || (!isIPv4Required && a.To4() == nil) { + result = append(result, ipStr) + } + } + } + return result +} + func guestNicInfoToInterfaceStatus( name string, deviceKey int32, @@ -1119,6 +1169,45 @@ func updateGuestNetworkStatus( } } + // See cloud-init issue: https://github.com/canonical/cloud-init/issues/6851 + // LinuxPrep does not report dual stack IPs. It reports only one primary IP. + // Fallback when guest-reported primaries are incomplete (common in some dual-stack / cloud-init cases). + // Same-interface cross-family fill only applies if that interface reports exactly one usable address + // of the missing family—otherwise we do not guess. The all-empty sole-NIC path uses the same rule + // per family. Usable addresses exclude loopback, unspecified, and link-local (validatePrimaryIP). + if len(ifaceStatuses) > 0 { + switch { + case primaryIP4 != "" && primaryIP6 == "": + // Try to find IPv6 on the same interface as IPv4 + if idx := findInterfaceContainingIP(primaryIP4, ifaceStatuses); idx >= 0 { + ipv6Addrs := extractIPsFromInterface(ifaceStatuses[idx], false, validatePrimaryIP) + if len(ipv6Addrs) == 1 { + primaryIP6 = ipv6Addrs[0] + } + } + case primaryIP6 != "" && primaryIP4 == "": + // Try to find IPv4 on the same interface as IPv6 + if idx := findInterfaceContainingIP(primaryIP6, ifaceStatuses); idx >= 0 { + ipv4Addrs := extractIPsFromInterface(ifaceStatuses[idx], true, validatePrimaryIP) + if len(ipv4Addrs) == 1 { + primaryIP4 = ipv4Addrs[0] + } + } + case primaryIP4 == "" && primaryIP6 == "": + // Both empty, try first interface, if its the only interface present. + if len(ifaceStatuses) == 1 { + ipv4Addrs := extractIPsFromInterface(ifaceStatuses[0], true, validatePrimaryIP) + ipv6Addrs := extractIPsFromInterface(ifaceStatuses[0], false, validatePrimaryIP) + if len(ipv4Addrs) == 1 { + primaryIP4 = ipv4Addrs[0] + } + if len(ipv6Addrs) == 1 { + primaryIP6 = ipv6Addrs[0] + } + } + } + } + var ( lip4 = len(primaryIP4) > 0 lip6 = len(primaryIP6) > 0 diff --git a/pkg/providers/vsphere/vmlifecycle/update_status_test.go b/pkg/providers/vsphere/vmlifecycle/update_status_test.go index 2a0353916..848aa7973 100644 --- a/pkg/providers/vsphere/vmlifecycle/update_status_test.go +++ b/pkg/providers/vsphere/vmlifecycle/update_status_test.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - apirecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" @@ -54,8 +54,11 @@ var _ = Describe("UpdateStatus", func() { BeforeEach(func() { ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{}) + nsInfo := ctx.CreateWorkloadNamespace() + vm := builder.DummyVirtualMachine() vm.Name = "update-status-test" + vm.Namespace = nsInfo.Namespace vmCtx = pkgctx.VirtualMachineContext{ Context: ctx, @@ -67,6 +70,14 @@ var _ = Describe("UpdateStatus", func() { vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + task, err := vcVM.Relocate(ctx, vimtypes.VirtualMachineRelocateSpec{ + Folder: ptr.To(nsInfo.Folder.Reference()), + Pool: ptr.To(nsRP.Reference()), + }, vimtypes.VirtualMachineMovePriorityDefaultPriority) + Expect(err).ToNot(HaveOccurred()) + Expect(task.Wait(ctx)).To(Succeed()) + // Initialize with the expected properties. Tests can overwrite this if needed. Expect(vcVM.Properties( ctx, @@ -484,6 +495,542 @@ var _ = Describe("UpdateStatus", func() { }) }) }) + + Context("Fallback from interface statuses", func() { + const ( + fallbackIP4 = "10.0.0.1" + fallbackIP6 = "2001:db8::1" + ) + + Context("Non-CloudInit: PrimaryIP4 set, PrimaryIP6 missing - fallback from same interface", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = validIP4 + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: validIP4, + State: "preferred", + }, + { + IpAddress: fallbackIP6, + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should populate PrimaryIP6 from the same interface", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(validIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(Equal(fallbackIP6)) + }) + }) + + Context("Non-CloudInit: PrimaryIP6 set, PrimaryIP4 missing - fallback from same interface", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = validIP6 + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: validIP6, + State: "preferred", + }, + { + IpAddress: fallbackIP4, + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should populate PrimaryIP4 from the same interface", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(fallbackIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(Equal(validIP6)) + }) + }) + + Context("Non-CloudInit: Both empty - fallback from sole interface", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: fallbackIP4, + State: "preferred", + }, + { + IpAddress: fallbackIP6, + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should populate both PrimaryIP4 and PrimaryIP6 from that interface", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(fallbackIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(Equal(fallbackIP6)) + }) + }) + + Context("CloudInit: PrimaryIP4 set, PrimaryIP6 missing - fallback from same interface", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Config = &vimtypes.VirtualMachineConfigInfo{} + vmCtx.VM.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ + CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, + } + vmCtx.MoVM.Config.ExtraConfig = []vimtypes.BaseOptionValue{ + &vimtypes.OptionValue{ + Key: constants.CloudInitGuestInfoLocalIPv4Key, + Value: validIP4, + }, + } + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: validIP4, + State: "preferred", + }, + { + IpAddress: fallbackIP6, + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should populate PrimaryIP6 from the same interface", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(validIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(Equal(fallbackIP6)) + }) + }) + + Context("CloudInit: PrimaryIP6 set, PrimaryIP4 missing - fallback from same interface", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Config = &vimtypes.VirtualMachineConfigInfo{} + vmCtx.VM.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ + CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, + } + vmCtx.MoVM.Config.ExtraConfig = []vimtypes.BaseOptionValue{ + &vimtypes.OptionValue{ + Key: constants.CloudInitGuestInfoLocalIPv6Key, + Value: validIP6, + }, + } + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: validIP6, + State: "preferred", + }, + { + IpAddress: fallbackIP4, + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should populate PrimaryIP4 from the same interface", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(fallbackIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(Equal(validIP6)) + }) + }) + + Context("CloudInit: Both empty - fallback from sole interface", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Config = &vimtypes.VirtualMachineConfigInfo{} + vmCtx.VM.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ + CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, + } + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: fallbackIP4, + State: "preferred", + }, + { + IpAddress: fallbackIP6, + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should populate both PrimaryIP4 and PrimaryIP6 from that interface", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(fallbackIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(Equal(fallbackIP6)) + }) + }) + + Context("Both empty, sole interface has only IPv4", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: fallbackIP4, + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should populate only PrimaryIP4", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(fallbackIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(BeEmpty()) + }) + }) + + Context("Both empty, sole interface has only IPv6", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: fallbackIP6, + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should populate only PrimaryIP6", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(BeEmpty()) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(Equal(fallbackIP6)) + }) + }) + + Context("Non-CloudInit: two global IPv6 on same NIC — no ambiguous IPv6 fallback", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = validIP4 + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: validIP4, + State: "preferred", + }, + { + IpAddress: "2001:db8::a", + State: "preferred", + }, + { + IpAddress: "2001:db8::b", + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should not populate PrimaryIP6", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(validIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(BeEmpty()) + }) + }) + + Context("Non-CloudInit: two global IPv4 on same NIC — no ambiguous IPv4 fallback", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = validIP6 + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: validIP6, + State: "preferred", + }, + { + IpAddress: "10.0.0.10", + State: "preferred", + }, + { + IpAddress: "10.0.0.11", + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should not populate PrimaryIP4", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(BeEmpty()) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(Equal(validIP6)) + }) + }) + + Context("Both empty, sole interface has one IPv4 and two global IPv6 — no ambiguous PrimaryIP6", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: fallbackIP4, + State: "preferred", + }, + { + IpAddress: "2001:db8::a", + State: "preferred", + }, + { + IpAddress: "2001:db8::b", + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should populate only PrimaryIP4", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(fallbackIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(BeEmpty()) + }) + }) + + Context("Both empty, sole interface has one IPv6 and two global IPv4 — no ambiguous PrimaryIP4", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: fallbackIP6, + State: "preferred", + }, + { + IpAddress: "10.0.0.10", + State: "preferred", + }, + { + IpAddress: "10.0.0.11", + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should populate only PrimaryIP6", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(BeEmpty()) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(Equal(fallbackIP6)) + }) + }) + + Context("Both empty, sole interface has two IPv4 and two IPv6 — fully ambiguous", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: "10.0.0.10", + State: "preferred", + }, + { + IpAddress: "10.0.0.11", + State: "preferred", + }, + { + IpAddress: "2001:db8::a", + State: "preferred", + }, + { + IpAddress: "2001:db8::b", + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should leave both PrimaryIP fields empty", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(BeEmpty()) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(BeEmpty()) + }) + }) + + Context("Non-CloudInit: both primaries empty with multiple NICs — no sole-interface fallback", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.VM.Spec.Network.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + {Name: "eth0"}, + {Name: "eth1"}, + } + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: fallbackIP4, + State: "preferred", + }, + }, + }, + }, + { + DeviceConfigId: 4001, + MacAddress: "00:11:22:33:44:66", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: fallbackIP6, + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + data.NetworkDeviceKeysToSpecIdx[4001] = 1 + }) + It("should leave both PrimaryIP fields empty", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(BeEmpty()) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(BeEmpty()) + }) + }) + + Context("Interface not found for PrimaryIP4", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = validIP4 + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: "10.0.0.99", + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should not populate PrimaryIP6", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(validIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(BeEmpty()) + }) + }) + + Context("CIDR notation in Address field", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:11:22:33:44:55", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: fallbackIP4 + "/24", + State: "preferred", + }, + { + IpAddress: fallbackIP6 + "/64", + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + It("should extract IPs correctly, stripping CIDR notation", func() { + Expect(vmCtx.VM.Status.Network).ToNot(BeNil()) + Expect(vmCtx.VM.Status.Network.PrimaryIP4).To(Equal(fallbackIP4)) + Expect(vmCtx.VM.Status.Network.PrimaryIP6).To(Equal(fallbackIP6)) + }) + }) + }) }) Context("Interfaces", func() { @@ -836,6 +1383,47 @@ var _ = Describe("UpdateStatus", func() { }) }) + When("guest IpAddress is empty and sole NIC reports ambiguous dual-stack IPs", func() { + BeforeEach(func() { + vmCtx.MoVM.Guest.IpAddress = "" + vmCtx.MoVM.Guest.Net = []vimtypes.GuestNicInfo{ + { + DeviceConfigId: 4000, + MacAddress: "00:50:56:00:00:01", + IpConfig: &vimtypes.NetIpConfigInfo{ + IpAddress: []vimtypes.NetIpConfigInfoIpAddress{ + { + IpAddress: "10.0.0.10", + State: "preferred", + }, + { + IpAddress: "10.0.0.11", + State: "preferred", + }, + { + IpAddress: "2001:db8::a", + State: "preferred", + }, + { + IpAddress: "2001:db8::b", + State: "preferred", + }, + }, + }, + }, + } + data.NetworkDeviceKeysToSpecIdx[4000] = 0 + }) + + It("should set VirtualMachineGuestNetworkConfigSynced condition to False", func() { + cond := conditions.Get(vmCtx.VM, vmopv1.VirtualMachineGuestNetworkConfigSynced) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal("NotSynced")) + Expect(cond.Message).To(Equal("Neither IPv4 nor IPv6 address reported by guest")) + }) + }) + When("guest property is nil", func() { BeforeEach(func() { vmCtx.MoVM.Guest = nil @@ -2470,6 +3058,79 @@ var _ = Describe("UpdateStatus", func() { }) }) + Context("Zone", func() { + var zoneName string + + BeforeEach(func() { + delete(vmCtx.VM.Labels, corev1.LabelTopologyZone) + vmCtx.VM.Status.Zone = "" + + zoneName = ctx.GetFirstZoneName() + }) + + assertZones := func() { + GinkgoHelper() + Expect(vmCtx.VM.Labels).To(HaveKeyWithValue(corev1.LabelTopologyZone, zoneName)) + Expect(vmCtx.VM.Status.Zone).To(Equal(zoneName)) + } + + Context("Neither label and status are set", func() { + It("Sets Zone", assertZones) + }) + + Context("Label and status are both set", func() { + BeforeEach(func() { + vmCtx.VM.Labels[corev1.LabelTopologyZone] = zoneName + vmCtx.VM.Status.Zone = zoneName + }) + It("Zone still set", assertZones) + }) + + Context("Label is set but status is not", func() { + BeforeEach(func() { + vmCtx.VM.Labels[corev1.LabelTopologyZone] = zoneName + }) + It("Sets Zone", assertZones) + }) + + Context("Label is not set but status is", func() { + BeforeEach(func() { + vmCtx.VM.Status.Zone = zoneName + }) + It("Sets Zone", assertZones) + }) + + Context("Label is not set but status is", func() { + BeforeEach(func() { + vmCtx.VM.Status.Zone = zoneName + }) + It("Sets Zone", assertZones) + }) + + Context("Label is not set but status is", func() { + BeforeEach(func() { + vmCtx.VM.Status.Zone = zoneName + }) + It("Sets Zone", assertZones) + }) + + Context("Label is set to incorrect value", func() { + BeforeEach(func() { + vmCtx.VM.Labels[corev1.LabelTopologyZone] = "bogus" + vmCtx.VM.Status.Zone = zoneName + }) + It("Sets Zone", assertZones) + }) + + Context("Status is set to incorrect value", func() { + BeforeEach(func() { + vmCtx.VM.Labels[corev1.LabelTopologyZone] = zoneName + vmCtx.VM.Status.Zone = "bogus" + }) + It("Sets Zone", assertZones) + }) + }) + Context("Controllers", func() { When("moVM.Config is nil", func() { BeforeEach(func() { @@ -3494,7 +4155,7 @@ var _ = Describe("UpdateStatus", func() { vmCtx.Context = record.WithContext( vmCtx.Context, - record.New(&apirecord.FakeRecorder{Events: chanRecord})) + record.New(&events.FakeRecorder{Events: chanRecord})) pkgcfg.SetContext(vmCtx, func(config *pkgcfg.Config) { config.AsyncSignalEnabled = true diff --git a/pkg/providers/vsphere/vmprovider_vmgroup_test.go b/pkg/providers/vsphere/vmprovider_vmgroup_test.go index 1cab1272f..d2630f509 100644 --- a/pkg/providers/vsphere/vmprovider_vmgroup_test.go +++ b/pkg/providers/vsphere/vmprovider_vmgroup_test.go @@ -177,7 +177,7 @@ var _ = Describe( Expect(pkgcond.IsTrue(&ms, vmopv1.VirtualMachineGroupMemberConditionPlacementReady)).To(BeTrue(), "No placement ready condition") Expect(ms.Placement.Zone).ToNot(BeEmpty(), "Missing Placement Zone") Expect(ms.Placement.Pool).ToNot(BeEmpty(), "Missing Placement Pool") - Expect(ms.Placement.Node).To(BeEmpty(), "Has Placement Node") + Expect(ms.Placement.Node).ToNot(BeEmpty(), "Missing Placement Node") if pkgcfg.FromContext(ctx).Features.FastDeploy { Expect(ms.Placement.Datastores).ToNot(BeEmpty(), "Missing Placement Datastores") // Verify against VirtualMachineImageCache.Status diff --git a/pkg/record/recorder.go b/pkg/record/recorder.go index 8993f2d96..070ce7191 100644 --- a/pkg/record/recorder.go +++ b/pkg/record/recorder.go @@ -1,5 +1,5 @@ // © Broadcom. All Rights Reserved. -// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // SPDX-License-Identifier: Apache-2.0 package record @@ -9,7 +9,7 @@ import ( "golang.org/x/text/language" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" ) // Recorder knows how to record events on behalf of a source. @@ -31,36 +31,44 @@ type Recorder interface { } // New returns a new instance of a Recorder. -func New(eventRecorder record.EventRecorder) Recorder { +func New(eventRecorder events.EventRecorder) Recorder { return recorder{EventRecorder: eventRecorder} } type recorder struct { - record.EventRecorder + events.EventRecorder +} + +// titleCase returns a title-cased string. A new Caser is created on each call +// because cases.Caser is not safe for concurrent use. +func titleCase(s string) string { + return cases.Title(language.English, cases.NoLower).String(s) } -// Event constructs an event from the given information and puts it in the queue for sending. func (r recorder) Event(object runtime.Object, reason, message string) { - r.EventRecorder.Event(object, corev1.EventTypeNormal, - cases.Title(language.English, cases.NoLower).String(reason), message) + reason = titleCase(reason) + r.EventRecorder.Eventf(object, nil, corev1.EventTypeNormal, + reason, reason, "%s", message) } // Eventf is just like Event, but with Sprintf for the message field. func (r recorder) Eventf(object runtime.Object, reason, message string, args ...interface{}) { - r.EventRecorder.Eventf(object, corev1.EventTypeNormal, - cases.Title(language.English, cases.NoLower).String(reason), message, args...) + reason = titleCase(reason) + r.EventRecorder.Eventf(object, nil, corev1.EventTypeNormal, + reason, reason, message, args...) } // Warn constructs a warning event from the given information and puts it in the queue for sending. func (r recorder) Warn(object runtime.Object, reason, message string) { - r.EventRecorder.Event(object, corev1.EventTypeWarning, - cases.Title(language.English, cases.NoLower).String(reason), message) + reason = titleCase(reason) + r.EventRecorder.Eventf(object, nil, corev1.EventTypeWarning, + reason, reason, "%s", message) } // Warnf is just like Event, but with Sprintf for the message field. func (r recorder) Warnf(object runtime.Object, reason, message string, args ...interface{}) { - r.EventRecorder.Eventf(object, corev1.EventTypeWarning, - cases.Title(language.English, cases.NoLower).String(reason), message, args...) + r.EventRecorder.Eventf(object, nil, corev1.EventTypeWarning, + titleCase(reason), titleCase(reason), message, args...) } // EmitEvent records a Success or Failure depending on whether or not an error occurred. diff --git a/pkg/record/recorder_context_test.go b/pkg/record/recorder_context_test.go index b403bafc7..ceab45a91 100644 --- a/pkg/record/recorder_context_test.go +++ b/pkg/record/recorder_context_test.go @@ -10,7 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - apirecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" "github.com/vmware-tanzu/vm-operator/pkg/record" ) @@ -22,7 +22,7 @@ var _ = Describe("WithContext", func() { ) BeforeEach(func() { left = context.Background() - leftVal = record.New(apirecord.NewFakeRecorder(0)) + leftVal = record.New(events.NewFakeRecorder(0)) }) When("parent is nil", func() { BeforeEach(func() { diff --git a/pkg/record/recorder_test.go b/pkg/record/recorder_test.go index 934b6057e..73a0866f6 100644 --- a/pkg/record/recorder_test.go +++ b/pkg/record/recorder_test.go @@ -8,13 +8,13 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - apirecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" "github.com/vmware-tanzu/vm-operator/pkg/record" ) var _ = Describe("Event utils", func() { - fakeRecorder := apirecord.NewFakeRecorder(100) + fakeRecorder := events.NewFakeRecorder(100) recorder := record.New(fakeRecorder) Context("Publish event", func() { diff --git a/pkg/util/kube/storage.go b/pkg/util/kube/storage.go index ca858d2f2..638946bd4 100644 --- a/pkg/util/kube/storage.go +++ b/pkg/util/kube/storage.go @@ -69,9 +69,13 @@ func GetPVCZoneConstraints( var zones sets.Set[string] for _, pvc := range pvcs { - if pvc.Spec.DataSourceRef != nil { - // Do not worry about PVCs with data source refs. - continue + if dsRef := pvc.Spec.DataSourceRef; dsRef != nil && dsRef.APIGroup != nil { + // Skip the zonal constraint check for PVCs with us as the DataSourceRef since + // those disks are in the placement ConfigSpec. Note that the reference may + // point to another object type such as VolumeSnapshot. + if *dsRef.APIGroup == vmopv1.GroupVersion.Group && dsRef.Kind == "VirtualMachine" { + continue + } } var z sets.Set[string] diff --git a/pkg/util/kube/storage_test.go b/pkg/util/kube/storage_test.go index 8ea2def9b..a8d06dfd9 100644 --- a/pkg/util/kube/storage_test.go +++ b/pkg/util/kube/storage_test.go @@ -449,6 +449,61 @@ var _ = Describe("GetPVCZoneConstraints", func() { Expect(zones.UnsortedList()).To(ConsistOf("zone1")) }) }) + + Context("PVC with DataSourceRef", func() { + + It("skips zone constraints when DataSourceRef points to a VirtualMachine", func() { + pvcs := []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "clone-from-vm-pvc", + Annotations: map[string]string{ + "csi.vsphere.volume-accessible-topology": `[{"topology.kubernetes.io/zone":"zone-is-ignored"}]`, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + DataSourceRef: &corev1.TypedObjectReference{ + APIGroup: ptr.To(vmopv1.GroupVersion.Group), + Kind: "VirtualMachine", + Name: "source-vm", + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + } + zones, err := kubeutil.GetPVCZoneConstraints(nil, pvcs) + Expect(err).ToNot(HaveOccurred()) + Expect(zones).To(BeNil()) + }) + + It("still applies zone constraints when DataSourceRef is not a VirtualMachine", func() { + pvcs := []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "restore-from-snapshot-pvc", + Annotations: map[string]string{ + "csi.vsphere.volume-accessible-topology": `[{"topology.kubernetes.io/zone":"zone-from-snapshot-pvc"}]`, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + DataSourceRef: &corev1.TypedObjectReference{ + APIGroup: ptr.To("snapshot.storage.k8s.io"), + Kind: "VolumeSnapshot", + Name: "src-snap", + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + }, + }, + } + zones, err := kubeutil.GetPVCZoneConstraints(nil, pvcs) + Expect(err).ToNot(HaveOccurred()) + Expect(zones.UnsortedList()).To(ConsistOf("zone-from-snapshot-pvc")) + }) + }) }) var _ = DescribeTableSubtree("GetPVCZoneConstraints Table", diff --git a/pkg/util/vmopv1/features.go b/pkg/util/vmopv1/features.go index bd6475da9..cb5fed7fb 100644 --- a/pkg/util/vmopv1/features.go +++ b/pkg/util/vmopv1/features.go @@ -128,7 +128,7 @@ const ( // FeatureVersionNetExtraConfig refers to VM network extra config (NIC type // and related fields) schema upgrade and backfill. - FeatureVersionNetExtraConfig // 8 + FeatureVersionTelcoVMServiceAPI // 8 ) const ( @@ -139,7 +139,7 @@ const ( FeatureVersionAll = FeatureVersionBase | FeatureVersionVMSharedDisks | FeatureVersionAllDisksArePVCs | - FeatureVersionNetExtraConfig // 15 + FeatureVersionTelcoVMServiceAPI // 15 ) // FeatureVersions returns all possible, valid FeatureVersion elements. @@ -148,7 +148,7 @@ func FeatureVersions() []FeatureVersion { FeatureVersionBase, FeatureVersionVMSharedDisks, FeatureVersionAllDisksArePVCs, - FeatureVersionNetExtraConfig, + FeatureVersionTelcoVMServiceAPI, } } @@ -234,8 +234,8 @@ func ActivatedFeatureVersion(ctx context.Context) FeatureVersion { if f.AllDisksArePVCs || f.VMSharedDisks { v.Set(FeatureVersionAllDisksArePVCs) } - if f.VMExtraConfig { - v.Set(FeatureVersionNetExtraConfig) + if f.TelcoVMServiceAPI { + v.Set(FeatureVersionTelcoVMServiceAPI) } return v } diff --git a/pkg/util/vmopv1/features_test.go b/pkg/util/vmopv1/features_test.go index dc4c52ce6..f5b5590a9 100644 --- a/pkg/util/vmopv1/features_test.go +++ b/pkg/util/vmopv1/features_test.go @@ -28,18 +28,18 @@ var _ = DescribeTable("IsVirtualMachineSchemaUpgraded", annotationsNil, vmSharedDisks, allDisksArePVCs, - vmExtraConfig bool, + telcoVMServiceAPI bool, expectErr bool, expectedErr error, ) { - ctx := pkgcfg.WithConfig(pkgcfg.Config{ - BuildVersion: buildVersion, - Features: pkgcfg.FeatureStates{ - VMSharedDisks: vmSharedDisks, - AllDisksArePVCs: allDisksArePVCs, - VMExtraConfig: vmExtraConfig, - }, - }) + ctx := pkgcfg.WithConfig(pkgcfg.Config{ + BuildVersion: buildVersion, + Features: pkgcfg.FeatureStates{ + VMSharedDisks: vmSharedDisks, + AllDisksArePVCs: allDisksArePVCs, + TelcoVMServiceAPI: telcoVMServiceAPI, + }, + }) vm := vmopv1.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{ @@ -136,7 +136,7 @@ var _ = DescribeTable("IsVirtualMachineSchemaUpgraded", nil, ), Entry( - "all annotations are set correctly with VMExtraConfig", + "all annotations are set correctly with TelcoVMServiceAPI", "1.2.3-test", ptr.To("1.2.3-test"), ptr.To(vmopv1.GroupVersion.Version), @@ -344,7 +344,7 @@ var _ = DescribeTable("IsVirtualMachineSchemaUpgraded", }, ), Entry( - "feature version annotation does not match when VMExtraConfig is enabled", + "feature version annotation does not match when TelcoVMServiceAPI is enabled", "1.2.3-test", ptr.To("1.2.3-test"), ptr.To(vmopv1.GroupVersion.Version), @@ -433,7 +433,7 @@ var _ = Describe("FeatureVersion", func() { Entry("Base + VMSharedDisks is valid", vmopv1util.FeatureVersionBase|vmopv1util.FeatureVersionVMSharedDisks, true), Entry("Base + AllDisksArePVCs is valid", vmopv1util.FeatureVersionBase|vmopv1util.FeatureVersionAllDisksArePVCs, true), Entry("All is valid", vmopv1util.FeatureVersionAll, true), - Entry("NetExtraConfig alone is valid", vmopv1util.FeatureVersionNetExtraConfig, true), + Entry("TelcoVMServiceAPI alone is valid", vmopv1util.FeatureVersionTelcoVMServiceAPI, true), Entry("invalid bit 16 is invalid", vmopv1util.FeatureVersion(16), false), Entry("invalid bit 255 is invalid", vmopv1util.FeatureVersion(255), false), ) @@ -470,7 +470,7 @@ var _ = Describe("FeatureVersion", func() { Entry("All has base", vmopv1util.FeatureVersionAll, vmopv1util.FeatureVersionBase, true), Entry("All has VMSharedDisks", vmopv1util.FeatureVersionAll, vmopv1util.FeatureVersionVMSharedDisks, true), Entry("All has AllDisksArePVCs", vmopv1util.FeatureVersionAll, vmopv1util.FeatureVersionAllDisksArePVCs, true), - Entry("All has NetExtraConfig", vmopv1util.FeatureVersionAll, vmopv1util.FeatureVersionNetExtraConfig, true), + Entry("All has TelcoVMServiceAPI", vmopv1util.FeatureVersionAll, vmopv1util.FeatureVersionTelcoVMServiceAPI, true), ) }) @@ -495,7 +495,7 @@ var _ = Describe("FeatureVersion", func() { fv.Set(vmopv1util.FeatureVersionBase) fv.Set(vmopv1util.FeatureVersionVMSharedDisks) fv.Set(vmopv1util.FeatureVersionAllDisksArePVCs) - fv.Set(vmopv1util.FeatureVersionNetExtraConfig) + fv.Set(vmopv1util.FeatureVersionTelcoVMServiceAPI) Ω(fv).Should(Equal(vmopv1util.FeatureVersionAll)) }) @@ -521,7 +521,7 @@ var _ = Describe("ParseFeatureVersion", func() { Entry("AllDisksArePVCs", "4", vmopv1util.FeatureVersionAllDisksArePVCs), Entry("Base + AllDisksArePVCs", "5", vmopv1util.FeatureVersionBase|vmopv1util.FeatureVersionAllDisksArePVCs), Entry("VMSharedDisks + AllDisksArePVCs", "6", vmopv1util.FeatureVersionVMSharedDisks|vmopv1util.FeatureVersionAllDisksArePVCs), - Entry("NetExtraConfig", "8", vmopv1util.FeatureVersionNetExtraConfig), + Entry("TelcoVMServiceAPI", "8", vmopv1util.FeatureVersionTelcoVMServiceAPI), Entry("All", "15", vmopv1util.FeatureVersionAll), Entry("invalid negative", "-1", vmopv1util.FeatureVersionEmpty), Entry("invalid text", "abc", vmopv1util.FeatureVersionEmpty), @@ -533,12 +533,12 @@ var _ = Describe("ParseFeatureVersion", func() { var _ = Describe("ActivatedFeatureVersion", func() { DescribeTable("returns activated feature version", - func(vmSharedDisks, allDisksArePVCs, vmExtraConfig bool, expected vmopv1util.FeatureVersion) { + func(vmSharedDisks, allDisksArePVCs, telcoVMServiceAPI bool, expected vmopv1util.FeatureVersion) { ctx := pkgcfg.WithConfig(pkgcfg.Config{ Features: pkgcfg.FeatureStates{ VMSharedDisks: vmSharedDisks, AllDisksArePVCs: allDisksArePVCs, - VMExtraConfig: vmExtraConfig, + TelcoVMServiceAPI: telcoVMServiceAPI, }, }) Ω(vmopv1util.ActivatedFeatureVersion(ctx)).Should(Equal(expected)) @@ -547,7 +547,7 @@ var _ = Describe("ActivatedFeatureVersion", func() { Entry("VMSharedDisks only", true, false, false, vmopv1util.FeatureVersionBase|vmopv1util.FeatureVersionVMSharedDisks|vmopv1util.FeatureVersionAllDisksArePVCs), Entry("AllDisksArePVCs only", false, true, false, vmopv1util.FeatureVersionBase|vmopv1util.FeatureVersionAllDisksArePVCs), Entry("both disk-related features", true, true, false, vmopv1util.FeatureVersionBase|vmopv1util.FeatureVersionVMSharedDisks|vmopv1util.FeatureVersionAllDisksArePVCs), - Entry("VMExtraConfig only", false, false, true, vmopv1util.FeatureVersionBase|vmopv1util.FeatureVersionNetExtraConfig), + Entry("TelcoVMServiceAPI only", false, false, true, vmopv1util.FeatureVersionBase|vmopv1util.FeatureVersionTelcoVMServiceAPI), Entry("every activated feature bit", true, true, true, vmopv1util.FeatureVersionAll), ) }) diff --git a/pkg/util/vmopv1/instancestorage_test.go b/pkg/util/vmopv1/instancestorage_test.go index b4306f550..113a15c24 100644 --- a/pkg/util/vmopv1/instancestorage_test.go +++ b/pkg/util/vmopv1/instancestorage_test.go @@ -16,7 +16,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - apirecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" @@ -292,7 +292,7 @@ var _ = Describe("ReconcileInstanceStoragePVCs", func() { Context: baseCtx, VM: vm, } - recorder = record.New(apirecord.NewFakeRecorder(100)) + recorder = record.New(events.NewFakeRecorder(100)) // Initialize helper function to create instance storage PVCs with common defaults newInstanceStoragePVC = func(opts ...func(*corev1.PersistentVolumeClaim)) *corev1.PersistentVolumeClaim { diff --git a/pkg/util/vsphere/fault/fault.go b/pkg/util/vsphere/fault/fault.go new file mode 100644 index 000000000..cb6f5e934 --- /dev/null +++ b/pkg/util/vsphere/fault/fault.go @@ -0,0 +1,50 @@ +package fault + +import ( + "strings" + + vimtypes "github.com/vmware/govmomi/vim25/types" +) + +// LocalizedMessagesFromFault extracts all localized messages from a LocalizedMethodFault, +// recursively traversing nested faults. +// +// This function handles the special case of NoCompatibleHost hierarchies by: +// - Extracting the top-level LocalizedMessage only +// - Recursively unwrapping Error field chains (e.g., NoCompatibleHost.Error) +// +// Returns an empty slice if both the LocalizedMessage and Fault are nil/empty. +func LocalizedMessagesFromFault(lmf vimtypes.LocalizedMethodFault) []string { + var messages []string + + if lmf.LocalizedMessage != "" { + msg := strings.TrimRight(lmf.LocalizedMessage, ": \n\r\t") + if msg != "" { + messages = append(messages, msg) + } + } + + if lmf.Fault == nil { + return messages + } + + // Only extract messages from the top level NoCompatibleHost fault. + // Placement APIs return faults from single host per ResourcePool, we won't end up with too many errors. + if nch, ok := lmf.Fault.(*vimtypes.NoCompatibleHost); ok { + for _, nestedLmf := range nch.Error { + messages = append(messages, LocalizedMessagesFromFault(nestedLmf)...) + } + } + + return messages +} + +// LocalizedMessagesFromFaults extracts all localized messages from a slice of LocalizedMethodFault. +// Returns an empty slice if the input faults slice is empty. +func LocalizedMessagesFromFaults(faults []vimtypes.LocalizedMethodFault) []string { + var allMessages []string + for _, f := range faults { + allMessages = append(allMessages, LocalizedMessagesFromFault(f)...) + } + return allMessages +} diff --git a/pkg/util/vsphere/fault/fault_test.go b/pkg/util/vsphere/fault/fault_test.go new file mode 100644 index 000000000..99491a899 --- /dev/null +++ b/pkg/util/vsphere/fault/fault_test.go @@ -0,0 +1,204 @@ +package fault + +import ( + "testing" + + vimtypes "github.com/vmware/govmomi/vim25/types" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestFault(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Fault Util Suite") +} + +var _ = Describe("LocalizedMessagesFromFault", func() { + + Context("when LocalizedMethodFault has no localized message and fault is nil", func() { + It("should return empty slice", func() { + result := LocalizedMessagesFromFault(vimtypes.LocalizedMethodFault{ + LocalizedMessage: "", + Fault: nil, + }) + Expect(result).To(BeEmpty()) + }) + }) + + Context("when LocalizedMethodFault has only localized message", func() { + It("should return the localized message", func() { + result := LocalizedMessagesFromFault(vimtypes.LocalizedMethodFault{ + LocalizedMessage: "Operation failed", + Fault: nil, + }) + Expect(result).To(Equal([]string{"Operation failed"})) + }) + }) + + Context("when LocalizedMessage has trailing whitespaces and colons", func() { + It("should trim them from the extracted message", func() { + result := LocalizedMessagesFromFault(vimtypes.LocalizedMethodFault{ + LocalizedMessage: "Operation failed due to: \n\t", + Fault: nil, + }) + Expect(result).To(Equal([]string{"Operation failed due to"})) + }) + }) + + Describe("when LocalizedMethodFault is NoCompatibleHost", func() { + var ( + baseFault *vimtypes.NoCompatibleHost + lmf vimtypes.LocalizedMethodFault + ) + + BeforeEach(func() { + baseFault = &vimtypes.NoCompatibleHost{ + Error: []vimtypes.LocalizedMethodFault{ + { + LocalizedMessage: "First error", + Fault: nil, + }, + { + LocalizedMessage: "Second error", + Fault: nil, + }, + }, + } + lmf = vimtypes.LocalizedMethodFault{ + LocalizedMessage: "Top level", + Fault: baseFault, + } + }) + + Context("when error slice is empty", func() { + It("should only return top level message", func() { + baseFault.Error = []vimtypes.LocalizedMethodFault{} + result := LocalizedMessagesFromFault(lmf) + Expect(result).To(Equal([]string{"Top level"})) + }) + }) + + Context("when fault has NoCompatibleHost with nested errors", func() { + It("should recursively collect all localized messages", func() { + lmf.LocalizedMessage = "No compatible host" + result := LocalizedMessagesFromFault(lmf) + Expect(result).To(Equal([]string{ + "No compatible host", + "First error", + "Second error", + })) + }) + }) + + Context("when nested fault has empty localized message", func() { + It("should skip the empty message", func() { + baseFault.Error[0].LocalizedMessage = "" + baseFault.Error[1].LocalizedMessage = "Valid message" + result := LocalizedMessagesFromFault(lmf) + Expect(result).To(Equal([]string{ + "Top level", + "Valid message", + })) + }) + }) + + Context("when deeply nested with mixed nil and non-nil faults", func() { + It("should handle gracefully", func() { + lmf.LocalizedMessage = "Level 0" + baseFault.Error = []vimtypes.LocalizedMethodFault{ + { + LocalizedMessage: "Level 1a", + Fault: nil, + }, + { + LocalizedMessage: "Level 1b", + Fault: &vimtypes.NoCompatibleHost{ + Error: []vimtypes.LocalizedMethodFault{ + { + LocalizedMessage: "Level 2a", + Fault: nil, + }, + { + LocalizedMessage: "", + Fault: nil, + }, + { + LocalizedMessage: "Level 2b", + Fault: nil, + }, + }, + }, + }, + } + result := LocalizedMessagesFromFault(lmf) + Expect(result).To(Equal([]string{ + "Level 0", + "Level 1a", + "Level 1b", + "Level 2a", + "Level 2b", + })) + }) + }) + }) +}) + +var _ = Describe("LocalizedMessagesFromFaults", func() { + Describe("Basic batch processing", func() { + Context("when faults is empty or nil", func() { + It("should return empty slice for empty slice", func() { + result := LocalizedMessagesFromFaults([]vimtypes.LocalizedMethodFault{}) + Expect(result).To(BeEmpty()) + }) + + It("should return empty slice for nil slice", func() { + var faults []vimtypes.LocalizedMethodFault + result := LocalizedMessagesFromFaults(faults) + Expect(result).To(BeEmpty()) + }) + }) + + Context("when faults contains multiple entries", func() { + It("should collect messages from all faults", func() { + result := LocalizedMessagesFromFaults([]vimtypes.LocalizedMethodFault{ + { + LocalizedMessage: "Fault 1 message", + Fault: nil, + }, + { + LocalizedMessage: "Fault 2 message", + Fault: nil, + }, + }) + Expect(result).To(Equal([]string{ + "Fault 1 message", + "Fault 2 message", + })) + }) + }) + + Context("when some faults have nil messages", func() { + It("should skip empty messages", func() { + result := LocalizedMessagesFromFaults([]vimtypes.LocalizedMethodFault{ + { + LocalizedMessage: "First", + Fault: nil, + }, + { + LocalizedMessage: "", + Fault: nil, + }, + { + LocalizedMessage: "Third", + Fault: nil, + }, + }) + Expect(result).To(Equal([]string{ + "First", + "Third", + })) + }) + }) + }) +}) diff --git a/pkg/vmconfig/bootoptions/bootoptions_reconciler.go b/pkg/vmconfig/bootoptions/bootoptions_reconciler.go index 30bd3767d..f7beb18ce 100644 --- a/pkg/vmconfig/bootoptions/bootoptions_reconciler.go +++ b/pkg/vmconfig/bootoptions/bootoptions_reconciler.go @@ -89,7 +89,10 @@ func (r reconciler) Reconcile( vmBootOptions = vm.Spec.BootOptions } - csBootOptions := vimtypes.VirtualMachineBootOptions{} + var csBootOptions vimtypes.VirtualMachineBootOptions + if configSpec.BootOptions != nil { + csBootOptions = *configSpec.BootOptions + } if vmBootOptions.BootDelay != nil { csBootOptions.BootDelay = vmBootOptions.BootDelay.Duration.Milliseconds() diff --git a/pkg/vmconfig/bootoptions/bootoptions_reconciler_test.go b/pkg/vmconfig/bootoptions/bootoptions_reconciler_test.go index 0d6df5e8d..6244516f4 100644 --- a/pkg/vmconfig/bootoptions/bootoptions_reconciler_test.go +++ b/pkg/vmconfig/bootoptions/bootoptions_reconciler_test.go @@ -900,6 +900,222 @@ var _ = Describe("Reconcile", Label(testlabels.V1Alpha4), func() { Expect(eth2.DeviceKey).To(Equal(int32(4002))) }) }) + + When("the VM class has boot options", func() { + BeforeEach(func() { + moVM.Config = nil + }) + Context("bootDelay", func() { + BeforeEach(func() { + configSpec.BootOptions = &vimtypes.VirtualMachineBootOptions{ + BootDelay: 10, + } + }) + When("the VM spec does not specify bootDelay", func() { + It("should keep the value from the configSpec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).ToNot(BeNil()) + Expect(configSpec.BootOptions.BootDelay).To(Equal(int64(10))) + }) + }) + When("the VM spec does specify bootDelay", func() { + BeforeEach(func() { + vm.Spec.BootOptions = &vmopv1.VirtualMachineBootOptions{ + BootDelay: &metav1.Duration{ + Duration: 100 * time.Millisecond, + }, + } + }) + It("should override the value from the configSpec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).ToNot(BeNil()) + Expect(configSpec.BootOptions.BootDelay).To(Equal(int64(100))) + }) + }) + }) + + Context("bootRetry", func() { + BeforeEach(func() { + configSpec.BootOptions = &vimtypes.VirtualMachineBootOptions{ + BootRetryEnabled: ptr.To(true), + } + }) + When("the VM spec does not specify bootRetry", func() { + It("should keep the value from the configSpec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).ToNot(BeNil()) + Expect(configSpec.BootOptions.BootRetryEnabled).ToNot(BeNil()) + Expect(*configSpec.BootOptions.BootRetryEnabled).To(BeTrue()) + }) + }) + When("the VM spec does specify bootRetry", func() { + BeforeEach(func() { + vm.Spec.BootOptions = &vmopv1.VirtualMachineBootOptions{ + BootRetry: vmopv1.VirtualMachineBootOptionsBootRetryDisabled, + } + }) + It("should override the value from the configSpec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).ToNot(BeNil()) + Expect(configSpec.BootOptions.BootRetryEnabled).ToNot(BeNil()) + Expect(*configSpec.BootOptions.BootRetryEnabled).To(BeFalse()) + }) + }) + }) + + Context("bootRetryDelay", func() { + BeforeEach(func() { + configSpec.BootOptions = &vimtypes.VirtualMachineBootOptions{ + BootRetryDelay: 5000, + } + }) + When("the VM spec does not specify bootRetryDelay", func() { + It("should keep the value from the configSpec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).ToNot(BeNil()) + Expect(configSpec.BootOptions.BootRetryDelay).To(Equal(int64(5000))) + }) + }) + When("the VM spec does specify bootRetryDelay", func() { + BeforeEach(func() { + vm.Spec.BootOptions = &vmopv1.VirtualMachineBootOptions{ + BootRetryDelay: &metav1.Duration{ + Duration: 2 * time.Second, + }, + } + }) + It("should override the value from the configSpec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).ToNot(BeNil()) + Expect(configSpec.BootOptions.BootRetryDelay).To(Equal(int64(2000))) + }) + }) + }) + + Context("efiSecureBootEnabled", func() { + BeforeEach(func() { + configSpec.BootOptions = &vimtypes.VirtualMachineBootOptions{ + EfiSecureBootEnabled: ptr.To(true), + } + }) + When("the VM spec does not specify efiSecureBoot", func() { + It("should keep the value from the configSpec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).ToNot(BeNil()) + Expect(configSpec.BootOptions.EfiSecureBootEnabled).ToNot(BeNil()) + Expect(*configSpec.BootOptions.EfiSecureBootEnabled).To(BeTrue()) + }) + }) + When("the VM spec does specify efiSecureBoot", func() { + BeforeEach(func() { + vm.Spec.BootOptions = &vmopv1.VirtualMachineBootOptions{ + EFISecureBoot: vmopv1.VirtualMachineBootOptionsEFISecureBootDisabled, + } + }) + It("should override the value from the configSpec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).ToNot(BeNil()) + Expect(configSpec.BootOptions.EfiSecureBootEnabled).ToNot(BeNil()) + Expect(*configSpec.BootOptions.EfiSecureBootEnabled).To(BeFalse()) + }) + }) + }) + + Context("networkBootProtocol", func() { + BeforeEach(func() { + configSpec.BootOptions = &vimtypes.VirtualMachineBootOptions{ + NetworkBootProtocol: string(vimtypes.VirtualMachineBootOptionsNetworkBootProtocolTypeIpv4), + } + }) + When("the VM spec does not specify networkBootProtocol", func() { + It("should clear boot options when only the class network protocol applied", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).To(BeNil()) + }) + }) + When("the VM spec does specify networkBootProtocol", func() { + BeforeEach(func() { + vm.Spec.BootOptions = &vmopv1.VirtualMachineBootOptions{ + NetworkBootProtocol: vmopv1.VirtualMachineBootOptionsNetworkBootProtocolIP6, + } + }) + It("should override the value from the configSpec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).ToNot(BeNil()) + Expect(configSpec.BootOptions.NetworkBootProtocol).To( + Equal(string(vimtypes.VirtualMachineBootOptionsNetworkBootProtocolTypeIpv6))) + }) + }) + }) + + Context("bootOrder", func() { + BeforeEach(func() { + configSpec.BootOptions = &vimtypes.VirtualMachineBootOptions{ + BootOrder: []vimtypes.BaseVirtualMachineBootOptionsBootableDevice{ + &vimtypes.VirtualMachineBootOptionsBootableCdromDevice{}, + }, + } + }) + When("the VM spec does not specify bootOrder", func() { + It("should clear boot options when only the class boot order applied", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).To(BeNil()) + }) + }) + When("the VM spec does specify bootOrder", func() { + BeforeEach(func() { + vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + {Name: "eth0"}, + }, + } + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: "disk-0", + DiskUUID: "6000C298-df15-fe89-ddcb-8ea33329595d", + }, + } + ethDevice := &vimtypes.VirtualVmxnet3{} + ethDevice.Key = 4000 + moVM = mo.VirtualMachine{ + Config: &vimtypes.VirtualMachineConfigInfo{ + Hardware: vimtypes.VirtualHardware{ + Device: []vimtypes.BaseVirtualDevice{ + &vimtypes.VirtualCdrom{}, + &vimtypes.VirtualDisk{ + VirtualDevice: vimtypes.VirtualDevice{ + Key: 2000, + Backing: &vimtypes.VirtualDiskFlatVer2BackingInfo{ + Uuid: "6000C298-df15-fe89-ddcb-8ea33329595d", + }, + }, + }, + ethDevice, + }, + }, + }, + } + vm.Spec.BootOptions = &vmopv1.VirtualMachineBootOptions{ + BootOrder: []vmopv1.VirtualMachineBootOptionsBootableDevice{ + { + Type: vmopv1.VirtualMachineBootOptionsBootableDiskDevice, + Name: "disk-0", + }, + }, + } + }) + It("should override the boot order from the configSpec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec.BootOptions).ToNot(BeNil()) + Expect(configSpec.BootOptions.BootOrder).To(HaveLen(1)) + Expect(configSpec.BootOptions.BootOrder[0]).To( + BeAssignableToTypeOf(&vimtypes.VirtualMachineBootOptionsBootableDiskDevice{})) + disk0 := configSpec.BootOptions.BootOrder[0].(*vimtypes.VirtualMachineBootOptionsBootableDiskDevice) + Expect(disk0.DeviceKey).To(Equal(int32(2000))) + }) + }) + }) + }) }) }) }) diff --git a/pkg/webconsolevalidation/server_test.go b/pkg/webconsolevalidation/server_test.go index f5a082f29..45c258d8d 100644 --- a/pkg/webconsolevalidation/server_test.go +++ b/pkg/webconsolevalidation/server_test.go @@ -6,6 +6,7 @@ package webconsolevalidation_test import ( "fmt" + "net" "net/http" "net/http/httptest" "time" @@ -30,6 +31,60 @@ func serverUnitTests() { serverAddr = "localhost:8080" ) + // testWildcardServer starts a server bound with bindFmt + // (a fmt format string accepting a single int port, e.g. "[::]:%d" + // or ":%d") and runs wildcardBindSpecs against it. + testWildcardServer := func(bindFmt string) { + var ( + server *webconsolevalidation.Server + serverPort int + ) + + BeforeEach(func() { + serverPort = getAvailablePort() + var err error + server, err = webconsolevalidation.NewServer( + fmt.Sprintf(bindFmt, serverPort), + serverPath, + builder.NewFakeClient(), + ) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + _ = server.Run() + }() + + Eventually(func(g Gomega) { + url := fmt.Sprintf("http://127.0.0.1:%d%s", serverPort, serverPath) + resp, err := http.Get(url) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp).NotTo(BeNil()) + g.Expect(resp.Body.Close()).To(Succeed()) + }).WithTimeout(2 * time.Second).Should(Succeed()) + }) + + It("should accept connections via IPv4 loopback (127.0.0.1)", func() { + url := fmt.Sprintf("http://127.0.0.1:%d%s?uuid=test&namespace=test", serverPort, serverPath) + resp, err := http.Get(url) + Expect(err).NotTo(HaveOccurred()) + Expect(resp).NotTo(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) + Expect(resp.Body.Close()).To(Succeed()) + }) + + It("should accept connections via IPv6 loopback ([::1])", func() { + url := fmt.Sprintf("http://[::1]:%d%s?uuid=test&namespace=test", serverPort, serverPath) + resp, err := http.Get(url) + if err != nil { + Skip("IPv6 not available on this system: " + err.Error()) + } + Expect(resp).NotTo(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) + Expect(resp.Body.Close()).To(Succeed()) + }) + } + Context("NewServer", func() { When("Server addr or path is empty", func() { @@ -58,36 +113,52 @@ func serverUnitTests() { }) }) + + When("IPv6 dual-stack bind address is provided", func() { + + It("should initialize a new Server with [::] address", func() { + server, err := webconsolevalidation.NewServer("[::]:8080", serverPath, fake.NewFakeClient()) + Expect(err).NotTo(HaveOccurred()) + Expect(server).NotTo(BeNil()) + Expect(server.Addr).To(Equal("[::]:8080")) + }) + + It("should initialize a new Server with [::1] loopback address", func() { + server, err := webconsolevalidation.NewServer("[::1]:8080", serverPath, fake.NewFakeClient()) + Expect(err).NotTo(HaveOccurred()) + Expect(server).NotTo(BeNil()) + Expect(server.Addr).To(Equal("[::1]:8080")) + }) + }) }) Context("RunServer", func() { - It("should start the server at the given address and path", func(done Done) { - - server := &webconsolevalidation.Server{ - Addr: serverAddr, - Path: serverPath, - KubeClient: builder.NewFakeClient(), - } + It("should start the server at the given address and path", func() { + srv, err := webconsolevalidation.NewServer(serverAddr, serverPath, builder.NewFakeClient()) + Expect(err).NotTo(HaveOccurred()) go func() { - Expect(server.Run()).To(Succeed()) + defer GinkgoRecover() + _ = srv.Run() }() - // Wait for the server to start. - time.Sleep(100 * time.Millisecond) - - resp, err := http.Get("http://" + serverAddr + serverPath) - Expect(err).NotTo(HaveOccurred()) - Expect(resp).NotTo(BeNil()) - // Verify the server path is correct. - Expect(resp.StatusCode).NotTo(Equal(http.StatusNotFound)) - Expect(resp.Body).NotTo(BeNil()) - Expect(resp.Body.Close()).To(Succeed()) + Eventually(func(g Gomega) { + resp, err := http.Get("http://" + serverAddr + serverPath) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp).NotTo(BeNil()) + g.Expect(resp.StatusCode).NotTo(Equal(http.StatusNotFound)) + g.Expect(resp.Body.Close()).To(Succeed()) + }).WithTimeout(2 * time.Second).Should(Succeed()) + }) - close(done) - }, 1.0) // Time out this after 1 second. + Context("RunServer with [::] bind address", func() { + testWildcardServer("[::]:%d") + }) + Context("RunServer with default bind address", func() { + testWildcardServer(":%d") + }) }) Context("HandleWebConsoleValidation", func() { @@ -243,3 +314,15 @@ func fakeValidationRequest(url string, server webconsolevalidation.Server) int { return response.StatusCode } + +func getAvailablePort() int { + listener, err := net.Listen("tcp", "[::]:0") + if err != nil { + listener, err = net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + } + defer func() { + Expect(listener.Close()).To(Succeed()) + }() + return listener.Addr().(*net.TCPAddr).Port +} diff --git a/services/vm-watcher/vm_watcher_service_test.go b/services/vm-watcher/vm_watcher_service_test.go index 046f3b3cd..2e4d5ad05 100644 --- a/services/vm-watcher/vm_watcher_service_test.go +++ b/services/vm-watcher/vm_watcher_service_test.go @@ -127,7 +127,7 @@ var _ = Describe( g.Expect(vsClient.Valid()).To(BeTrue()) g.Expect(atomic.LoadInt32(&numNewClientCalls)).To(Equal(int32(1))) - }).Should(Succeed()) + }, "2s").Should(Succeed()) By("log out the client session", func() { vsClientMu.Lock() @@ -173,7 +173,7 @@ var _ = Describe( g.Expect(vsClient.Valid()).To(BeTrue()) g.Expect(atomic.LoadInt32(&numNewClientCalls)).To(Equal(int32(1))) - }).Should(Succeed()) + }, "2s").Should(Succeed()) By("invalidate the credentials", func() { vsClientMu.Lock() @@ -235,7 +235,7 @@ var _ = Describe( g.Expect(vsClient.Valid()).To(BeTrue()) g.Expect(atomic.LoadInt32(&numNewClientCalls)).To(Equal(int32(1))) - }).Should(Succeed()) + }, "2s").Should(Succeed()) By("invalidate the port", func() { vsClientMu.Lock() @@ -292,7 +292,7 @@ var _ = Describe( for _, zone := range zoneList.Items { g.Expect(zone.Finalizers).To(ConsistOf(zonectrl.Finalizer)) } - }).Should(Succeed()) + }, "2s").Should(Succeed()) }) }) diff --git a/test/builder/fake.go b/test/builder/fake.go index de85aacf3..ef1f8450f 100644 --- a/test/builder/fake.go +++ b/test/builder/fake.go @@ -7,7 +7,7 @@ package builder import ( "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - clientgorecord "k8s.io/client-go/tools/record" + "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -89,7 +89,7 @@ func KnownObjectTypes() []client.Object { } func NewFakeRecorder() (record.Recorder, chan string) { - fakeEventRecorder := clientgorecord.NewFakeRecorder(1024) + fakeEventRecorder := events.NewFakeRecorder(1024) recorder := record.New(fakeEventRecorder) return recorder, fakeEventRecorder.Events } diff --git a/test/builder/vcsim_test_context.go b/test/builder/vcsim_test_context.go index 7088716c0..ba3333380 100644 --- a/test/builder/vcsim_test_context.go +++ b/test/builder/vcsim_test_context.go @@ -1001,6 +1001,10 @@ func (c *TestContextForVCSim) SimulatorContext() *simulator.Context { return c.model.Service.Context } +func (c *TestContextForVCSim) SimulatorService() *simulator.Service { + return c.model.Service +} + func (c *TestContextForVCSim) ContentLibraryItemTemplate(srcVMName, templateName string) { clID := c.ContentLibraryID Expect(clID).ToNot(BeEmpty()) diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 000000000..7510a64bd --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,272 @@ +# VM Operator E2E Test Framework + +This directory contains the End-to-End (E2E) test framework for VM Operator, providing comprehensive testing capabilities. + +## Overview + +The E2E framework validates VM Operator functionality by: +- Creating and managing VirtualMachine resources in real Kubernetes clusters +- Interacting with actual vSphere infrastructure +- Testing feature functionality across different networking topologies (VDS/NSX) +- Validating integration with WCP components and services + +## Architecture + +``` +test/e2e/ +├── vmservice/ # Main test suite for VM Service functionality +│ ├── config/ # Test configuration files (wcp.yaml) +│ ├── vmservice/ # Core VM lifecycle and feature tests +│ │ └── virtualmachine/ # VM-specific test scenarios +│ ├── vmserviceapp/ # Application deployment tests +│ ├── lib/ # Test libraries and helpers +│ ├── common/ # Shared test utilities +│ └── consts/ # Test constants and labels +├── framework/ # Core testing framework components +├── infrastructure/ # Infrastructure interaction (vSphere, WCP) +│ └── vsphere/ # vSphere-specific clients and utilities +├── manifestbuilders/ # Test manifest generation utilities +├── utils/ # General test utilities +├── fixtures/ # Test data and YAML fixtures +└── testutils/ # Additional testing utilities +``` + +## Quick Start + +### Prerequisites + +1. **Access to a WCP cluster** with VM Operator deployed +2. **vSphere environment** with proper permissions +3. **Kubeconfig** for the WCP supervisor cluster +4. **Go 1.21+** installed locally + +### Configuration + +1. **Set up the env** for your WCP cluster: +The following will export all the required environment variables +required for the test and download the kubeconfig from the Supervisor cluster. +The kubeconfig will be stored under the directory `~/.kube/.kubeconfig` +and copied to `~/.kube/wcp-config` which the E2E test reads from. + +```bash +source ./hack/setup-testbed-env.sh --e2e +``` + +2. **Running Tests** + +#### All E2E Tests +```bash +make test-e2e +``` + +Use `make test-e2e` (auto-detects), `make test-e2e-prebuilt`, or `make test-e2e-ginkgo`. The first uses `./e2e-tests` at the repo root when present (e.g. slim E2E image). Override the binary path with `E2E_PREBUILT_BINARY=/path/to/binary`. + +#### Test Categories + +**Smoke Tests** (quick validation): +```bash +make e2e-smoke +``` + +**Core Functional Tests** (comprehensive functionality): +```bash +make e2e-core +``` + +**Extended Functional Tests** (advanced features): +```bash +make e2e-extended +``` + +#### Focused Test Execution + +**Run specific test patterns**: +```bash +make test-e2e TEST_FOCUS="VM-HARDWARE" +``` + +Precompiled `./e2e-tests`: pass `-e2e.*` and `--ginkgo.*` in one contiguous argument list (as `make test-e2e-prebuilt` does). Do **not** put `--` between them; otherwise `--ginkgo.focus`, `--ginkgo.label-filter`, etc. are ignored and nearly all specs still run. + +**Skip specific tests**: +```bash +make test-e2e TEST_SKIP="encryption|snapshot" +``` + +**Label-based filtering**: +```bash +make test-e2e LABEL_FILTER="networking && !encryption" +``` + +**Flake retry**: +```bash +make test-e2e FLAKE_ATTEMPTS=3 +``` + +**Use specific namespace**: +```bash +E2E_NAMESPACE=my-test-ns make e2e-smoke +``` + +**Combine multiple options**: +```bash +E2E_NAMESPACE=debug-ns TEST_FOCUS="VM-HARDWARE" FLAKE_ATTEMPTS=2 make e2e-core +``` + +## Test Categories and Labels + +Tests are organized using Ginkgo labels for easy filtering: + +### Core Labels +- `smoke` - Quick validation tests +- `core-functional` - Core feature tests +- `extended-functional` - Advanced feature tests + +## Configuration + +### Main Configuration File + +The primary configuration is in `vmservice/config/wcp.yaml`: + +### Environment Variables + +| Variable | Description | Default | Makefile Support | +|----------|-------------|---------|------------------| +| `NETWORK` | Networking topology (vds/nsx) | `vds` | ❌ | +| `STORAGE_CLASS` | Storage class for VMs | `wcpglobal-storage-profile` | ❌ | +| `E2E_NAMESPACE` | Fixed namespace for tests | Random namespace | ✅ | +| `TEST_FOCUS` | Ginkgo focus pattern | All tests | ✅ | +| `TEST_SKIP` | Ginkgo skip pattern | No skips | ✅ | +| `LABEL_FILTER` | Ginkgo label filter | No filter | ✅ | +| `FLAKE_ATTEMPTS` | Retry count for flaky tests | No retries | ✅ | + +#### Namespace Management + +The `E2E_NAMESPACE` variable controls which Kubernetes namespace the tests use. It's automatically passed through all Makefile targets to the test configuration. + +**Default behavior** (random namespace): +```bash +make e2e-smoke # Creates random namespace like "vmservice-e2e-abc123" +``` + +**Fixed namespace** (reuse existing namespace): +```bash +make e2e-smoke E2E_NAMESPACE=my-test-ns # Uses "my-test-ns" namespace +``` + +**Benefits of fixed namespace**: +- Faster test execution (no namespace creation/deletion) +- Easier debugging +- Shared test environment setup +- Consistent test isolation + +## Writing New Tests + +### Test Structure + +Follow the established patterns in `vmservice/vmservice/virtualmachine/`: + +```go +var _ = Describe("My Feature", Label("my-feature"), func() { + var ( + ctx context.Context + vm *vmopv1.VirtualMachine + vmKey types.NamespacedName + ) + + BeforeEach(func() { + // Setup test environment + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vm", + Namespace: namespace, + }, + Spec: vmopv1.VirtualMachineSpec{ + // VM specification + }, + } + }) + + When("feature is enabled", func() { + It("should work correctly", func() { + // Test implementation + Expect(client.Create(ctx, vm)).To(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(client.Get(ctx, vmKey, vm)).To(Succeed()) + g.Expect(vm.Status.Phase).To(Equal(vmopv1.Created)) + }).Should(Succeed()) + }) + }) +}) +``` + +### Best Practices + +1. **Use descriptive test names** that explain the scenario +2. **Apply appropriate labels** for filtering +3. **Use Eventually/Consistently** for async operations +4. **Clean up resources** in AfterEach hooks +5. **Test both positive and negative cases** +6. **Use manifest builders** for complex objects +7. **Validate status conditions** and events + +### Helper Libraries + +#### VM Operations (`lib/vmoperator`) +```go +vmoperator.WaitForVirtualMachineCreation(ctx, config, client, vm) +vmoperator.VerifyVirtualMachinePowerState(ctx, config, client, vm, vmopv1.VirtualMachinePowerStateOn) +``` + +#### Manifest Builders (`manifestbuilders`) +```go +vm := manifestbuilders.CreateVMManifest(manifestbuilders.CreateVMParams{ + VMName: "test-vm", + Namespace: namespace, + VMClass: "best-effort-small", + VMImage: "photon-5.0", +}) +``` + +#### Infrastructure Access (`infrastructure/vsphere`) +```go +vcClient := vcenter.NewVimClientFromKubeconfig(ctx, kubeconfigPath) +``` + +### Debug Mode + +**Enable verbose output**: +```bash +make test-e2e GINKGO_ARGS="-v --trace" +``` + +**Test-specific debugging**: +```go +By("Debug checkpoint: VM creation") +framework.Logf("VM status: %+v", vm.Status) +``` + +## Development Workflow + +1. **Write tests** following established patterns +2. **Add appropriate labels** for categorization +3. **Test locally** with `make test-e2e TEST_FOCUS="MyFeature"` +4. **Clean up artifacts** with `make clean` before Docker builds +5. **Update documentation** for new test categories + +### Cleanup and Optimization + +**Remove build artifacts** before Docker operations: +```bash +make clean # Removes *.test binaries, coverage files, and build artifacts +``` + +## Integration Points + +- **VM Operator Controllers** - Tests controller reconciliation +- **vSphere APIs** - Validates infrastructure integration +- **WCP Services** - Tests platform service interactions +- **Kubernetes APIs** - Validates CRD and webhook behavior +- **Feature Gates** - Tests feature flag functionality + +This E2E framework ensures VM Operator reliability across diverse environments and use cases, providing confidence in production deployments. \ No newline at end of file diff --git a/test/e2e/appple2e/lib/supervisorservice.go b/test/e2e/appple2e/lib/supervisorservice.go new file mode 100644 index 000000000..d322db3b6 --- /dev/null +++ b/test/e2e/appple2e/lib/supervisorservice.go @@ -0,0 +1,18 @@ +package lib + +type SupervisorServiceType int + +const ( + Custom SupervisorServiceType = iota + VSphereApp +) + +func (sst SupervisorServiceType) String() string { + return [...]string{"custom-service", "vsphere-app-service", "carvel-service"}[sst] +} + +const ( + VapiNotFoundErrMsg = "Server error: com.vmware.vapi.std.errors.NotFound" + VapiUnauthorizedErrMsg = "Server error: com.vmware.vapi.std.errors.Unauthorized" + VapiAlreadyInDesiredStateErrMsg = "Server error: com.vmware.vapi.std.errors.AlreadyInDesiredState" +) diff --git a/test/e2e/appple2e/util/util.go b/test/e2e/appple2e/util/util.go new file mode 100644 index 000000000..0f5d51031 --- /dev/null +++ b/test/e2e/appple2e/util/util.go @@ -0,0 +1,25 @@ +package util + +import ( + "fmt" + "strings" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + e2essh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" +) + +func IsFSSEnabled(sshCommandRunner e2essh.SSHCommandRunner, fss string) (bool, string) { + output, err := sshCommandRunner.RunCommand(fmt.Sprintf("python /usr/sbin/feature-state-wrapper.py %s", fss)) + if err != nil || strings.TrimSpace(string(output)) != "enabled" { + msg := fmt.Sprintf("%s FSS is not enabled on the testbed", fss) + if err != nil { + msg = fmt.Sprintf("failed to get FSS for %s", fss) + e2eframework.Logf("%s, err: %s", msg, err.Error()) + } + + return false, msg + } + + return true, "" +} diff --git a/test/e2e/fixtures/yaml.go b/test/e2e/fixtures/yaml.go new file mode 100644 index 000000000..f3eec332a --- /dev/null +++ b/test/e2e/fixtures/yaml.go @@ -0,0 +1,49 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package fixtures + +import ( + "os" + "path/filepath" + "runtime" + + "k8s.io/kubernetes/test/e2e/framework/testfiles" +) + +func ReadFile(path, file string) string { + return string(ReadFileBytes(path, file)) +} + +func ReadFileBytes(path, file string) []byte { + from := filepath.Join(path, file) + + out, err := testfiles.Read(from) + if err != nil { + _, thisFile, _, ok := runtime.Caller(0) + if ok { + repoRoot := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(thisFile)))) + + absPath := filepath.Join(repoRoot, from) + if out, err2 := os.ReadFile(absPath); err2 == nil { + return out + } + } + + panic(err) + } + + return out +} diff --git a/test/e2e/fixtures/yaml/podvm/podvm.yaml.in b/test/e2e/fixtures/yaml/podvm/podvm.yaml.in new file mode 100644 index 000000000..d5d88195a --- /dev/null +++ b/test/e2e/fixtures/yaml/podvm/podvm.yaml.in @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: Pod +metadata: + {{- if .Name}} + name: {{.Name}} + {{- else}} + name: jumpbox + {{- end}} + namespace: {{.Namespace}} +spec: + containers: + - image: "dockerhub.packages.vcfd.broadcom.net/library/photon:3.0" + {{- if .Name}} + name: {{.Name}} + {{- else}} + name: jumpbox + {{- end}} + securityContext: + runAsUser: 0 + {{- if or .MemoryRequest .MemoryLimit .CPURequest .CPULimit }} + resources: + {{- if or .MemoryRequest .CPURequest }} + requests: + {{- if .MemoryRequest }} + memory: {{ .MemoryRequest }} + {{- end }} + {{- if .CPURequest }} + cpu: {{ .CPURequest }} + {{- end }} + {{- end }} + {{- if or .MemoryLimit .CPULimit }} + limits: + {{- if .MemoryLimit }} + memory: {{ .MemoryLimit }} + {{- end }} + {{- if .CPULimit }} + cpu: {{ .CPULimit }} + {{- end }} + {{- end }} + {{- end }} + command: [ "/bin/bash", "-c", "--" ] + args: + - | + rm -f /etc/yum.repos.d/photon-updates.repo /etc/yum.repos.d/photon-extras.repo + yum install -y iputils openssh-server sshpass + mkdir /root/.ssh + echo > /root/.completed + while true; do sleep 30; done + {{- if or .PrivateKeySecretName .CustomUserPrivateKeySecretName }} + volumeMounts: + {{- if .PrivateKeySecretName }} + - name: cluster-private-key + mountPath: /secrets/luster-private-key + {{- end }} + {{- if .CustomUserPrivateKeySecretName }} + - name: custom-user-private-key + mountPath: /secrets/custom-user-private-key + {{- end }} + {{- end }} + {{- if or .PrivateKeySecretName .CustomUserPrivateKeySecretName }} + volumes: + {{- if .PrivateKeySecretName }} + - name: cluster-private-key + secret: + secretName: {{ .PrivateKeySecretName }} + defaultMode: 0600 + {{- end }} + {{- if .CustomUserPrivateKeySecretName }} + - name: custom-user-private-key + secret: + secretName: {{ .CustomUserPrivateKeySecretName }} + defaultMode: 0600 + {{- end }} + {{- end }} diff --git a/test/e2e/fixtures/yaml/vmoperator/configmap/configmapOvfEnv.yaml.in b/test/e2e/fixtures/yaml/vmoperator/configmap/configmapOvfEnv.yaml.in new file mode 100644 index 000000000..9b9fc8246 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/configmap/configmapOvfEnv.yaml.in @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +data: + #user-data has the base64 encoded string input: + #cloud-config + #ssh_pwauth: true + #users: + # - name: vmware + # sudo: ALL=(ALL) NOPASSWD:ALL + # lock_passwd: false + # # Password set to Admin!23 + # passwd: '$1$salt$SOC33fVbA/ZxeIwD5yw1u1' + # shell: /bin/bash + #write_files: + # - content: | + # VMSVC Says Hello World + # path: /helloworld + user-data: I2Nsb3VkLWNvbmZpZwpzc2hfcHdhdXRoOiB0cnVlCnVzZXJzOgogIC0gbmFtZTogdm13YXJlCiAgICBzdWRvOiBBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMCiAgICBsb2NrX3Bhc3N3ZDogZmFsc2UKICAgICMgUGFzc3dvcmQgc2V0IHRvIEFkbWluITIzCiAgICBwYXNzd2Q6ICckMSRzYWx0JFNPQzMzZlZiQS9aeGVJd0Q1eXcxdTEnCiAgICBzaGVsbDogL2Jpbi9iYXNoCndyaXRlX2ZpbGVzOgogIC0gY29udGVudDogfAogICAgICBWTVNWQyBTYXlzIEhlbGxvIFdvcmxkCiAgICBwYXRoOiAvaGVsbG93b3JsZAo= \ No newline at end of file diff --git a/test/e2e/fixtures/yaml/vmoperator/configmap/configmapgosc.yaml.in b/test/e2e/fixtures/yaml/vmoperator/configmap/configmapgosc.yaml.in new file mode 100644 index 000000000..8e266d87b --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/configmap/configmapgosc.yaml.in @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +data: + user-data: | + #cloud-config + ssh_pwauth: true + users: + - name: vmware + sudo: ALL=(ALL) NOPASSWD:ALL + lock_passwd: false + # Password set to Admin!23 + passwd: '$1$salt$SOC33fVbA/ZxeIwD5yw1u1' + shell: /bin/bash + write_files: + - content: | + VMSVC Says Hello World + path: /helloworld diff --git a/test/e2e/fixtures/yaml/vmoperator/configmap/configmapvapp.yaml.in b/test/e2e/fixtures/yaml/vmoperator/configmap/configmapvapp.yaml.in new file mode 100644 index 000000000..375884377 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/configmap/configmapvapp.yaml.in @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{.Name}} + namespace: {{.Namespace}} diff --git a/test/e2e/fixtures/yaml/vmoperator/configmap/vappData.yaml b/test/e2e/fixtures/yaml/vmoperator/configmap/vappData.yaml new file mode 100644 index 000000000..307879088 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/configmap/vappData.yaml @@ -0,0 +1,5 @@ +data: + nameservers: "{{ (index .V1alpha1.Net.Nameservers 0) }}" + hostname: "{{ .V1alpha1.VM.Name }}" + management_ip: "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}" + management_gateway: "{{ (index .V1alpha1.Net.Devices 0).Gateway4 }}" \ No newline at end of file diff --git a/test/e2e/fixtures/yaml/vmoperator/contentsources/contentsourcebindings.yaml.in b/test/e2e/fixtures/yaml/vmoperator/contentsources/contentsourcebindings.yaml.in new file mode 100644 index 000000000..a429e838b --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/contentsources/contentsourcebindings.yaml.in @@ -0,0 +1,9 @@ +apiVersion: vmoperator.vmware.com/v1alpha1 +kind: ContentSourceBinding +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +contentSourceRef: + apiVersion: vmoperator.vmware.com/v1alpha1 + kind: ContentSource + name: {{.Name}} \ No newline at end of file diff --git a/test/e2e/fixtures/yaml/vmoperator/secret/secretCloudConfig.yaml.in b/test/e2e/fixtures/yaml/vmoperator/secret/secretCloudConfig.yaml.in new file mode 100644 index 000000000..8a97cc984 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/secret/secretCloudConfig.yaml.in @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +stringData: + user-data: | + #cloud-config + ssh_pwauth: true + users: + - name: vmware + sudo: ALL=(ALL) NOPASSWD:ALL + lock_passwd: false + # Password set to Admin!23 + passwd: '$1$salt$SOC33fVbA/ZxeIwD5yw1u1' + shell: /bin/bash + write_files: + - content: | + VMSVC Says Hello World + path: /helloworld diff --git a/test/e2e/fixtures/yaml/vmoperator/secret/secretEmpty.yaml.in b/test/e2e/fixtures/yaml/vmoperator/secret/secretEmpty.yaml.in new file mode 100644 index 000000000..d26b8f10a --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/secret/secretEmpty.yaml.in @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}} + namespace: {{.Namespace}} diff --git a/test/e2e/fixtures/yaml/vmoperator/secret/secretInlineCloudInitData.yaml.in b/test/e2e/fixtures/yaml/vmoperator/secret/secretInlineCloudInitData.yaml.in new file mode 100644 index 000000000..68cd1535d --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/secret/secretInlineCloudInitData.yaml.in @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +stringData: + # Password set to Admin!23 + vmsvc-pwd: '$1$salt$SOC33fVbA/ZxeIwD5yw1u1' + hello: 'Hello World!' diff --git a/test/e2e/fixtures/yaml/vmoperator/secret/secretInlineSysprepData.yaml.in b/test/e2e/fixtures/yaml/vmoperator/secret/secretInlineSysprepData.yaml.in new file mode 100644 index 000000000..e2b7f85f3 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/secret/secretInlineSysprepData.yaml.in @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +stringData: + vmsvc-pwd: vmware diff --git a/test/e2e/fixtures/yaml/vmoperator/secret/secretOvfEnv.yaml.in b/test/e2e/fixtures/yaml/vmoperator/secret/secretOvfEnv.yaml.in new file mode 100644 index 000000000..36f17c86a --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/secret/secretOvfEnv.yaml.in @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +stringData: + #user-data has the base64 encoded string input: + #cloud-config + #ssh_pwauth: true + #users: + # - name: vmware + # sudo: ALL=(ALL) NOPASSWD:ALL + # lock_passwd: false + # # Password set to Admin!23 + # passwd: '$1$salt$SOC33fVbA/ZxeIwD5yw1u1' + # shell: /bin/bash + #write_files: + # - content: | + # VMSVC Says Hello World + # path: /helloworld + user-data: I2Nsb3VkLWNvbmZpZwpzc2hfcHdhdXRoOiB0cnVlCnVzZXJzOgogIC0gbmFtZTogdm13YXJlCiAgICBzdWRvOiBBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMCiAgICBsb2NrX3Bhc3N3ZDogZmFsc2UKICAgICMgUGFzc3dvcmQgc2V0IHRvIEFkbWluITIzCiAgICBwYXNzd2Q6ICckMSRzYWx0JFNPQzMzZlZiQS9aeGVJd0Q1eXcxdTEnCiAgICBzaGVsbDogL2Jpbi9iYXNoCndyaXRlX2ZpbGVzOgogIC0gY29udGVudDogfAogICAgICBWTVNWQyBTYXlzIEhlbGxvIFdvcmxkCiAgICBwYXRoOiAvaGVsbG93b3JsZAo= \ No newline at end of file diff --git a/test/e2e/fixtures/yaml/vmoperator/secret/sysprepStringData.yaml b/test/e2e/fixtures/yaml/vmoperator/secret/sysprepStringData.yaml new file mode 100644 index 000000000..6ece484d6 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/secret/sysprepStringData.yaml @@ -0,0 +1,66 @@ +stringData: + unattend: | + + + + + + + + false + + + false + + {{ V1alpha1_FirstNicMacAddr }} + + {{ V1alpha1_FirstIP }} + + + + 0 + 10 + {{ (index .V1alpha1.Net.Devices 0).Gateway4 }} + {{ V1alpha1_SubnetMask V1alpha1_FirstIP }} + + + + + + + + + {{ V1alpha1_FirstNicMacAddr }} + {{ range .V1alpha1.Net.Nameservers }} {{ end }} + + + + + + + C:\sysprep\guestcustutil.exe restoreMountedDevices + 1 + + + C:\sysprep\guestcustutil.exe flagComplete + 2 + + + C:\sysprep\guestcustutil.exe deleteContainingFolder + 3 + + + + + diff --git a/test/e2e/fixtures/yaml/vmoperator/secret/vAppStringData.yaml b/test/e2e/fixtures/yaml/vmoperator/secret/vAppStringData.yaml new file mode 100644 index 000000000..918a97ec0 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/secret/vAppStringData.yaml @@ -0,0 +1,5 @@ +stringData: + nameservers: "{{ V1alpha1_FormatNameservers 2 \",\" }}" + hostname: "{{ .V1alpha1.VM.Name }} " + management_ip: "{{ V1alpha1_FirstIP }}" + management_gateway: "{{ (index .V1alpha1.Net.Devices 0).Gateway4 }}" diff --git a/test/e2e/fixtures/yaml/vmoperator/securitypolicy/securitypolicy.yaml.in b/test/e2e/fixtures/yaml/vmoperator/securitypolicy/securitypolicy.yaml.in new file mode 100644 index 000000000..4b91db597 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/securitypolicy/securitypolicy.yaml.in @@ -0,0 +1,14 @@ +apiVersion: crd.nsx.vmware.com/v1alpha1 +kind: SecurityPolicy +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + priority: 10 + appliedTo: + - vmSelector: + matchLabels: + role: allow-ingress + rules: + - direction: in + action: allow diff --git a/test/e2e/fixtures/yaml/vmoperator/storageclass/gc-storage-profile.yaml b/test/e2e/fixtures/yaml/vmoperator/storageclass/gc-storage-profile.yaml new file mode 100644 index 000000000..2d97c8379 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/storageclass/gc-storage-profile.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: gc-storage-profile +provisioner: docker.io/hostpath +parameters: + storagePolicyID: "aa6d5a82-1c88-45da-85d3-3d74b91a5bad" +reclaimPolicy: Delete +volumeBindingMode: Immediate diff --git a/test/e2e/fixtures/yaml/vmoperator/storageclass/gc-storage-quota.yaml b/test/e2e/fixtures/yaml/vmoperator/storageclass/gc-storage-quota.yaml new file mode 100644 index 000000000..1b12e2d6b --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/storageclass/gc-storage-quota.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: gc-storage-quota +spec: + hard: + gc-storage-profile.storageclass.storage.k8s.io/requests.storage: 1Gi diff --git a/test/e2e/fixtures/yaml/vmoperator/subnet/subnet.yaml.in b/test/e2e/fixtures/yaml/vmoperator/subnet/subnet.yaml.in new file mode 100644 index 000000000..e9b6a7b61 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/subnet/subnet.yaml.in @@ -0,0 +1,5 @@ +apiVersion: crd.nsx.vmware.com/v1alpha1 +kind: {{.Kind}} +metadata: + name: {{.Name}} + namespace: {{.Namespace}} diff --git a/test/e2e/fixtures/yaml/vmoperator/subnet/subnetCIDR.yaml b/test/e2e/fixtures/yaml/vmoperator/subnet/subnetCIDR.yaml new file mode 100644 index 000000000..6bcfe5bb5 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/subnet/subnetCIDR.yaml @@ -0,0 +1,3 @@ +spec: + ipv4SubnetSize: 16 + diff --git a/test/e2e/fixtures/yaml/vmoperator/subnet/subnetDHCP.yaml b/test/e2e/fixtures/yaml/vmoperator/subnet/subnetDHCP.yaml new file mode 100644 index 000000000..fb3f3bfb8 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/subnet/subnetDHCP.yaml @@ -0,0 +1,3 @@ +spec: + subnetDHCPConfig: + mode: DHCPServer diff --git a/test/e2e/fixtures/yaml/vmoperator/subnet/subnetPrivateAccessMode.yaml b/test/e2e/fixtures/yaml/vmoperator/subnet/subnetPrivateAccessMode.yaml new file mode 100644 index 000000000..593655e26 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/subnet/subnetPrivateAccessMode.yaml @@ -0,0 +1 @@ + accessMode: Private diff --git a/test/e2e/fixtures/yaml/vmoperator/subnet/subnetPublicAccessMode.yaml b/test/e2e/fixtures/yaml/vmoperator/subnet/subnetPublicAccessMode.yaml new file mode 100644 index 000000000..09cede975 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/subnet/subnetPublicAccessMode.yaml @@ -0,0 +1 @@ + accessMode: Public diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachineclasses/vmclass.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachineclasses/vmclass.yaml.in new file mode 100644 index 000000000..accfefac6 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachineclasses/vmclass.yaml.in @@ -0,0 +1,6 @@ +apiVersion: vmoperator.vmware.com/v1alpha2 +kind: VirtualMachineClass +metadata: + name: {{.Name}} + namespace: {{.Namespace}} + diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachineclasses/vmclassbindings.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachineclasses/vmclassbindings.yaml.in new file mode 100644 index 000000000..094cc885c --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachineclasses/vmclassbindings.yaml.in @@ -0,0 +1,9 @@ +apiVersion: vmoperator.vmware.com/v1alpha1 +kind: VirtualMachineClassBinding +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +classRef: + apiVersion: vmoperator.vmware.com/v1alpha1 + kind: VirtualMachineClass + name: {{.Name}} diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachinegrouppublishrequests/vm-group-publish.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachinegrouppublishrequests/vm-group-publish.yaml.in new file mode 100644 index 000000000..574abe2e5 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachinegrouppublishrequests/vm-group-publish.yaml.in @@ -0,0 +1,14 @@ +apiVersion: vmoperator.vmware.com/v1alpha5 +kind: VirtualMachineGroupPublishRequest +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + source: {{.Source}} + target: {{.Target}} + {{if .VirtualMachines}} + virtualMachines: {{.VirtualMachines}} + {{end}} + {{if .TTLSecondsAfterFinished}} + ttlSecondsAfterFinished: {{.TTLSecondsAfterFinished}} + {{end}} diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachinegroups/vm-group-with-boot-order.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachinegroups/vm-group-with-boot-order.yaml.in new file mode 100644 index 000000000..bb6fcd0a8 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachinegroups/vm-group-with-boot-order.yaml.in @@ -0,0 +1,31 @@ +apiVersion: vmoperator.vmware.com/v1alpha5 +kind: VirtualMachineGroup +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: +{{- if .GroupName}} + groupName: {{.GroupName}} +{{- end}} +{{- if .PowerState}} + powerState: {{.PowerState}} +{{- end}} +{{- if .PowerOffMode}} + powerOffMode: {{.PowerOffMode}} +{{- end}} +{{- if .NextForcePowerStateSyncTime}} + nextForcePowerStateSyncTime: {{.NextForcePowerStateSyncTime}} +{{- end}} +{{- if .BootOrder}} + bootOrder: +{{- range .BootOrder}} + - members: +{{- range .Members}} + - name: {{.Name}} + kind: {{.Kind}} +{{- end}} +{{- if .PowerOnDelay}} + powerOnDelay: {{.PowerOnDelay}} +{{- end}} +{{- end}} +{{- end}} diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachinegroups/vm-group.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachinegroups/vm-group.yaml.in new file mode 100644 index 000000000..b2b76f2e5 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachinegroups/vm-group.yaml.in @@ -0,0 +1,18 @@ +apiVersion: vmoperator.vmware.com/v1alpha5 +kind: VirtualMachineGroup +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: +{{- if .GroupName}} + groupName: {{.GroupName}} +{{- end}} +{{- if .Members}} + bootOrder: + - members: +{{- range .Members}} + - name: {{.Name}} + kind: {{.Kind}} +{{- end}} +{{- end}} + diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachinepublishrequests/singlevirtualmachinepublishrequest.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachinepublishrequests/singlevirtualmachinepublishrequest.yaml.in new file mode 100644 index 000000000..2241c3262 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachinepublishrequests/singlevirtualmachinepublishrequest.yaml.in @@ -0,0 +1,26 @@ +apiVersion: vmoperator.vmware.com/v1alpha2 +kind: VirtualMachinePublishRequest +metadata: + name: {{.Name}} + namespace: {{.Namespace}} + {{if .Annotations}} + annotations: + {{range $key, $value := .Annotations}} + {{$key}}: {{$value}} + {{end}} + {{end}} + {{if .Labels}} + labels: + {{range $key, $value := .Labels}} + {{$key}}: {{$value}} + {{end}} + {{end}} +spec: + source: + name: {{.Source.Name}} + target: + item: + name: {{.Target.Item.Name}} + description: {{.Target.Item.Description}} + location: + name: {{.Target.Location.Name}} \ No newline at end of file diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachines/default-vmset-rp.yaml b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/default-vmset-rp.yaml new file mode 100644 index 000000000..f5fed91b0 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/default-vmset-rp.yaml @@ -0,0 +1,7 @@ +apiVersion: vmoperator.vmware.com/v1alpha2 +kind: VirtualMachineSetResourcePolicy +metadata: + name: default-vm-set-rp + namespace: {{.Namespace}} +spec: + diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachines/pvc.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/pvc.yaml.in new file mode 100644 index 000000000..584bda18d --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/pvc.yaml.in @@ -0,0 +1,21 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{.ClaimName}} + namespace: {{.Namespace}} +spec: + accessModes: + {{- if gt (len .AccessModes) 0}} + {{- range .AccessModes}} + - {{.}} + {{- end}} + {{- else}} + - ReadWriteOnce + {{- end}} + resources: + requests: + storage: {{.RequestSize}} + storageClassName: {{.StorageClassName}} + {{- if .VolumeMode}} + volumeMode: {{.VolumeMode}} + {{- end}} diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachines/singlevm.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/singlevm.yaml.in new file mode 100644 index 000000000..ddd41d014 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/singlevm.yaml.in @@ -0,0 +1,50 @@ +apiVersion: vmoperator.vmware.com/v1alpha1 +kind: VirtualMachine +metadata: + name: {{.Name}} + namespace: {{.Namespace}} + {{if .Annotations}} + annotations: + {{range $key, $value := .Annotations}} + {{$key}}: {{$value}} + {{end}} + {{end}} + {{if .Labels}} + labels: + {{range $key, $value := .Labels}} + {{$key}}: {{$value}} + {{end}} + {{end}} +spec: + {{if .Network.Type}} + networkInterfaces: + - networkName: {{.Network.Name}} + networkType: {{.Network.Type}} + {{end}} + className: {{.VMClassName}} + storageClass: {{.StorageClassName}} + imageName: {{.ImageName}} + {{if .ResourcePolicy}} + resourcePolicyName: {{.ResourcePolicy}} + {{end}} + powerState: {{.PowerState}} + {{if or .ConfigMapName .SecretName .Transport}} + vmMetadata: + {{if .ConfigMapName}} + configMapName: {{.ConfigMapName}} + {{end}} + {{if .SecretName}} + secretName: {{.SecretName}} + {{end}} + {{if .Transport}} + transport: {{.Transport}} + {{end}} + {{end}} + {{if .PVCNames}} + volumes: + {{range .PVCNames}} + - name: {{.}} + persistentVolumeClaim: + claimName: {{.}} + {{end}} + {{end}} diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a2singlevm.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a2singlevm.yaml.in new file mode 100644 index 000000000..e798da36c --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a2singlevm.yaml.in @@ -0,0 +1,87 @@ +apiVersion: vmoperator.vmware.com/v1alpha2 +kind: VirtualMachine +metadata: + name: {{.Name}} + namespace: {{.Namespace}} + {{if .Annotations}} + annotations: + {{range $key, $value := .Annotations}} + {{$key}}: {{$value}} + {{end}} + {{end}} + {{if .Labels}} + labels: + {{range $key, $value := .Labels}} + {{$key}}: {{$value}} + {{end}} + {{end}} +spec: + {{if .NetworkA2.Interfaces}} + network: + interfaces: + - name: eth0 + network: + apiVersion: {{(index .NetworkA2.Interfaces 0).APIVersion}} + kind: {{(index .NetworkA2.Interfaces 0).Kind}} + name: {{(index .NetworkA2.Interfaces 0).Name}} + {{end}} + className: {{.VMClassName}} + storageClass: {{.StorageClassName}} + imageName: {{.ImageName}} + {{if .ResourcePolicy}} + reserved: + resourcePolicyName: {{.ResourcePolicy}} + {{end}} + powerState: {{.PowerState}} + {{if and .Bootstrap (or .Bootstrap.CloudInit .Bootstrap.Sysprep .Bootstrap.VAppConfig .Bootstrap.LinuxPrep)}} + bootstrap: + {{if .Bootstrap.CloudInit }} + cloudInit: + {{if .Bootstrap.CloudInit.RawCloudConfig}} + rawCloudConfig: + key: {{ .Bootstrap.CloudInit.RawCloudConfig.Key}} + name: {{.Bootstrap.CloudInit.RawCloudConfig.Name}} + {{end}} + {{if .Bootstrap.CloudInit.CloudConfig}} + cloudConfig: + {{ .Bootstrap.CloudInit.CloudConfig}} + {{end}} + {{end}} + {{if .Bootstrap.Sysprep }} + sysprep: + {{if .Bootstrap.Sysprep.RawSysprep }} + rawSysprep: + key: {{.Bootstrap.Sysprep.RawSysprep.Key }} + name: {{.Bootstrap.Sysprep.RawSysprep.Name }} + {{end}} + {{if .Bootstrap.Sysprep.Sysprep }} + sysprep: + {{.Bootstrap.Sysprep.Sysprep }} + {{end}} + {{end}} + {{ if .Bootstrap.VAppConfig }} + vAppConfig: + {{ if .Bootstrap.VAppConfig.RawProperties }} + rawProperties: {{ .Bootstrap.VAppConfig.RawProperties }} + {{end}} + {{ if .Bootstrap.VAppConfig.Properties }} + properties: + - key: {{ (index .Bootstrap.VAppConfig.Properties 0).Key}} + value: + value: "{{ (index .Bootstrap.VAppConfig.Properties 0).Value.Value}}" + {{end}} + {{end}} + {{ if .Bootstrap.LinuxPrep }} + linuxPrep: + hardwareClockIsUTC: {{ .Bootstrap.LinuxPrep.HardwareClockIsUTC }} + timeZone: {{ .Bootstrap.LinuxPrep.TimeZone }} + {{end}} + {{end}} + {{if .PVCNames}} + volumes: + {{range .PVCNames}} + - name: {{.}} + persistentVolumeClaim: + claimName: {{.}} + {{end}} + {{end}} diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a2vm-multi-network.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a2vm-multi-network.yaml.in new file mode 100644 index 000000000..66280fffb --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a2vm-multi-network.yaml.in @@ -0,0 +1,84 @@ +apiVersion: vmoperator.vmware.com/v1alpha2 +kind: VirtualMachine +metadata: + name: {{.Name}} + namespace: {{.Namespace}} + {{if .Annotations}} + annotations: + {{range $key, $value := .Annotations}} + {{$key}}: {{$value}} + {{end}} + {{end}} + {{if .Labels}} + labels: + {{range $key, $value := .Labels}} + {{$key}}: {{$value}} + {{end}} + {{end}} +spec: + {{if .NetworkA2.Interfaces}} + network: + interfaces: + - name: eth0 + network: + apiVersion: {{(index .NetworkA2.Interfaces 0).APIVersion}} + kind: {{(index .NetworkA2.Interfaces 0).Kind}} + name: {{(index .NetworkA2.Interfaces 0).Name}} + - name: eth1 + network: + apiVersion: {{(index .NetworkA2.Interfaces 1).APIVersion}} + kind: {{(index .NetworkA2.Interfaces 1).Kind}} + name: {{(index .NetworkA2.Interfaces 1).Name}} + {{end}} + className: {{.VMClassName}} + storageClass: {{.StorageClassName}} + imageName: {{.ImageName}} + {{if .ResourcePolicy}} + reserved: + resourcePolicyName: {{.ResourcePolicy}} + {{end}} + powerState: {{.PowerState}} + {{if and .Bootstrap (or .Bootstrap.CloudInit .Bootstrap.Sysprep .Bootstrap.VAppConfig .Bootstrap.LinuxPrep)}} + bootstrap: + {{if .Bootstrap.CloudInit }} + cloudInit: + {{if .Bootstrap.CloudInit.RawCloudConfig}} + rawCloudConfig: + key: {{ .Bootstrap.CloudInit.RawCloudConfig.Key}} + name: {{.Bootstrap.CloudInit.RawCloudConfig.Name}} + {{end}} + {{if .Bootstrap.CloudInit.CloudConfig}} + cloudConfig: + {{ .Bootstrap.CloudInit.CloudConfig}} + {{end}} + {{end}} + {{if .Bootstrap.Sysprep }} + sysprep: + {{if .Bootstrap.Sysprep.RawSysprep }} + rawSysprep: + key: {{.Bootstrap.Sysprep.RawSysprep.Key }} + name: {{.Bootstrap.Sysprep.RawSysprep.Name }} + {{end}} + {{if .Bootstrap.Sysprep.Sysprep }} + sysprep: + {{.Bootstrap.Sysprep.Sysprep }} + {{end}} + {{end}} + {{ if .Bootstrap.VAppConfig }} + vAppConfig: + {{ if .Bootstrap.VAppConfig.RawProperties }} + rawProperties: {{ .Bootstrap.VAppConfig.RawProperties }} + {{end}} + {{ if .Bootstrap.VAppConfig.Properties }} + properties: + - key: {{ (index .Bootstrap.VAppConfig.Properties 0).Key}} + value: + value: "{{ (index .Bootstrap.VAppConfig.Properties 0).Value.Value}}" + {{end}} + {{end}} + {{ if .Bootstrap.LinuxPrep }} + linuxPrep: + hardwareClockIsUTC: {{ .Bootstrap.LinuxPrep.HardwareClockIsUTC }} + timeZone: {{ .Bootstrap.LinuxPrep.TimeZone }} + {{end}} + {{end}} diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a3singlevm.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a3singlevm.yaml.in new file mode 100644 index 000000000..e7e90d206 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a3singlevm.yaml.in @@ -0,0 +1,55 @@ +apiVersion: vmoperator.vmware.com/v1alpha3 +kind: VirtualMachine +metadata: + name: {{.Name}} + namespace: {{.Namespace}} + {{if .Annotations}} + annotations: + {{range $key, $value := .Annotations}} + {{$key}}: {{$value}} + {{end}} + {{end}} + {{if .Labels}} + labels: + {{range $key, $value := .Labels}} + {{$key}}: {{$value}} + {{end}} + {{end}} +spec: + className: {{.VMClassName}} + storageClass: {{.StorageClassName}} + {{if .ImageName}} + imageName: {{.ImageName}} + {{end}} + {{if .PowerState}} + powerState: {{.PowerState}} + {{end}} + {{if .GuestID}} + guestID: {{.GuestID}} + {{end}} + {{if .Crypto}} + crypto: + encryptionClassName: {{.Crypto.EncryptionClassName}} + useDefaultKeyProvider: {{.Crypto.UseDefaultKeyProvider}} + {{end}} + {{if .PVCNames}} + volumes: + {{range .PVCNames}} + - name: {{.}} + persistentVolumeClaim: + claimName: {{.}} + {{end}} + {{end}} + {{if .Cdrom}} + cdrom: + {{range .Cdrom }} + - name: {{.Name}} + image: + name: {{.ImageName}} + {{if .ImageKind}} + kind: {{.ImageKind}} + {{end}} + connected: {{.Connected}} + allowGuestControl: {{.AllowGuestControl}} + {{end}} + {{end}} diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a5singlevm.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a5singlevm.yaml.in new file mode 100644 index 000000000..b658e7596 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachines/v1a5singlevm.yaml.in @@ -0,0 +1,253 @@ +apiVersion: vmoperator.vmware.com/v1alpha5 +kind: VirtualMachine +metadata: + name: {{.Name}} + namespace: {{.Namespace}} + {{- if .Labels}} + labels: + {{- range $key, $value := .Labels}} + {{$key}}: "{{$value}}" + {{- end}} + {{- end}} + {{- if .Annotations}} + annotations: + {{- range $key, $value := .Annotations}} + {{$key}}: "{{$value}}" + {{- end}} + {{- end}} +spec: +{{- if .GroupName}} + groupName: {{.GroupName}} +{{- end}} + className: {{.VMClassName}} + storageClass: {{.StorageClassName}} + imageName: {{.ImageName}} + {{- if .GuestID}} + guestID: {{.GuestID}} + {{- end}} + {{- if and .Bootstrap (or .Bootstrap.CloudInit .Bootstrap.Sysprep .Bootstrap.VAppConfig .Bootstrap.LinuxPrep)}} + bootstrap: + {{if .Bootstrap.CloudInit }} + cloudInit: + {{if .Bootstrap.CloudInit.RawCloudConfig}} + rawCloudConfig: + key: {{ .Bootstrap.CloudInit.RawCloudConfig.Key}} + name: {{.Bootstrap.CloudInit.RawCloudConfig.Name}} + {{end}} + {{if .Bootstrap.CloudInit.CloudConfig}} + cloudConfig: + {{ .Bootstrap.CloudInit.CloudConfig}} + {{end}} + {{end}} + {{if .Bootstrap.Sysprep }} + sysprep: + {{if .Bootstrap.Sysprep.RawSysprep }} + rawSysprep: + key: {{.Bootstrap.Sysprep.RawSysprep.Key }} + name: {{.Bootstrap.Sysprep.RawSysprep.Name }} + {{end}} + {{if .Bootstrap.Sysprep.Sysprep }} + sysprep: + {{.Bootstrap.Sysprep.Sysprep }} + {{end}} + {{end}} + {{ if .Bootstrap.VAppConfig }} + vAppConfig: + {{ if .Bootstrap.VAppConfig.RawProperties }} + rawProperties: {{ .Bootstrap.VAppConfig.RawProperties }} + {{end}} + {{ if .Bootstrap.VAppConfig.Properties }} + properties: + - key: {{ (index .Bootstrap.VAppConfig.Properties 0).Key}} + value: + value: "{{ (index .Bootstrap.VAppConfig.Properties 0).Value.Value}}" + {{end}} + {{end}} + {{ if .Bootstrap.LinuxPrep }} + linuxPrep: + hardwareClockIsUTC: {{ .Bootstrap.LinuxPrep.HardwareClockIsUTC }} + timeZone: {{ .Bootstrap.LinuxPrep.TimeZone }} + customizeAtNextPowerOn: {{ .Bootstrap.LinuxPrep.CustomizeAtNextPowerOn }} + {{end}} + {{end}} +{{- if .PowerState}} + powerState: {{.PowerState}} +{{- end}} +{{- if .PVCs}} + volumes: + {{- range .PVCs}} + - name: {{.VolumeName}} + persistentVolumeClaim: + claimName: {{.ClaimName}} + {{- if .ControllerBusNumber }} + controllerBusNumber: {{.ControllerBusNumber}} + {{- end}} + {{- if .ControllerType }} + controllerType: {{.ControllerType}} + {{- end}} + {{- if .ControllerType }} + controllerType: {{.ControllerType}} + {{- end}} + {{- if .UnitNumber }} + unitNumber: {{.UnitNumber}} + {{- end}} + {{- if .SharingMode }} + sharingMode: {{.SharingMode}} + {{- end}} + {{- if .ApplicationType }} + applicationType: {{.ApplicationType}} + {{- end}} + {{- if .DiskMode }} + diskMode: {{.DiskMode}} + {{- end}} + {{- end}} +{{- end}} +{{- if .Affinity}} + affinity: + {{- if .Affinity.VMAffinity}} + vmAffinity: + {{- if .Affinity.VMAffinity.RequiredDuringSchedulingPreferredDuringExecution}} + requiredDuringSchedulingPreferredDuringExecution: + {{- range .Affinity.VMAffinity.RequiredDuringSchedulingPreferredDuringExecution}} + {{- if .LabelSelector}} + - labelSelector: + {{- if .LabelSelector.MatchLabels}} + matchLabels: + {{- range $key, $value := .LabelSelector.MatchLabels}} + {{$key}}: "{{$value}}" + {{- end}} + {{- end}} + {{- if .LabelSelector.MatchExpressions}} + matchExpressions: + {{- range .LabelSelector.MatchExpressions}} + - key: {{.Key}} + operator: {{.Operator}} + {{- if .Values}} + values: + {{- range .Values}} + - "{{.}}" + {{- end}} + {{- end}} + {{- end}} + {{- end}} + {{- end}} + topologyKey: {{.TopologyKey}} + {{- end}} + {{- end}} + {{- end}} + {{- if .Affinity.VMAntiAffinity}} + vmAntiAffinity: + {{- if .Affinity.VMAntiAffinity.RequiredDuringSchedulingPreferredDuringExecution}} + requiredDuringSchedulingPreferredDuringExecution: + {{- range .Affinity.VMAntiAffinity.RequiredDuringSchedulingPreferredDuringExecution}} + {{- if .LabelSelector}} + - labelSelector: + {{- if .LabelSelector.MatchLabels}} + matchLabels: + {{- range $key, $value := .LabelSelector.MatchLabels}} + {{$key}}: "{{$value}}" + {{- end}} + {{- end}} + {{- if .LabelSelector.MatchExpressions}} + matchExpressions: + {{- range .LabelSelector.MatchExpressions}} + - key: {{.Key}} + operator: {{.Operator}} + {{- if .Values}} + values: + {{- range .Values}} + - "{{.}}" + {{- end}} + {{- end}} + {{- end}} + {{- end}} + {{- end}} + topologyKey: {{.TopologyKey}} + {{- end}} + {{- end}} + {{- end}} +{{- end}} +{{- with .Hardware }} + hardware: + {{- with .SCSIControllers }} + scsiControllers: + {{- range . }} + - busNumber: {{.BusNumber}} + type: {{.Type}} + sharingMode: {{.SharingMode}} + {{- end }} + {{- end }} + {{- with .NVMEControllers}} + nvmeControllers: + {{- range . }} + - busNumber: {{.BusNumber}} + sharingMode: {{.SharingMode}} + {{- end}} + {{- end}} + {{- with .SATAControllers}} + sataControllers: + {{- range . }} + - busNumber: {{.BusNumber}} + {{- end}} + {{- end}} + {{- with .Cdrom}} + cdrom: + {{- range .}} + - name: {{.Name}} + {{- with .Image}} + image: + name: {{.Name}} + kind: {{.Kind}} + {{- end}} + {{- if .ControllerBusNumber}} + controllerBusNumber: {{.ControllerBusNumber}} + {{- end}} + {{- if .ControllerType}} + controllerType: {{.ControllerType}} + {{- end}} + {{- if .UnitNumber}} + unitNumber: {{.UnitNumber}} + {{- end}} + {{- if .Connected}} + connected: {{.Connected}} + {{- end}} + {{- if .AllowGuestControl}} + allowGuestControl: {{.AllowGuestControl}} + {{- end}} + {{- end}} + {{- end}} +{{- end }} +{{- if .Policies}} + policies: + {{- range .Policies}} + - apiVersion: {{.APIVersion}} + kind: {{.Kind}} + name: {{.Name}} + {{- end}} +{{- end}} +{{- if .PVCs}} +{{- range .PVCs}} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{.ClaimName}} + namespace: {{.Namespace}} +spec: + accessModes: + {{- if gt (len .AccessModes) 0}} + {{- range .AccessModes}} + - {{.}} + {{- end}} + {{- else}} + - ReadWriteOnce + {{- end}} + resources: + requests: + storage: {{.RequestSize}} + storageClassName: {{.StorageClassName}} + {{- if .VolumeMode}} + volumeMode: {{.VolumeMode}} + {{- end}} +{{- end}} +{{- end}} diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachinesnapshot/v1alpha5-vmsnapshot.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachinesnapshot/v1alpha5-vmsnapshot.yaml.in new file mode 100644 index 000000000..1d5b6d54e --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachinesnapshot/v1alpha5-vmsnapshot.yaml.in @@ -0,0 +1,21 @@ +apiVersion: vmoperator.vmware.com/v1alpha5 +kind: VirtualMachineSnapshot +metadata: + name: {{.Name}} + namespace: {{.Namespace}} + {{- if .ImportedSnapshot}} + annotations: + "vmoperator.vmware.com/imported-snapshot": "" + {{- end}} +spec: + vmName: "{{ .VMName }}" + {{- if .Memory}} + memory: {{.Memory}} + {{- end}} + {{- if .Quiesce}} + quiesce: + timeout: {{.Quiesce}} + {{- end}} + {{- if .Description}} + description: "{{.Description}}" + {{- end}} diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachinewebconsolerequests/virtualmachinewebconsolerequests.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachinewebconsolerequests/virtualmachinewebconsolerequests.yaml.in new file mode 100644 index 000000000..d15e0f1c5 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachinewebconsolerequests/virtualmachinewebconsolerequests.yaml.in @@ -0,0 +1,16 @@ +apiVersion: vmoperator.vmware.com/v1alpha2 +kind: VirtualMachineWebConsoleRequest +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + publicKey: | + -----BEGIN PUBLIC KEY----- + MIIBCgKCAQEAs8F7eAedZ4R1qKDQVOqyOjzToYs62iFqUZ9TnW+0HVO+tmnWq0Tj + TlJ7w46KGpBKxN8KlO82+ovrqkBr4OudQFkn7BbmrZ134phIcc0SQZs2nz9+h1AX + 1hSHhozp1mS91XvGlrK0k44a2i6boh257de2rWHh3L5zfPJe31h3L90F43Je9/Oh + FVrm8NUlRzIUd8ADm/dBEu5bUQ+vHIoh/Xqfglf7oRjp8UHuvV/nHI7XmR607QxI + o7QLbIgh3wv4TbfFFJelGpkj7gORSG7gdF7EY0lz5jm/Or4qVUUqAAybyAT1UyiW + hfIIwodsz4QjCpL2LDgxro//gFqWRZurZwIDAQAB + -----END PUBLIC KEY----- + name: {{.VMName}} \ No newline at end of file diff --git a/test/e2e/fixtures/yaml/vmoperator/virtualmachinewebconsolerequests/webconsolerequests.yaml.in b/test/e2e/fixtures/yaml/vmoperator/virtualmachinewebconsolerequests/webconsolerequests.yaml.in new file mode 100644 index 000000000..884057bd7 --- /dev/null +++ b/test/e2e/fixtures/yaml/vmoperator/virtualmachinewebconsolerequests/webconsolerequests.yaml.in @@ -0,0 +1,16 @@ +apiVersion: vmoperator.vmware.com/v1alpha1 +kind: WebConsoleRequest +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + publicKey: | + -----BEGIN PUBLIC KEY----- + MIIBCgKCAQEAs8F7eAedZ4R1qKDQVOqyOjzToYs62iFqUZ9TnW+0HVO+tmnWq0Tj + TlJ7w46KGpBKxN8KlO82+ovrqkBr4OudQFkn7BbmrZ134phIcc0SQZs2nz9+h1AX + 1hSHhozp1mS91XvGlrK0k44a2i6boh257de2rWHh3L5zfPJe31h3L90F43Je9/Oh + FVrm8NUlRzIUd8ADm/dBEu5bUQ+vHIoh/Xqfglf7oRjp8UHuvV/nHI7XmR607QxI + o7QLbIgh3wv4TbfFFJelGpkj7gORSG7gdF7EY0lz5jm/Or4qVUUqAAybyAT1UyiW + hfIIwodsz4QjCpL2LDgxro//gFqWRZurZwIDAQAB + -----END PUBLIC KEY----- + virtualMachineName: {{.VMName}} \ No newline at end of file diff --git a/test/e2e/framework/cluster_proxy.go b/test/e2e/framework/cluster_proxy.go new file mode 100644 index 000000000..87d28d027 --- /dev/null +++ b/test/e2e/framework/cluster_proxy.go @@ -0,0 +1,422 @@ +/* +Copyright 2020 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Copyright (c) 2019 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package framework + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/controller-runtime/pkg/client" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + maxRetries = 5 + initialDuration = 1 * time.Minute + retryFactor = 2 + jitterFactor = 0.1 +) + +// https://github.com/kubernetes-sigs/cluster-api/blob/master/test/framework/cluster_proxy.go +// ClusterProxy defines the behavior of a type that acts as an intermediary with an existing Kubernetes cluster. +// It should work with any Kubernetes cluster, no matter if the Cluster was created by a bootstrap.ClusterProvider, +// by Cluster API (a workload cluster or a self-hosted cluster) or else. + +type ClusterProxyInterface interface { + // GetName returns the name of the cluster. + GetName() string + + // GetKubeconfigPath returns the path to the kubeconfig file to be used to access the Kubernetes cluster. + GetKubeconfigPath() string + + // GetScheme returns the scheme defining the types hosted in the Kubernetes cluster. + // It is used when creating a controller-runtime client. + GetScheme() *runtime.Scheme + + // GetClient returns a controller-runtime client to the Kubernetes cluster. + GetClient() client.Client + + // GetDynamicClient returns a client-go dynamic client for the cluster. + GetDynamicClient() dynamic.Interface + + // GetClientSet returns a client-go client to the Kubernetes cluster. + GetClientSet() *kubernetes.Clientset + + // GetRESTConfig returns the REST config for direct use with client-go if needed. + GetRESTConfig() *rest.Config + + // Apply applies YAML to the Kubernetes cluster, `kubectl apply`. + // Optional noRetryPatterns: if provided, doesn't retry on errors matching these patterns. + Apply(context.Context, []byte, ...string) error + + // Delete to delete YAML to the Kubernetes cluster, `kubectl delete`. + Delete(context.Context, []byte) error + + // GetWorkloadCluster returns a workload cluster proxy interface provisioned by the management cluster + GetWorkloadCluster(ctx context.Context, namespace, name string) WorkloadClusterProxy + + // Dispose proxy's internal resources (the operation does not affects the Kubernetes cluster). + // This should be implemented as a synchronous function. + Dispose(context.Context) +} + +// clusterProxy provides a base implementation of the ClusterProxy interface. +type clusterProxy struct { + name string + kubeconfigPath string + shouldCleanupKubeconfig bool + scheme *runtime.Scheme + // TODO: + // for collecting logs in namespace of a cluster + // logCollector ClusterLogCollector +} + +// NewClusterProxy returns a clusterProxy given a KubeconfigPath and the scheme defining the types hosted in the cluster. +// If a kubeconfig file isn't provided, standard kubeconfig locations will be used (kubectl loading rules apply). +func NewClusterProxy(name string, kubeconfigPath string, scheme *runtime.Scheme) *clusterProxy { + Expect(scheme).NotTo(BeNil(), "scheme is required for NewClusterProxy") + + if kubeconfigPath == "" { + kubeconfigPath = clientcmd.NewDefaultClientConfigLoadingRules().GetDefaultFilename() + } + + proxy := &clusterProxy{ + name: name, + kubeconfigPath: kubeconfigPath, + scheme: scheme, + shouldCleanupKubeconfig: false, + } + + return proxy +} + +// GetName returns the name of the cluster. +func (p *clusterProxy) GetName() string { + return p.name +} + +// GetKubeconfigPath returns the path to the kubeconfig file for the cluster. +func (p *clusterProxy) GetKubeconfigPath() string { + return p.kubeconfigPath +} + +// GetScheme returns the scheme defining the types hosted in the cluster. +func (p *clusterProxy) GetScheme() *runtime.Scheme { + return p.scheme +} + +// GetClient returns a controller-runtime client for the cluster. +func (p *clusterProxy) GetClient() client.Client { + config := p.GetRESTConfig() + + var c client.Client + + Eventually(func() client.Client { + var err error + + c, err = client.New(config, client.Options{Scheme: p.scheme}) + if err != nil { + fmt.Printf("error getting controller-runtime client, error : %s\n", err.Error()) + return nil + } + + return c + }, 5*time.Minute, 10*time.Second).ShouldNot(BeNil(), "Failed to get controller-runtime client") + + return c +} + +// GetClientSet returns a client-go client for the cluster. +func (p *clusterProxy) GetClientSet() *kubernetes.Clientset { + restConfig := p.GetRESTConfig() + + cs, err := kubernetes.NewForConfig(restConfig) + Expect(err).ToNot(HaveOccurred(), "Failed to get client-go client") + + return cs +} + +// Apply wraps `kubectl apply` and prints the output so we can see what gets applied to the cluster. +// Optional noRetryPatterns: if provided, doesn't retry on errors matching these patterns. +func (p *clusterProxy) Apply(ctx context.Context, resources []byte, noRetryPatterns ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Apply") + Expect(resources).NotTo(BeNil(), "resources is required for Apply") + + backoff := wait.Backoff{ + Steps: maxRetries, + Duration: initialDuration, + Factor: retryFactor, + Jitter: jitterFactor, + } + + return wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (bool, error) { + err := KubectlApply(ctx, p.kubeconfigPath, resources) + if err != nil { + e2eframework.Logf("KubectlApply error: '%s'", err.Error()) + + // Check if error matches any no-retry patterns + errMsg := strings.ToLower(err.Error()) + for _, pattern := range noRetryPatterns { + if strings.Contains(errMsg, strings.ToLower(pattern)) { + e2eframework.Logf("No-retry pattern '%s' matched, failing immediately without retry", pattern) + return false, err // Return error immediately, don't retry + } + } + + return false, nil // Retry on other errors + } + + return true, nil // Success, stop retrying + }) +} + +// ApplyWithArgs wraps `kubectl apply ...` and prints the output so we can see what gets applied to the cluster. +func (p *clusterProxy) ApplyWithArgs(ctx context.Context, resources []byte, args ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Apply") + Expect(resources).NotTo(BeNil(), "resources is required for Apply") + + return KubectlApplyWithArgs(ctx, p.kubeconfigPath, resources, args...) +} + +// Delete wraps `kubectl delete` and prints the output so we can see what gets deleted. +func (p *clusterProxy) Delete(ctx context.Context, resources []byte) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Delete") + Expect(resources).NotTo(BeNil(), "resources is required for Delete") + + return KubectlDelete(ctx, p.kubeconfigPath, resources) +} + +// GetRESTConfig retrieves RESTConfig from kubeconfig and if not provided, from default's client config. +func (p *clusterProxy) GetRESTConfig() *rest.Config { + config, err := clientcmd.LoadFromFile(p.kubeconfigPath) + Expect(err).ToNot(HaveOccurred(), "Failed to load Kubeconfig file from %q", p.kubeconfigPath) + + restConfig, err := clientcmd.NewDefaultClientConfig(*config, &clientcmd.ConfigOverrides{}).ClientConfig() + Expect(err).ToNot(HaveOccurred(), "Failed to get ClientConfig from %q", p.kubeconfigPath) + + restConfig.UserAgent = "e2e" + + return restConfig +} + +// GetDynamicClient returns a client-go dynamic client for the cluster. +func (p *clusterProxy) GetDynamicClient() dynamic.Interface { + restConfig := p.GetRESTConfig() + + ds, err := dynamic.NewForConfig(restConfig) + Expect(err).ToNot(HaveOccurred(), "Failed to get client-go dynamic client") + + return ds +} + +// newFromAPIConfig returns a clusterProxy given a api.Config and the scheme defining the types hosted in the cluster. +func newFromAPIConfig(name string, config *api.Config, scheme *runtime.Scheme) WorkloadClusterProxy { + // NB. the ClusterProvider is responsible for the cleanup of this file + f, err := os.CreateTemp("", "e2e-kubeconfig") + Expect(err).ToNot(HaveOccurred(), "Failed to create kubeconfig file for the kind cluster %q") + + kubeconfigPath := f.Name() + + err = clientcmd.WriteToFile(*config, kubeconfigPath) + Expect(err).ToNot(HaveOccurred(), "Failed to write kubeconfig for the kind cluster to a file %q") + + return newWorkloadClusterProxy(name, kubeconfigPath, scheme) +} + +// getKubeconfig retrieves kubeconfig values from the secret of a workload cluster. +func (p *clusterProxy) getKubeconfig(ctx context.Context, namespace string, name string) *api.Config { + cl := p.GetClient() + + secret := &corev1.Secret{} + key := client.ObjectKey{ + Name: fmt.Sprintf("%s-kubeconfig", name), + Namespace: namespace, + } + Expect(cl.Get(ctx, key, secret)).To(Succeed(), "Failed to get %s", key) + Expect(secret.Data).To(HaveKey("value"), "Invalid secret %s", key) + + config, err := clientcmd.Load(secret.Data["value"]) + Expect(err).ToNot(HaveOccurred(), "Failed to convert %s into a kubeconfig file", key) + + return config +} + +// GetWorkloadCluster returns ClusterProxy for the workload cluster. +func (p *clusterProxy) GetWorkloadCluster(ctx context.Context, namespace, name string) WorkloadClusterProxy { + Expect(ctx).NotTo(BeNil(), "ctx is required for GetWorkloadCluster") + Expect(namespace).NotTo(BeEmpty(), "namespace is required for GetWorkloadCluster") + Expect(name).NotTo(BeEmpty(), "name is required for GetWorkloadCluster") + + // gets the kubeconfig from the cluster + config := p.getKubeconfig(ctx, namespace, name) + + return newFromAPIConfig(name, config, p.GetScheme()) +} + +// CollectWorkloadClusterLogs collects machines logs from the workload cluster. +func (p *clusterProxy) CollectWorkloadClusterLogs(ctx context.Context, namespace, name, outputPath string) { +} + +// Dispose clean up the kubeconfig path in os environment. +func (p *clusterProxy) Dispose(ctx context.Context) { + Expect(ctx).NotTo(BeNil(), "ctx is required for Dispose") + + if p.shouldCleanupKubeconfig { + err := os.Remove(p.kubeconfigPath) + if err != nil { + fmt.Printf("deleting the kubeconfig file %q file. You may need to remove this by hand.\n", p.kubeconfigPath) + } + } +} + +// TODO: Shrink the behavior of workload cluster proxy interface after the new E2E tests +// +// has been re-written +type WorkloadClusterProxy interface { + ClusterProxyInterface + Get(ctx context.Context, args ...string) ([]byte, error) + GetClientNoExpect(timeoutMins int) (client.Client, error) + ApplyWithArgs(ctx context.Context, resources []byte, args ...string) error + Create(ctx context.Context, resources []byte) error + CreateWithArgs(ctx context.Context, resources []byte, args ...string) error + Delete(ctx context.Context, resources []byte) error + DeleteWithArgs(ctx context.Context, resources []byte, args ...string) error + DeleteWithNamespacedName(ctx context.Context, resource, ns, name string) error + Exec(ctx context.Context, args ...string) ([]byte, error) + Config(ctx context.Context, args ...string) ([]byte, error) +} + +type workloadClusterProxy struct { + *clusterProxy +} + +// NewClusterProxy returns a workloadClusterProxy given a KubeconfigPath and the scheme defining the types hosted in the cluster. +// If a kubeconfig file isn't provided, standard kubeconfig locations will be used (kubectl loading rules apply). +func newWorkloadClusterProxy(name string, kubeconfigPath string, scheme *runtime.Scheme) *workloadClusterProxy { + Expect(scheme).NotTo(BeNil(), "scheme is required for NewWorkloadClusterProxy") + + if kubeconfigPath == "" { + kubeconfigPath = clientcmd.NewDefaultClientConfigLoadingRules().GetDefaultFilename() + } + + proxy := &clusterProxy{ + name: name, + kubeconfigPath: kubeconfigPath, + scheme: scheme, + shouldCleanupKubeconfig: false, + } + + return &workloadClusterProxy{ + clusterProxy: proxy, + } +} + +// Get retrieves the status of a resource, `kubectl get`. +func (w *workloadClusterProxy) Get(ctx context.Context, args ...string) ([]byte, error) { + Expect(ctx).NotTo(BeNil(), "ctx is required for Get") + + return KubectlGet(ctx, w.GetKubeconfigPath(), args...) +} + +// GetClientNoExpect returns a controller-runtime client for the cluster. +// This function was made for retrying to work (when a client takes too long to load). +func (w *workloadClusterProxy) GetClientNoExpect(timeoutMins int) (client.Client, error) { + var ( + err error + c client.Client + ) + + config := w.GetRESTConfig() + + c, err = client.New(config, client.Options{Scheme: w.scheme}) + + return c, err +} + +// Apply wraps `kubectl apply ...` and prints the output so we can see what gets applied to the cluster. +func (w *workloadClusterProxy) ApplyWithArgs(ctx context.Context, resources []byte, args ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Apply") + Expect(resources).NotTo(BeNil(), "resources is required for Apply") + + return KubectlApplyWithArgs(ctx, w.GetKubeconfigPath(), resources, args...) +} + +// Create wraps `kubectl create` and prints the output so we can see what gets applied to the cluster. +func (w *workloadClusterProxy) Create(ctx context.Context, resources []byte) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Create") + Expect(resources).NotTo(BeNil(), "resources is required for Create") + + return KubectlCreate(ctx, w.GetKubeconfigPath(), resources) +} + +// Create wraps `kubectl create ...` and prints the output so we can see what gets applied to the cluster. +func (w *workloadClusterProxy) CreateWithArgs(ctx context.Context, resources []byte, args ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Create") + Expect(resources).NotTo(BeNil(), "resources is required for Create") + + return KubectlCreateWithArgs(ctx, w.GetKubeconfigPath(), resources, args...) +} + +// Delete wraps `kubectl delete` and prints the output so we can see what gets deleted. +func (w *workloadClusterProxy) Delete(ctx context.Context, resources []byte) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Delete") + Expect(resources).NotTo(BeNil(), "resources is required for Delete") + + return KubectlDelete(ctx, w.GetKubeconfigPath(), resources) +} + +// DeleteWithArgs wraps `kubectl delete ...` and prints the output so we can see what gets deleted. +func (w *workloadClusterProxy) DeleteWithArgs(ctx context.Context, resources []byte, args ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Delete") + Expect(resources).NotTo(BeNil(), "resources is required for Delete") + + return KubectlDeleteWithArgs(ctx, w.GetKubeconfigPath(), resources, args...) +} + +// DeleteWithNamespacedName delete a manifestbuilders from the cluster by its namespace and name. +func (w *workloadClusterProxy) DeleteWithNamespacedName(ctx context.Context, resource, ns, name string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Delete") + + return KubectlDeleteWithNamespacedName(ctx, w.GetKubeconfigPath(), resource, ns, name) +} + +// Exec performs kubectl exec with following flags. +func (w *workloadClusterProxy) Exec(ctx context.Context, args ...string) ([]byte, error) { + Expect(ctx).NotTo(BeNil(), "ctx is required for Exec") + + return KubectlExec(ctx, w.GetKubeconfigPath(), args...) +} + +// Config performs kubectl exec with following flags. +func (w *workloadClusterProxy) Config(ctx context.Context, args ...string) ([]byte, error) { + Expect(ctx).NotTo(BeNil(), "ctx is required for Config") + + return KubectlConfig(ctx, w.GetKubeconfigPath(), args...) +} diff --git a/test/e2e/framework/config.go b/test/e2e/framework/config.go new file mode 100644 index 000000000..66653e2f9 --- /dev/null +++ b/test/e2e/framework/config.go @@ -0,0 +1,63 @@ +// Copyright (c) 2019 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/kubernetes-sigs/cluster-api/blob/a260cb4c3b2cf2f32379f6ae5d8c6312780e9ca5/test/framework/clusterctl/e2e_config.go + +package framework + +import ( + "fmt" + + . "github.com/onsi/gomega" +) + +type ConfigInterface interface { + GetIntervals(spec, key string) []any + GetVariable(varName string) string +} + +type Config struct { + Variables map[string]string `json:"variables,omitempty"` + // Intervals to be used for long operations during tests + Intervals map[string][]string `json:"intervals,omitempty"` +} + +// GetIntervals returns the value in the format: "default/key: ["10m", "5s"]". +func (c *Config) GetIntervals(spec, key string) []any { + intervals, ok := c.Intervals[fmt.Sprintf("%s/%s", spec, key)] + if !ok { + if intervals, ok = c.Intervals[fmt.Sprintf("default/%s", key)]; !ok { + return nil + } + } + + intervalsInterfaces := make([]any, len(intervals)) + for i := range intervals { + intervalsInterfaces[i] = intervals[i] + } + + return intervalsInterfaces +} + +// GetVariable returns a variable from the e2e config file. +func (c *Config) GetVariable(varName string) string { + version, ok := c.Variables[varName] + Expect(ok).NotTo(BeFalse(), "failed to get variable %q", varName) + + return version +} + +// https://github.com/kubernetes-sigs/cluster-api/blob/defb99c408b54dd8b58fc586eb424425b4198484/test/framework/clusterctl/e2e_config.go#L174 +func ErrInvalidArg(format string, args ...any) error { + msg := fmt.Sprintf(format, args...) + return fmt.Errorf("invalid argument: %s", msg) +} + +func ErrEmptyArg(argName string) error { + return ErrInvalidArg("%s is empty", argName) +} + +// SetVariable sets a variable in the loaded e2e config. +// Use it with caution as it will not be persisted to the config file. +func (c *Config) SetVariable(varName string, value string) { + c.Variables[varName] = value +} diff --git a/test/e2e/framework/ginkgo.go b/test/e2e/framework/ginkgo.go new file mode 100644 index 000000000..96c1b416e --- /dev/null +++ b/test/e2e/framework/ginkgo.go @@ -0,0 +1,15 @@ +// Copyright (c) 2019 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package framework + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" +) + +// https://github.com/kubernetes-sigs/cluster-api/blob/5ac19dc6a5f78f98282f13d5159dcb2d91e4d89f/test/e2e/common.go#L45 +func Byf(format string, a ...any) { + By(fmt.Sprintf(format, a...)) +} diff --git a/test/e2e/framework/kubectl.go b/test/e2e/framework/kubectl.go new file mode 100644 index 000000000..c6c9cf81d --- /dev/null +++ b/test/e2e/framework/kubectl.go @@ -0,0 +1,354 @@ +/* +Copyright 2020 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// https://github.com/kubernetes-sigs/cluster-api/blob/a260cb4c3b2cf2f32379f6ae5d8c6312780e9ca5/test/framework/exec/kubectl.go + +package framework + +import ( + "bytes" + "context" + "fmt" + "io" + "os/exec" + "strings" +) + +func KubectlRawWithArgs(ctx context.Context, kubeconfigPath, oper string, resources []byte, args ...string) ( + []byte, []byte, error) { + aargs := append([]string{oper, "--kubeconfig", kubeconfigPath, "-f", "-"}, args...) + rbytes := bytes.NewReader(resources) + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + WithStdin(rbytes), + ) + + return applyCmd.Run(ctx) +} + +// TODO: Remove this usage of kubectl and replace with a function from apply.go using the controller-runtime client. +func KubectlApply(ctx context.Context, kubeconfigPath string, resources []byte) error { + return KubectlApplyWithArgs(ctx, kubeconfigPath, resources) +} + +// Example: +// ApplyWithArgs(ctx, workloadClusterTemplate, "--selector", "kcp-adoption.step1"). +func KubectlApplyWithArgs(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) error { + stdout, stderr, err := KubectlApplyRawWithArgs(ctx, kubeconfigPath, resources, args...) + if err != nil { + // Check if stderr is not empty and append it to the error message + if len(stderr) > 0 { + err = fmt.Errorf("%w: %s", err, string(stderr)) + } + + fmt.Println(string(stderr)) + + return err + } + + fmt.Println(string(stdout)) + + return nil +} + +// KubectlApplyRawWithArgs applies the given yaml in bytes and arguments and return the kubectl command stdout, stderr and the error if there is any. +func KubectlApplyRawWithArgs(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) ([]byte, []byte, error) { + return KubectlRawWithArgs(ctx, kubeconfigPath, "apply", resources, args...) +} + +func KubectlCreate(ctx context.Context, kubeconfigPath string, resources []byte) error { + return KubectlCreateWithArgs(ctx, kubeconfigPath, resources) +} + +func KubectlCreateWithArgs(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) error { + aargs := append([]string{"create", "--kubeconfig", kubeconfigPath, "-f", "-"}, args...) + rbytes := bytes.NewReader(resources) + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + WithStdin(rbytes), + ) + + stdout, stderr, err := applyCmd.Run(ctx) + if err != nil { + fmt.Println(string(stderr)) + return err + } + + fmt.Println(string(stdout)) + + return nil +} + +func KubectlCreateRawWithArgs(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) ([]byte, []byte, error) { + return KubectlRawWithArgs(ctx, kubeconfigPath, "create", resources, args...) +} + +func KubectlDelete(ctx context.Context, kubeconfigPath string, resources []byte) error { + return KubectlDeleteWithArgs(ctx, kubeconfigPath, resources) +} + +func KubectlDeleteWithNamespacedName(ctx context.Context, kubeconfigPath, resource, ns, name string) error { + aargs := append([]string{"--kubeconfig", kubeconfigPath, "delete"}, resource, "-n", ns, name) + + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + ) + + _, stderr, err := applyCmd.Run(ctx) + if err != nil { + fmt.Println(string(stderr)) + return err + } + + return nil +} + +func KubectlDeleteWithArgs(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) error { + aargs := append([]string{"delete", "--kubeconfig", kubeconfigPath, "-f", "-"}, args...) + rbytes := bytes.NewReader(resources) + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + WithStdin(rbytes), + ) + + stdout, stderr, err := applyCmd.Run(ctx) + if err != nil { + fmt.Println(string(stderr)) + return err + } + + fmt.Println(string(stdout)) + + return nil +} + +func KubectlWait(ctx context.Context, kubeconfigPath string, args ...string) error { + wargs := append([]string{"wait", "--kubeconfig", kubeconfigPath}, args...) + wait := NewCommand( + WithCommand("kubectl"), + WithArgs(wargs...), + ) + + _, stderr, err := wait.Run(ctx) + if err != nil { + fmt.Println(string(stderr)) + return err + } + + return nil +} + +func KubectlGet(ctx context.Context, kubeconfigPath string, args ...string) ([]byte, error) { + aargs := append([]string{"--kubeconfig", kubeconfigPath, "get"}, args...) + + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + ) + + stdout, stderr, err := applyCmd.Run(ctx) + if err != nil { + fmt.Printf("stderr: %s", string(stderr)) + fmt.Printf("err: %v", err) + + return []byte(""), err + } + + return stdout, nil +} + +func KubectlExec(ctx context.Context, kubeconfigPath string, args ...string) ([]byte, error) { + aargs := append([]string{"--kubeconfig", kubeconfigPath, "exec"}, args...) + + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + ) + + stdout, stderr, err := applyCmd.Run(ctx) + if err != nil { + fmt.Println(string(stderr)) + return []byte(""), err + } + + return stdout, nil +} + +func KubectlCreateWithoutFile(ctx context.Context, kubeconfigPath string, args ...string) ([]byte, error) { + aargs := append([]string{"--kubeconfig", kubeconfigPath, "create"}, args...) + + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + ) + + stdout, stderr, err := applyCmd.Run(ctx) + if err != nil { + fmt.Println(string(stderr)) + return []byte(""), err + } + + return stdout, nil +} + +func KubectlConfig(ctx context.Context, kubeconfigPath string, args ...string) ([]byte, error) { + aargs := append([]string{"--kubeconfig", kubeconfigPath, "config"}, args...) + + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + ) + + stdout, stderr, err := applyCmd.Run(ctx) + if err != nil { + fmt.Println(string(stderr)) + return []byte(""), err + } + + return stdout, nil +} + +func KubectlAuth(ctx context.Context, kubeconfigPath string, args ...string) ([]byte, error) { + aargs := append([]string{"--kubeconfig", kubeconfigPath, "auth"}, args...) + + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + ) + fmt.Printf("Running command: %s %s\n", applyCmd.Cmd, strings.Join(applyCmd.Args, " ")) + stdout, stderr, err := applyCmd.Run(ctx) + fmt.Printf("stdout: %s\n", string(stdout)) + + if err != nil { + fmt.Println(string(stderr)) + return stdout, err + } + + return stdout, nil +} + +func KubectlLabel(ctx context.Context, kubeconfigPath string, args ...string) error { + aargs := append([]string{"--kubeconfig", kubeconfigPath, "label"}, args...) + + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + ) + + stdout, stderr, err := applyCmd.Run(ctx) + if err != nil { + fmt.Println(string(stderr)) + return err + } + + fmt.Println(string(stdout)) + + return nil +} + +func KubectlDescribeWithNamespacedName(ctx context.Context, kubeconfigPath, resource, ns, name string) ([]byte, []byte, error) { + aargs := append([]string{"--kubeconfig", kubeconfigPath, "describe"}, resource, "-n", ns, name) + applyCmd := NewCommand( + WithCommand("kubectl"), + WithArgs(aargs...), + ) + + return applyCmd.Run(ctx) +} + +// Command wraps Command with specific functionality. +// This differentiates itself from the standard library by always collecting stdout and stderr. +// Command improves the UX of Command for our specific use case. +type Command struct { + Cmd string + Args []string + Stdin io.Reader +} + +// Option is a functional option type that modifies a Command. +type Option func(*Command) + +// NewCommand returns a configured Command. +func NewCommand(opts ...Option) *Command { + cmd := &Command{ + Stdin: nil, + } + for _, option := range opts { + option(cmd) + } + + return cmd +} + +// WithStdin sets up the command to read from this io.Reader. +func WithStdin(stdin io.Reader) Option { + return func(cmd *Command) { + cmd.Stdin = stdin + } +} + +// WithCommand defines the command to run such as `kubectl` or `kind`. +func WithCommand(command string) Option { + return func(cmd *Command) { + cmd.Cmd = command + } +} + +// WithArgs sets the arguments for the command such as `get pods -n kube-system` to the command `kubectl`. +func WithArgs(args ...string) Option { + return func(cmd *Command) { + cmd.Args = args + } +} + +// Run executes the command and returns stdout, stderr and the error if there is any. +func (c *Command) Run(ctx context.Context) ([]byte, []byte, error) { + cmd := exec.CommandContext(ctx, c.Cmd, c.Args...) //nolint:gosec + if c.Stdin != nil { + cmd.Stdin = c.Stdin + } + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, nil, err + } + + if err := cmd.Start(); err != nil { + return nil, nil, err + } + + output, err := io.ReadAll(stdout) + if err != nil { + return nil, nil, err + } + + errout, err := io.ReadAll(stderr) + if err != nil { + return nil, nil, err + } + + if err := cmd.Wait(); err != nil { + return output, errout, err + } + + return output, errout, nil +} diff --git a/test/e2e/framework/namespace.go b/test/e2e/framework/namespace.go new file mode 100644 index 000000000..4a5719d58 --- /dev/null +++ b/test/e2e/framework/namespace.go @@ -0,0 +1,212 @@ +/* +Copyright 2020 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// https://github.com/kubernetes-sigs/cluster-api/blob/0008b5ba109e839067b96f4c5b52eab9fb657eca/test/e2e/common.go +package framework + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/kubernetes/test/e2e/framework" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Creator can creates resources. +type Creator interface { + Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error +} + +type WatchNamespaceEventsInput struct { + ClientSet *kubernetes.Clientset + Name string + LogFolder string +} + +// WatchNamespaceEvents creates a watcher that streams namespace events into a file. +// Example usage: +// +// ctx, cancelWatches := context.WithCancel(context.Background()) +// go func() { +// defer GinkgoRecover() +// framework.WatchNamespaceEvents(ctx, framework.WatchNamespaceEventsInput{ +// ClientSet: clientSet, +// Name: namespace.Name, +// LogFolder: logFolder, +// }) +// }() +// defer cancelWatches() +func WatchNamespaceEvents(ctx context.Context, input WatchNamespaceEventsInput) { + Expect(ctx).NotTo(BeNil(), "ctx is required for WatchNamespaceEvents") + Expect(input.ClientSet).NotTo(BeNil(), "input.ClientSet is required for WatchNamespaceEvents") + Expect(input.Name).NotTo(BeEmpty(), "input.Name is required for WatchNamespaceEvents") + + logFile := filepath.Clean(path.Join(input.LogFolder, "resources", input.Name, "events.log")) + Expect(os.MkdirAll(filepath.Dir(logFile), 0750)).To(Succeed()) + + f, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + Expect(err).NotTo(HaveOccurred()) + + defer func() { _ = f.Close() }() + + informerFactory := informers.NewSharedInformerFactoryWithOptions( + input.ClientSet, + 10*time.Minute, + informers.WithNamespace(input.Name), + ) + eventInformer := informerFactory.Core().V1().Events().Informer() + _, err = eventInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + e := obj.(*corev1.Event) + _, _ = fmt.Fprintf(f, "[New Event] %s/%s\n\tresource: %s/%s/%s\n\treason: %s\n\tmessage: %s\n\tfull: %#v\n", + e.Namespace, e.Name, e.InvolvedObject.APIVersion, e.InvolvedObject.Kind, e.InvolvedObject.Name, e.Reason, e.Message, e) + }, + UpdateFunc: func(_, obj any) { + e := obj.(*corev1.Event) + _, _ = fmt.Fprintf(f, "[Updated Event] %s/%s\n\tresource: %s/%s/%s\n\treason: %s\n\tmessage: %s\n\tfull: %#v\n", + e.Namespace, e.Name, e.InvolvedObject.APIVersion, e.InvolvedObject.Kind, e.InvolvedObject.Name, e.Reason, e.Message, e) + }, + DeleteFunc: func(obj any) {}, + }) + Expect(err).NotTo(HaveOccurred(), "failed to add event handler for namespace %s", input.Name) + + stopInformer := make(chan struct{}) + defer close(stopInformer) + + informerFactory.Start(stopInformer) + <-ctx.Done() + + stopInformer <- struct{}{} +} + +// CreateNamespaceInput is the input type for CreateNamespace. +type CreateNamespaceInput struct { + Creator Creator + Name string + Labels map[string]string + + // IgnoreAlreadyExists if set to true will ignore the "AlreadyExists" error if the function + // is trying to create a namespace that already exists. + // If false, it will error and cause test failure. + IgnoreAlreadyExists bool +} + +// CreateNamespace is used to create a namespace object. +// If name is empty, a "test-" + util.RandomString(6) name will be generated. +func CreateNamespace(ctx context.Context, input CreateNamespaceInput, intervals ...any) *corev1.Namespace { + Expect(ctx).NotTo(BeNil(), "ctx is required for DeleteNamespace") + Expect(input.Creator).NotTo(BeNil(), "input.Creator is required for CreateNamespace") + + if input.Name == "" { + input.Name = fmt.Sprintf("test-%s", util.RandomString(6)) + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: input.Name, + Labels: input.Labels, + }, + } + framework.Logf("Creating namespace %s", input.Name) + Eventually(func() error { + err := input.Creator.Create(ctx, ns) + if err != nil { + if input.IgnoreAlreadyExists && apierrors.IsAlreadyExists(err) { + return nil + } + + return err + } + + return nil + }, intervals...).Should(Succeed(), "Failed to create namespace %s", input.Name) + + return ns +} + +// CreateNamespaceAndWatchEventsInput is the input type for CreateNamespaceAndWatchEvents. +type CreateNamespaceAndWatchEventsInput struct { + Creator Creator + ClientSet *kubernetes.Clientset + Name string + LogFolder string +} + +// CreateNamespaceAndWatchEvents creates a namespace and setups a watch for the namespace events. +func CreateNamespaceAndWatchEvents(ctx context.Context, input CreateNamespaceAndWatchEventsInput) (*corev1.Namespace, context.CancelFunc) { + Expect(ctx).NotTo(BeNil(), "ctx is required for CreateNamespaceAndWatchEvents") + Expect(input.Creator).ToNot(BeNil(), "Invalid argument. input.Creator can't be nil when calling CreateNamespaceAndWatchEvents") + Expect(input.ClientSet).ToNot(BeNil(), "Invalid argument. input.ClientSet can't be nil when calling ClientSet") + Expect(input.Name).ToNot(BeEmpty(), "Invalid argument. input.Name can't be empty when calling ClientSet") + Expect(os.MkdirAll(input.LogFolder, 0750)).To(Succeed(), "Invalid argument. input.LogFolder can't be created in CreateNamespaceAndWatchEvents") + + namespace := CreateNamespace(ctx, CreateNamespaceInput{Creator: input.Creator, Name: input.Name}, "40s", "10s") + Expect(namespace).ToNot(BeNil(), "Failed to create namespace %q", input.Name) + + framework.Logf("Creating event watcher for namespace %q", input.Name) + + watchesCtx, cancelWatches := context.WithCancel(ctx) + + go func() { + defer GinkgoRecover() + + WatchNamespaceEvents(watchesCtx, WatchNamespaceEventsInput{ + ClientSet: input.ClientSet, + Name: namespace.Name, + LogFolder: input.LogFolder, + }) + }() + + return namespace, cancelWatches +} + +// Deleter can delete resources. +type Deleter interface { + Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error +} + +// DeleteNamespaceInput is the input type for DeleteNamespace. +type DeleteNamespaceInput struct { + Deleter Deleter + Name string +} + +// DeleteNamespace is used to delete namespace object. +func DeleteNamespace(ctx context.Context, input DeleteNamespaceInput, intervals ...any) { + Expect(ctx).NotTo(BeNil(), "ctx is required for DeleteNamespace") + Expect(input.Deleter).NotTo(BeNil(), "input.Deleter is required for DeleteNamespace") + Expect(input.Name).NotTo(BeEmpty(), "input.Name is required for DeleteNamespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: input.Name, + }, + } + framework.Logf("Deleting namespace %s", input.Name) + Eventually(func() error { + return input.Deleter.Delete(ctx, ns) + }, intervals...).Should(Succeed(), "Failed to delete namespace %s", input.Name) +} diff --git a/test/e2e/framework/skipper.go b/test/e2e/framework/skipper.go new file mode 100644 index 000000000..3f3c05780 --- /dev/null +++ b/test/e2e/framework/skipper.go @@ -0,0 +1,120 @@ +/* +Copyright 2014 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// https://github.com/kubernetes/kubernetes/blob/8058942aa2552bcaf13208d1ea678e9f3f5f4605/test/e2e/framework/skipper/skipper.go +package framework + +import ( + "bufio" + "bytes" + "fmt" + "regexp" + "runtime" + "runtime/debug" + "strings" + + "github.com/onsi/ginkgo/v2" //nolint:depguard // E2E skipper mirrors k8s framework and must call ginkgo.Skip. + "k8s.io/kubernetes/test/e2e/framework" +) + +// Skipper is a generic function called by other test-specific skipper. +func SkipInternalf(caller int, format string, args ...any) { + msg := fmt.Sprintf(format, args...) + framework.Logf("%s", msg) + skip(msg, caller+1) +} + +// SkipPanic is the value that will be panicked from Skip. +type SkipPanic struct { + Message string // The failure message passed to Fail + Filename string // The filename that is the source of the failure + Line int // The line number of the filename that is the source of the failure + FullStackTrace string // A full stack trace starting at the source of the failure +} + +const ginkgoPanic = ` +Your test failed. +Ginkgo panics to prevent subsequent assertions from running. +Normally Ginkgo rescues this panic so you shouldn't see it. + +But, if you make an assertion in a goroutine, Ginkgo can't capture the panic. +To circumvent this, you should call + + defer GinkgoRecover() + +at the top of the goroutine that caused this panic. +` + +// String makes SkipPanic look like the old Ginkgo panic when printed. +func (SkipPanic) String() string { return ginkgoPanic } + +// Skip wraps ginkgo.Skip so that it panics with more useful +// information about why the test is being skipped. This function will +// panic with a SkipPanic. +func skip(message string, callerSkip ...int) { + skip := 1 + if len(callerSkip) > 0 { + skip += callerSkip[0] + } + + _, file, line, _ := runtime.Caller(skip) + sp := SkipPanic{ + Message: message, + Filename: file, + Line: line, + FullStackTrace: pruneStack(skip), + } + + defer func() { + e := recover() + if e != nil { + panic(sp) + } + }() + + ginkgo.Skip(message, skip) +} + +// ginkgo adds a lot of test running infrastructure to the stack, so +// we filter those out. +var stackSkipPattern = regexp.MustCompile(`onsi/ginkgo`) + +func pruneStack(skip int) string { + skip += 2 // one for pruneStack and one for debug.Stack + stack := debug.Stack() + scanner := bufio.NewScanner(bytes.NewBuffer(stack)) + + var prunedStack []string + + // skip the top of the stack + for i := 0; i < 2*skip+1; i++ { + scanner.Scan() + } + + for scanner.Scan() { + if stackSkipPattern.Match(scanner.Bytes()) { + scanner.Scan() // these come in pairs + } else { + prunedStack = append(prunedStack, scanner.Text()) + scanner.Scan() // these come in pairs + prunedStack = append(prunedStack, scanner.Text()) + } + } + + return strings.Join(prunedStack, "\n") +} + +// Skipf skips with information about why the test is being skipped. +func Skipf(format string, args ...any) { + SkipInternalf(1, format, args...) +} diff --git a/test/e2e/framework/state_booleans.go b/test/e2e/framework/state_booleans.go new file mode 100644 index 000000000..0af60cc82 --- /dev/null +++ b/test/e2e/framework/state_booleans.go @@ -0,0 +1,32 @@ +// Copyright (c) 2019 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package framework + +import "strings" + +// formal is test's requirement +// actual is the actual of system running the test + +// StateIs is a general concept of comparing an actual parameter with formal parameters. +func StateIs(sysActual string, testFormals ...string) bool { + for _, testFormal := range testFormals { + if strings.EqualFold(sysActual, testFormal) { + return true + } + } + + return false +} + +func InfraIs(configInfra string, testReqInfra ...string) bool { + return StateIs(configInfra, testReqInfra...) +} + +func CNIIs(configCNI string, testReqCNI ...string) bool { + return StateIs(configCNI, testReqCNI...) +} + +func NetworkTopologyIs(configNetworking string, testReqNetworking ...string) bool { + return StateIs(configNetworking, testReqNetworking...) +} diff --git a/test/e2e/framework/utils.go b/test/e2e/framework/utils.go new file mode 100644 index 000000000..65e0f82c4 --- /dev/null +++ b/test/e2e/framework/utils.go @@ -0,0 +1,65 @@ +package framework + +import ( + "context" + "io" + "os" + "path/filepath" + "syscall" + + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/podlogs" +) + +// https://github.com/kubernetes/kubernetes/blob/3d6026499b674020b4f8eec11f0b8a860a330d8a/test/e2e/storage/podlogs/podlogs.go +// https://github-vcf.devops.broadcom.net/vcf/vks-gce2e/blob/cd04354b144b707d113b21a740f83f156cabaea5/test/e2e/lib/e2e/lib.go#L42 +func WatchPodLogsAndEvents(ctx context.Context, cs kubernetes.Interface, podArtifactFolder, ns string) { + // Needed in case directory permission bits get messed up due to umask. + oldUMask := syscall.Umask(0) + + Expect(os.MkdirAll(podArtifactFolder, 0755)).To(Succeed(), "failed to create pod artifact log folder: %s", podArtifactFolder) + + logOutput, err := os.Create(filepath.Join(podArtifactFolder, "podStatus.txt")) + Expect(err).NotTo(HaveOccurred(), "failed to create file for pod status: %v", err) + + // Reset the umask. + syscall.Umask(oldUMask) + + to := podlogs.LogOutput{ + StatusWriter: logOutput, + // LogPathPrefix is used as both the initial path and the start of the filename, + // so we prefix it with 'logs-' to ensure pod events correctly end up in the same + // directory, and it's possible to distinguish logs and events by looking at the prefix. + LogPathPrefix: podArtifactFolder + "/logs-", + } + + Expect(podlogs.CopyAllLogs(ctx, cs, ns, to)).To(Succeed(), "collect pod logs failed") + + podEventLogFile := filepath.Join(podArtifactFolder, "podEvents.txt") + eventWriters := make([]io.Writer, 0) + + eventOutputFile, err := os.Create(podEventLogFile) + if err != nil { + e2eframework.Failf("failed to open event file %s : %+v", podEventLogFile, err) + } + + eventWriters = append(eventWriters, eventOutputFile) + + eventOutput := io.MultiWriter(eventWriters...) + + var eventLogCloser io.Closer + Expect(podlogs.WatchPods(ctx, cs, ns, eventOutput, eventLogCloser)).To(Succeed(), "error occurred during watching pod events") +} + +func WatchPodLogsAndEventsInNamespaces(ctx context.Context, watchNsList []string, cs kubernetes.Interface, podArtifactFolder string) context.CancelFunc { + watchesCtx, cancelWatches := context.WithCancel(ctx) + + for _, ns := range watchNsList { + WatchPodLogsAndEvents(watchesCtx, cs, podArtifactFolder, ns) + } + + return cancelWatches +} diff --git a/test/e2e/go.mod b/test/e2e/go.mod new file mode 100644 index 000000000..97f700af7 --- /dev/null +++ b/test/e2e/go.mod @@ -0,0 +1,260 @@ +module github.com/vmware-tanzu/vm-operator/test/e2e + +go 1.26.2 + +replace ( + github.com/onsi/ginkgo/v2 => github.com/onsi/ginkgo/v2 v2.28.1 + github.com/onsi/gomega => github.com/onsi/gomega v1.39.1 + github.com/vmware-tanzu/vm-operator => ../../ + github.com/vmware-tanzu/vm-operator/api => ../../api + github.com/vmware-tanzu/vm-operator/external/appplatform => ../../external/appplatform + github.com/vmware-tanzu/vm-operator/external/byok => ../../external/byok + github.com/vmware-tanzu/vm-operator/external/capabilities => ../../external/capabilities + github.com/vmware-tanzu/vm-operator/external/image-registry-operator => ../../external/image-registry-operator + github.com/vmware-tanzu/vm-operator/external/infra => ../../external/infra + github.com/vmware-tanzu/vm-operator/external/mobility-operator => ../../external/mobility-operator + github.com/vmware-tanzu/vm-operator/external/ncp => ../../external/ncp + github.com/vmware-tanzu/vm-operator/external/net-operator => ../../external/net-operator + github.com/vmware-tanzu/vm-operator/external/nsx-operator => ../../external/nsx-operator + github.com/vmware-tanzu/vm-operator/external/storage-policy-quota => ../../external/storage-policy-quota + github.com/vmware-tanzu/vm-operator/external/tanzu-topology => ../../external/tanzu-topology + github.com/vmware-tanzu/vm-operator/external/vsphere-csi-driver => ../../external/vsphere-csi-driver + github.com/vmware-tanzu/vm-operator/external/vsphere-policy => ../../external/vsphere-policy + github.com/vmware-tanzu/vm-operator/pkg/backup/api => ../../pkg/backup/api + github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels => ../../pkg/constants/testlabels + + k8s.io/api => k8s.io/api v0.35.4 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.35.4 + k8s.io/apimachinery => k8s.io/apimachinery v0.35.4 + k8s.io/apiserver => k8s.io/apiserver v0.35.4 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.35.4 + k8s.io/client-go => k8s.io/client-go v0.35.4 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.35.4 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.35.4 + k8s.io/component-base => k8s.io/component-base v0.35.4 + k8s.io/component-helpers => k8s.io/component-helpers v0.35.4 + k8s.io/controller-manager => k8s.io/controller-manager v0.35.4 + k8s.io/cri-api => k8s.io/cri-api v0.35.4 + k8s.io/cri-client => k8s.io/cri-client v0.35.4 + k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.35.4 + k8s.io/endpointslice => k8s.io/endpointslice v0.35.4 + k8s.io/externaljwt => k8s.io/externaljwt v0.35.4 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.35.4 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.35.4 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.35.4 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.35.4 + k8s.io/kubectl => k8s.io/kubectl v0.35.4 + k8s.io/mount-utils => k8s.io/mount-utils v0.35.4 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.35.4 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.35.4 + sigs.k8s.io/structured-merge-diff/v4 => sigs.k8s.io/structured-merge-diff/v4 v4.7.0 +) + +require ( + github.com/antchfx/htmlquery v1.3.6 + github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f + github.com/onsi/ginkgo/v2 v2.28.1 + github.com/onsi/gomega v1.39.1 + github.com/sirupsen/logrus v1.9.3 + github.com/vmware-tanzu/vm-operator/api v0.0.0-00010101000000-000000000000 + github.com/vmware-tanzu/vm-operator/external/image-registry-operator v0.0.0-00010101000000-000000000000 + github.com/vmware-tanzu/vm-operator/external/mobility-operator v0.0.0-00010101000000-000000000000 + github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20260414073537-dff710623a20 + github.com/vmware-tanzu/vm-operator/external/net-operator v0.0.0-00010101000000-000000000000 + github.com/vmware-tanzu/vm-operator/external/nsx-operator v0.0.0-00010101000000-000000000000 + github.com/vmware-tanzu/vm-operator/external/storage-policy-quota v0.0.0-20260414073537-dff710623a20 + github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20260414073537-dff710623a20 + github.com/vmware-tanzu/vm-operator/external/vsphere-csi-driver v0.0.0-20260414073537-dff710623a20 + github.com/vmware-tanzu/vm-operator/pkg/backup/api v0.0.0-20260414073537-dff710623a20 + github.com/vmware/govmomi v0.53.0 + golang.org/x/crypto v0.50.0 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.35.4 + k8s.io/apiextensions-apiserver v0.35.4 + k8s.io/apimachinery v0.35.4 + k8s.io/client-go v0.35.4 + k8s.io/klog/v2 v2.140.0 + k8s.io/kubernetes v0.0.0-00010101000000-000000000000 + sigs.k8s.io/cluster-api v1.12.5 + sigs.k8s.io/controller-runtime v0.23.3 + sigs.k8s.io/yaml v1.6.0 +) + +require ( + github.com/pkg/errors v0.9.1 // indirect + k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 // indirect +) + +require ( + cel.dev/expr v0.25.1 // indirect + cyphar.com/go-pathrs v0.2.4 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/antchfx/xpath v1.3.6 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gkampitakis/go-snaps v0.5.21 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.22.5 // indirect + github.com/go-openapi/jsonreference v0.21.5 // indirect + github.com/go-openapi/swag v0.25.5 // indirect + github.com/go-openapi/swag/cmdutils v0.25.5 // indirect + github.com/go-openapi/swag/conv v0.25.5 // indirect + github.com/go-openapi/swag/fileutils v0.25.5 // indirect + github.com/go-openapi/swag/jsonname v0.25.5 // indirect + github.com/go-openapi/swag/jsonutils v0.25.5 // indirect + github.com/go-openapi/swag/loading v0.25.5 // indirect + github.com/go-openapi/swag/mangling v0.25.5 // indirect + github.com/go-openapi/swag/netutils v0.25.5 // indirect + github.com/go-openapi/swag/stringutils v0.25.5 // indirect + github.com/go-openapi/swag/typeutils v0.25.5 // indirect + github.com/go-openapi/swag/yamlutils v0.25.5 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.26.0 // indirect + github.com/google/gnostic-models v0.7.1 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f // indirect + github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.5 // indirect + github.com/moby/spdystream v0.5.1 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/selinux v1.13.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.20.1 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stoewer/go-strcase v1.3.1 // indirect + github.com/vmware-tanzu/vm-operator v1.9.0 + github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/term v0.42.0 // indirect + golang.org/x/text v0.36.0 // indirect + golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.44.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/grpc v1.80.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/apiserver v0.35.4 // indirect + k8s.io/component-base v0.35.4 // indirect + k8s.io/component-helpers v0.35.4 // indirect + k8s.io/controller-manager v0.0.0 // indirect + k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 // indirect + k8s.io/kubectl v0.0.0 // indirect + k8s.io/kubelet v0.0.0 // indirect + k8s.io/mount-utils v0.0.0 // indirect + k8s.io/pod-security-admission v0.0.0 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect +) + +replace ( + github.com/gobuffalo/flect => github.com/gobuffalo/flect v1.0.2 + github.com/golang/glog => github.com/golang/glog v1.1.0 + github.com/google/gofuzz => github.com/google/gofuzz v1.2.0 + github.com/kr/pretty => github.com/kr/pretty v0.3.1 + github.com/moby/sys/mountinfo => github.com/moby/sys/mountinfo v0.6.2 + github.com/tidwall/gjson => github.com/tidwall/gjson v1.14.4 + github.com/tidwall/pretty => github.com/tidwall/pretty v1.2.1 + github.com/tidwall/sjson => github.com/tidwall/sjson v1.2.5 +) + +replace ( + github.com/go-logr/zapr => github.com/go-logr/zapr v1.3.0 + github.com/go-task/slim-sprig/v3 => github.com/go-task/slim-sprig/v3 v3.0.0 + github.com/stretchr/testify => github.com/stretchr/testify v1.9.0 + + go.uber.org/goleak => go.uber.org/goleak v1.3.0 + gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c +) + +replace github.com/stretchr/objx => github.com/stretchr/objx v0.5.2 + +replace github.com/tidwall/match => github.com/tidwall/match v1.1.1 + +replace ( + github.com/dougm/pretty => github.com/dougm/pretty v0.0.0-20160325215624-add1dbc86daf + github.com/inconshreveable/mousetrap => github.com/inconshreveable/mousetrap v1.1.0 + github.com/joshdk/go-junit => github.com/joshdk/go-junit v1.0.0 + github.com/mfridman/tparse => github.com/mfridman/tparse v0.18.0 +) + +replace ( + github.com/gkampitakis/ciinfo => github.com/gkampitakis/ciinfo v0.3.4 + github.com/goccy/go-yaml => github.com/goccy/go-yaml v1.19.2 + github.com/maruel/natural => github.com/maruel/natural v1.3.0 +) + +replace ( + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 +) + +replace k8s.io/kubelet => k8s.io/kubelet v0.35.4 + +replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.35.4 + +replace k8s.io/kubernetes => k8s.io/kubernetes v1.35.4 + +replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.1 + +replace go.uber.org/zap => go.uber.org/zap v1.26.0 + +replace go.uber.org/multierr => go.uber.org/multierr v1.10.0 + +replace github.com/kylelemons/godebug => github.com/kylelemons/godebug v1.1.0 + +replace github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 + +replace github.com/pmezard/go-difflib => github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 + +replace github.com/google/cel-go => github.com/google/cel-go v0.22.1 diff --git a/test/e2e/go.sum b/test/e2e/go.sum new file mode 100644 index 000000000..40654b07f --- /dev/null +++ b/test/e2e/go.sum @@ -0,0 +1,444 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cyphar.com/go-pathrs v0.2.4 h1:iD/mge36swa1UFKdINkr1Frkpp6wZsy3YYEildj9cLY= +cyphar.com/go-pathrs v0.2.4/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antchfx/htmlquery v1.3.6 h1:RNHHL7YehO5XdO8IM8CynwLKONwRHWkrghbYhQIk9ag= +github.com/antchfx/htmlquery v1.3.6/go.mod h1:kcVUqancxPygm26X2rceEcagZFFVkLEE7xgLkGSDl/4= +github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= +github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dougm/pretty v0.0.0-20160325215624-add1dbc86daf h1:A2XbJkAuMMFy/9EftoubSKBUIyiOm6Z8+X5G7QpS6so= +github.com/dougm/pretty v0.0.0-20160325215624-add1dbc86daf/go.mod h1:7NQ3kWOx2cZOSjtcveTa5nqupVr2s6/83sG+rTlI7uA= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.4 h1:5eBSibVuSMbb/H6Elc0IIEFbkzCJi3lm94n0+U7Z0KY= +github.com/gkampitakis/ciinfo v0.3.4/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-snaps v0.5.21 h1:SvhSFeZviQXwlT+dnGyAIATVehkhqRVW6qfQZhCZH+Y= +github.com/gkampitakis/go-snaps v0.5.21/go.mod h1:gC3YqxQTPyIXvQrw/Vpt3a8VqR1MO8sVpZFWN4DGwNs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= +github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= +github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE= +github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw= +github.com/go-openapi/swag v0.25.5 h1:pNkwbUEeGwMtcgxDr+2GBPAk4kT+kJ+AaB+TMKAg+TU= +github.com/go-openapi/swag v0.25.5/go.mod h1:B3RT6l8q7X803JRxa2e59tHOiZlX1t8viplOcs9CwTA= +github.com/go-openapi/swag/cmdutils v0.25.5 h1:yh5hHrpgsw4NwM9KAEtaDTXILYzdXh/I8Whhx9hKj7c= +github.com/go-openapi/swag/cmdutils v0.25.5/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.5 h1:wAXBYEXJjoKwE5+vc9YHhpQOFj2JYBMF2DUi+tGu97g= +github.com/go-openapi/swag/conv v0.25.5/go.mod h1:CuJ1eWvh1c4ORKx7unQnFGyvBbNlRKbnRyAvDvzWA4k= +github.com/go-openapi/swag/fileutils v0.25.5 h1:B6JTdOcs2c0dBIs9HnkyTW+5gC+8NIhVBUwERkFhMWk= +github.com/go-openapi/swag/fileutils v0.25.5/go.mod h1:V3cT9UdMQIaH4WiTrUc9EPtVA4txS0TOmRURmhGF4kc= +github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo= +github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= +github.com/go-openapi/swag/jsonutils v0.25.5 h1:XUZF8awQr75MXeC+/iaw5usY/iM7nXPDwdG3Jbl9vYo= +github.com/go-openapi/swag/jsonutils v0.25.5/go.mod h1:48FXUaz8YsDAA9s5AnaUvAmry1UcLcNVWUjY42XkrN4= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5 h1:SX6sE4FrGb4sEnnxbFL/25yZBb5Hcg1inLeErd86Y1U= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5/go.mod h1:/2KvOTrKWjVA5Xli3DZWdMCZDzz3uV/T7bXwrKWPquo= +github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU= +github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g= +github.com/go-openapi/swag/mangling v0.25.5 h1:hyrnvbQRS7vKePQPHHDso+k6CGn5ZBs5232UqWZmJZw= +github.com/go-openapi/swag/mangling v0.25.5/go.mod h1:6hadXM/o312N/h98RwByLg088U61TPGiltQn71Iw0NY= +github.com/go-openapi/swag/netutils v0.25.5 h1:LZq2Xc2QI8+7838elRAaPCeqJnHODfSyOa7ZGfxDKlU= +github.com/go-openapi/swag/netutils v0.25.5/go.mod h1:lHbtmj4m57APG/8H7ZcMMSWzNqIQcu0RFiXrPUara14= +github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M= +github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII= +github.com/go-openapi/swag/typeutils v0.25.5 h1:EFJ+PCga2HfHGdo8s8VJXEVbeXRCYwzzr9u4rJk7L7E= +github.com/go-openapi/swag/typeutils v0.25.5/go.mod h1:itmFmScAYE1bSD8C4rS0W+0InZUBrB2xSPbWt6DLGuc= +github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ= +github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeDPuAvB/xWrdxFJkoFag= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM= +github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= +github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= +github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.22.1 h1:AfVXx3chM2qwoSbM7Da8g8hX8OVSkBFwX+rz2+PcK40= +github.com/google/cel-go v0.22.1/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= +github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= +github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f h1:7MmqygqdeJtziBUpm4Z9ThROFZUaVGaePMfcDnluf1E= +github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f/go.mod h1:n1ej5+FqyEytMt/mugVDZLIiqTMO+vsrgY+kM6ohzN0= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f h1:5CjVwnuUcp5adK4gmY6i72gpVFVnZDP2h5TmPScB6u4= +github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= +github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= +github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/selinux v1.13.0 h1:Zza88GWezyT7RLql12URvoxsbLfjFx988+LGaWfbL84= +github.com/opencontainers/selinux v1.13.0/go.mod h1:XxWTed+A/s5NNq4GmYScVy+9jzXhGBVEOAyucdRUY8s= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= +github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/vmware/govmomi v0.53.0 h1:e1bZCotAq7wm4xy95ePN2uoWwz28pNp/ewZZhpBY7/4= +github.com/vmware/govmomi v0.53.0/go.mod h1:EWfuzPfxT5NV+aS2we02SLFdhvJkgeY7t7+TszgBSMY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/ziutek/telnet v0.0.0-20180329124119-c3b780dc415b/go.mod h1:IZpXDfkJ6tWD3PhBK5YzgQT+xJWh7OsdwiG8hA2MkO4= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 h1:RAE+JPfvEmvy+0LzyUA25/SGawPwIUbZ6u0Wug54sLc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0/go.mod h1:AGmbycVGEsRx9mXMZ75CsOyhSP6MFIcj/6dnG+vhVjk= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= +gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.35.4 h1:P7nFYKl5vo9AGUp1Z+Pmd3p2tA7bX2wbFWCvDeRv988= +k8s.io/api v0.35.4/go.mod h1:yl4lqySWOgYJJf9RERXKUwE9g2y+CkuwG+xmcOK8wXU= +k8s.io/apiextensions-apiserver v0.35.4 h1:HeP+Upp7ItdvnyGmub0yoix+2z5+ev4M5cE5TCgtOUU= +k8s.io/apiextensions-apiserver v0.35.4/go.mod h1:ogQlk+stIE8mnoRthSYCwlOS12fVqgWFiErMwPaXA7c= +k8s.io/apimachinery v0.35.4 h1:xtdom9RG7e+yDp71uoXoJDWEE2eOiHgeO4GdBzwWpds= +k8s.io/apimachinery v0.35.4/go.mod h1:NNi1taPOpep0jOj+oRha3mBJPqvi0hGdaV8TCqGQ+cc= +k8s.io/apiserver v0.35.4 h1:vtuFqNFmF9bPRdHDL2lpK6qCTPWDreZJL4LRPwVM6ho= +k8s.io/apiserver v0.35.4/go.mod h1:JnBcb+J8kFXKpZkgcbcUnPBBHi4qgBii1I7dLxFY/oo= +k8s.io/client-go v0.35.4 h1:DN6fyaGuzK64UvnKO5fOA6ymSjvfGAnCAHAR0C66kD8= +k8s.io/client-go v0.35.4/go.mod h1:2Pg9WpsS4NeOpoYTfHHfMxBG8zFMSAUi4O/qoiJC3nY= +k8s.io/component-base v0.35.4 h1:6n1tNJ87johN0Hif0Fs8K2GMthsaUwMqCebUDLYyv7U= +k8s.io/component-base v0.35.4/go.mod h1:qaDJgz5c1KYKla9occFmlJEfPpkuA55s90G509R+PeY= +k8s.io/component-helpers v0.35.4 h1:WJM/+fAeeJTAqxPDxgH0aB0q7t8DP+AbV5WkRkOoxYA= +k8s.io/component-helpers v0.35.4/go.mod h1:mE7X9mnMQEX6IbZejdMlWvCx3EPVt1/9PhH/FW0XHDI= +k8s.io/controller-manager v0.35.4 h1:hkd4rVb39Xmb/zCElz4fDc/xUgVHXZcrZpa+Wy09qVc= +k8s.io/controller-manager v0.35.4/go.mod h1:3ETsjkqokyv3uLkb7Miz7ZS1GweLcQFt2TnA+gGyP5k= +k8s.io/csi-translation-lib v0.35.4 h1:r9p+eDGywrUxoWoC6dPdCOZpGe9XEivCNKksrgVW/ZE= +k8s.io/csi-translation-lib v0.35.4/go.mod h1:JHjdsj4zeazo+GtGoK5L1e0aDO3xgXV2Co9or/k+Kcs= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 h1:V+sn9a/1fEYDGwnllCmqXBk8x7obZ+hl869Q3Abumkg= +k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/kubectl v0.35.4 h1:IHitney6OUeH29rBQnt6Cas6az8HpFeSAohormITNMc= +k8s.io/kubectl v0.35.4/go.mod h1:CGWAaof9ae4vGDAyhnSf1bSQN/U7jiWQHLVbMbLMjRI= +k8s.io/kubelet v0.35.4 h1:g/qX1F6PdJQYzAzje3BDRGGEAmeYiiRi9QlLuyliRyw= +k8s.io/kubelet v0.35.4/go.mod h1:T3X1s+/TM23j8j3hjIem0PCBoSc7VNaKDyOkzAHUiDU= +k8s.io/kubernetes v1.35.4 h1:o/8dBC/pHVpYoGV4OAIytAlNPZbdYVTXJHoYvXC4qzM= +k8s.io/kubernetes v1.35.4/go.mod h1:fPfnQs8GtfrLQ+KuOcpvwQ+mV17jVcgdvPL6ZHxKp10= +k8s.io/mount-utils v0.35.4 h1:CRlXPCzdoFZ0sR+W42nX9NH67aV+YxMhp5yyu4feEY8= +k8s.io/mount-utils v0.35.4/go.mod h1:ppC4d+mUpfbAJr/V2E8vvxeCEckNM+S5b0kQBQjd3Pw= +k8s.io/pod-security-admission v0.35.4 h1:2IK2zlpjCPCgtJwtGGIxP9rIj4mwUp5nQ2EKPn940E4= +k8s.io/pod-security-admission v0.35.4/go.mod h1:5D35RsrmTiKQ+qHm8FP+xReIKQodHSy/9I/TfejFjKQ= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/cluster-api v1.12.5 h1:kPfz096EpFuPz0lRurJvhSIzUgDfdbHzP68wOrZPb/M= +sigs.k8s.io/cluster-api v1.12.5/go.mod h1:RdmTGGRMvAGIIQBljHUHNov/6Lgz7rmYXqzZNCK+Z4o= +sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80= +sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/test/e2e/infrastructure/vsphere/dcli/command_runner.go b/test/e2e/infrastructure/vsphere/dcli/command_runner.go new file mode 100644 index 000000000..2ee3bcbec --- /dev/null +++ b/test/e2e/infrastructure/vsphere/dcli/command_runner.go @@ -0,0 +1,115 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package dcli + +import ( + "bytes" + "context" + "fmt" + "strings" + "time" + + "github.com/vmware/govmomi/vim25/types" + "golang.org/x/crypto/ssh" + "k8s.io/apimachinery/pkg/util/wait" + + e2essh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" +) + +// DCLICommandRunner knows how to run DCLI commands on a vCenter instance. +type DCLICommandRunner interface { + RunDCLICommand(string) ([]byte, error) + RunCommandAndUnmarshalJSONResult(string, any) error +} + +// These credentials are useful both for VC API and kubectl plugin interactions. +type VCenterUserCredentials struct { + Username string + Password string +} + +type dcliCommandRunnerImpl struct { + vCenterHostname string + vCenterPort int + adminCredentials VCenterUserCredentials + sshHelper e2essh.SSHCommandRunner +} + +func NewDCLICommandRunner(vCenterHostname string, vCenterPort int, adminUsername, adminPassword, sshUser, sshPassword string) (DCLICommandRunner, error) { + return newDCLICommandRunnerImpl(vCenterHostname, vCenterPort, adminUsername, adminPassword, sshUser, sshPassword) +} + +func newDCLICommandRunnerImpl(vCenterHostname string, vCenterPort int, adminUsername, adminPassword, sshUser, sshPassword string) (*dcliCommandRunnerImpl, error) { + newSSHCommandRunner, err := e2essh.NewSSHCommandRunner(vCenterHostname, vCenterPort, sshUser, []ssh.AuthMethod{ssh.Password(sshPassword)}) + if err != nil { + return nil, err + } + + retVal := &dcliCommandRunnerImpl{ + vCenterHostname: vCenterHostname, + vCenterPort: vCenterPort, + adminCredentials: VCenterUserCredentials{Username: adminUsername, Password: adminPassword}, + sshHelper: newSSHCommandRunner, + } + + return retVal, nil +} + +func (d *dcliCommandRunnerImpl) RunCommandAndUnmarshalJSONResult(cmd string, unmarshalDestination any) error { + output, err := d.RunDCLICommand(cmd) + if err != nil { + return err + } + + dec := types.NewJSONDecoder(bytes.NewReader(output)) + + return dec.Decode(unmarshalDestination) +} + +func (d *dcliCommandRunnerImpl) RunDCLICommand(cmd string) ([]byte, error) { + cmdWithCreds := addDCLIParameters(cmd, d.adminCredentials.Username, d.adminCredentials.Password) + fmt.Printf("\nRunning command: %s", cmdWithCreds) + + var ( + stdout []byte + lastErr error + ) + + // Retry on transient errors like SERVICE_UNAVAILABLE + pollErr := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) { + var err error + + stdout, err = d.sshHelper.RunCommand(cmdWithCreds) + fmt.Printf("\nSTDOUT: %s", string(stdout)) + + if err == nil { + return true, nil + } + + lastErr = err + + // Check for transient errors that should be retried + errStr := err.Error() + string(stdout) + if strings.Contains(errStr, "ServiceUnavailable") || strings.Contains(errStr, "SERVICE_UNAVAILABLE") { + fmt.Printf("\nTransient error detected, retrying...") + return false, nil // Continue polling + } + + // Non-transient error, stop polling + return false, err + }) + if pollErr != nil { + if lastErr != nil { + return stdout, lastErr + } + + return stdout, pollErr + } + + return stdout, nil +} + +func addDCLIParameters(cmd, username, password string) string { + return fmt.Sprintf("dcli %s +username '%s' +password '%s'", cmd, username, password) +} diff --git a/test/e2e/infrastructure/vsphere/kubectl/auth.go b/test/e2e/infrastructure/vsphere/kubectl/auth.go new file mode 100644 index 000000000..eb4df2759 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/kubectl/auth.go @@ -0,0 +1,44 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package kubectl + +import ( + "context" + "strings" + "time" + + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" +) + +// AssertKubectlUserCan uses kubectl auth can-i to verify a user's privileges. +func AssertKubectlUserCan(ctx context.Context, kubeconfigPath string, args ...string) { + Eventually(func() bool { + // We ignore errors as they can come from two possible sources- + // 1) RBAC rules do not allow resource access. + // 2) kubectl command failed for some reason. (eg. network blip) + // In either case, the result will not be 'yes', and the + // Eventually() means it'll be retried. + kubectlArgs := []string{"can-i"} + kubectlArgs = append(kubectlArgs, args...) + result, _ := framework.KubectlAuth(ctx, kubeconfigPath, kubectlArgs...) + + return strings.TrimSpace(string(result)) == "yes" + }, 1*time.Minute, 5*time.Second).Should(BeTrue(), "User should eventually be able to", args) +} + +// AssertKubectlUserCannot uses kubectl auth can-i to verify the absence of a user's privileges. +func AssertKubectlUserCannot(ctx context.Context, kubeconfigPath string, args ...string) { + // This is a bit of a hack, but passing in the flag twice causes the second one to be preferred. + // It's a cheap way of overriding kubectl. + Eventually(func() bool { + kubectlArgs := []string{"can-i"} + kubectlArgs = append(kubectlArgs, args...) + result, _ := framework.KubectlAuth(ctx, kubeconfigPath, kubectlArgs...) + // Don't use an Expect() inside this function as that will prematurely fail the test. + // The assertion is that this function should eventually return true. + return strings.TrimSpace(string(result)) == "no" + }, 2*time.Minute, 5*time.Second).Should(BeTrue(), "User should not be able to", args) +} diff --git a/test/e2e/infrastructure/vsphere/kubectl/cluster.go b/test/e2e/infrastructure/vsphere/kubectl/cluster.go new file mode 100644 index 000000000..1a4cf9de2 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/kubectl/cluster.go @@ -0,0 +1,72 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package kubectl + +import ( + "context" + "fmt" + "strings" + + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" +) + +const ( + getContextClusterNameJSONPath = `jsonpath={.contexts[?(@.name == "%s")].context.cluster}` + getClusterAddressJSONPath = `jsonpath={.clusters[?(@.name == "%s")].cluster.server}` + getClusterCADataJSONPath = `jsonpath={.clusters[?(@.name == "%s")].cluster.certificate-authority-data}` +) + +// Get the current cluster from the current context +// kubectl config view -o jsonpath='{.contexts[?(@.name == "test-gc-e2e-demo-ns")].context.cluster} +// Return the address of the cluster associated with the current context. +func GetKubectlClusterForCurrentContext(ctx context.Context, kubeconfigPath string) string { + stdout, err := framework.KubectlConfig(ctx, kubeconfigPath, "current-context") + Expect(err).NotTo(HaveOccurred(), "get current context failed with kubeconfig %q", kubeconfigPath) + + currentContext := strings.TrimSpace(string(stdout)) + + return GetKubectlClusterAPIServerFromContext(ctx, currentContext, kubeconfigPath) +} + +// GetClusterAddressFromCluster returns the server address for the given context in a kubeconfig file. +func GetKubectlClusterAPIServerFromContext(ctx context.Context, contextName string, kubeconfigPath string) string { + kubectlArgs := []string{"view", "-o", fmt.Sprintf(getContextClusterNameJSONPath, contextName)} + stdout, err := framework.KubectlConfig(ctx, kubeconfigPath, kubectlArgs...) + Expect(err).NotTo(HaveOccurred(), "get context cluster failed with kubeconfig %q", kubeconfigPath) + + currentCluster := strings.TrimSpace(string(stdout)) + + return GetClusterAddressFromCluster(ctx, currentCluster, kubeconfigPath) +} + +// GetClusterAddressFromCluster returns the server address for the given cluster in a kubeconfig file. +func GetClusterAddressFromCluster(ctx context.Context, clusterName string, kubeconfigPath string) string { + kubectlArgs := []string{"view", "-o", fmt.Sprintf(getClusterAddressJSONPath, clusterName), "--kubeconfig", kubeconfigPath} + stdout, err := framework.KubectlConfig(ctx, kubeconfigPath, kubectlArgs...) + Expect(err).NotTo(HaveOccurred(), "get cluster server address failed with kubeconfig %q", kubeconfigPath) + + return strings.TrimSpace(string(stdout)) +} + +// GetClusterCertificateAuthorityDataFromCluster returns the CA data for the given context. +func GetClusterCertificateAuthorityDataFromContext(ctx context.Context, contextName string, kubeconfigPath string) string { + kubectlArgs := []string{"view", "-o", fmt.Sprintf(getContextClusterNameJSONPath, contextName)} + stdout, err := framework.KubectlConfig(ctx, kubeconfigPath, kubectlArgs...) + Expect(err).NotTo(HaveOccurred(), "get cluster context with kubeconfig %q", kubeconfigPath) + + clusterName := strings.TrimSpace(string(stdout)) + + return GetClusterCertificateAuthorityDataFromCluster(ctx, clusterName, kubeconfigPath) +} + +// GetClusterCertificateAuthorityDataFromCluster returns the CA data for the given cluster. +func GetClusterCertificateAuthorityDataFromCluster(ctx context.Context, clusterName string, kubeconfigPath string) string { + kubectlArgs := []string{"view", "-o", fmt.Sprintf(getClusterCADataJSONPath, clusterName)} + stdout, err := framework.KubectlConfig(ctx, kubeconfigPath, kubectlArgs...) + Expect(err).NotTo(HaveOccurred(), "get CA failed with kubeconfig %q", kubeconfigPath) + + return strings.TrimSpace(string(stdout)) +} diff --git a/test/e2e/infrastructure/vsphere/kubectl/plugin.go b/test/e2e/infrastructure/vsphere/kubectl/plugin.go new file mode 100644 index 000000000..4120241c4 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/kubectl/plugin.go @@ -0,0 +1,180 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package kubectl + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + expect "github.com/google/goexpect" +) + +const ( + kubectlCommand = "kubectl" + pluginCommand = "kubectl-vsphere" +) + +// Libraries to run the kubectl plugin against a WCP testbed. +// TODO a helper to download it as part of the framework (eg. BeforeSuite or something) would be nice, to allow us to move +// away from bash. However, that doesn't buy us much in the short term, so leaving this as a TODO for now. + +// KubectlPlugin knows how to run commands using the kubectl-vsphere plugin. +// It assumes that the kubectl-vsphere plugin is in the user's PATH. +type KubectlPlugin struct { + username string + password string + server string + insecure bool + kubeconfigPath string + tanzuKubernetesClusterName string + tanzuKubernetesClusterNamespace string +} + +func NewKubectlPlugin(kubeconfigPath string) *KubectlPlugin { + return &KubectlPlugin{kubeconfigPath: kubeconfigPath} +} + +// WithUsername configures the username to use when logging in. +func (k *KubectlPlugin) WithUsername(username string) *KubectlPlugin { + k.username = username + return k +} + +// WithPassword configures the password to use when logging in. +func (k *KubectlPlugin) WithPassword(password string) *KubectlPlugin { + k.password = password + return k +} + +// WithServer sets the supervisor cluster to log in to. +func (k *KubectlPlugin) WithServer(server string) *KubectlPlugin { + k.server = server + return k +} + +// WithInsecureFlag runs the kubectl-vsphere plugin (and thus sets up the kubeconfig contexts) +// to ignore TLS checks. +func (k *KubectlPlugin) WithInsecureFlag(insecure bool) *KubectlPlugin { + k.insecure = insecure + return k +} + +// WithTanzuKubernetesClusterName sets the name for the TKC to be logged in to. +func (k *KubectlPlugin) WithTanzuKubernetesClusterName(name string) *KubectlPlugin { + k.tanzuKubernetesClusterName = name + return k +} + +// WithTanzuKubernetesClusterNamespace sets the namespace for the TKC to be logged in to. +func (k *KubectlPlugin) WithTanzuKubernetesClusterNamespace(namespace string) *KubectlPlugin { + k.tanzuKubernetesClusterNamespace = namespace + return k +} + +var passwordPromptRE = regexp.MustCompile("Password:") + +// noopWriteCloser is a WriteCloser that does nothing when Close is called, so that goexpect doesn't cause a panic. +type noopWriteCloser struct { + io.Writer +} + +func (*noopWriteCloser) Close() error { + return nil +} + +// Login runs the kubectl-vsphere plugin to login to the supervisor cluster that the plugin +// is configured with. +func (k *KubectlPlugin) Login() error { + kubectlPath, err := checkPluginExistsInPath() + if err != nil { + return fmt.Errorf("failed to find kubectl or kubectl-vsphere in PATH: %w", err) + } + // TODO Support non vsphere.local domains + vsphereUsername := k.username + if !strings.HasSuffix(k.username, "@vsphere.local") { + vsphereUsername = k.username + "@vsphere.local" + } + + args := []string{"login", "--vsphere-username", vsphereUsername, "--server", k.server} + if k.insecure { + args = append(args, "--insecure-skip-tls-verify") + } + + if k.tanzuKubernetesClusterName != "" { + args = append(args, "--tanzu-kubernetes-cluster-name", k.tanzuKubernetesClusterName) + } + + if k.tanzuKubernetesClusterNamespace != "" { + args = append(args, "--tanzu-kubernetes-cluster-namespace", k.tanzuKubernetesClusterNamespace) + } + + kcmd, errch, err := expect.Spawn("kubectl-vsphere "+strings.Join(args, " "), 2*time.Minute, //nolint:mnd + expect.Tee(&noopWriteCloser{os.Stdout}), + expect.SetEnv([]string{ + "KUBECONFIG=" + k.kubeconfigPath, + "HTTP_PROXY=" + os.Getenv("HTTP_PROXY"), + "HTTPS_PROXY=" + os.Getenv("HTTPS_PROXY"), + "PATH=" + filepath.Dir(kubectlPath), + }), + ) + if err != nil { + return fmt.Errorf("failed to spawn kubectl-vsphere: %w", err) + } + + defer func() { _ = kcmd.Close() }() + + if out, _, err := kcmd.Expect(passwordPromptRE, -1); err != nil { + return fmt.Errorf("failed to get password prompt: %w, output: %s", err, out) + } + + if err := kcmd.Send(k.password + "\n"); err != nil { + return fmt.Errorf("failed to send password: %+w", err) + } + + if err := <-errch; err != nil { + return fmt.Errorf("failed to login: %+w", err) + } + + return nil +} + +// Helper method that looks for the kubectl and kubectl-vsphere binaries in the user's +// PATH, and returns the paths to each, or an error if one of them is not found. +func checkPluginExistsInPath() (string, error) { + kubectlPath, err := exec.LookPath(kubectlCommand) + if err != nil { + return "", fmt.Errorf("failed to find '%s' in PATH: %+w", kubectlCommand, err) + } + + _, err = exec.LookPath(pluginCommand) + if err != nil { + return "", fmt.Errorf("failed to find '%s' in PATH: %+w", pluginCommand, err) + } + + return kubectlPath, nil +} + +// Logout removes the WCP enabled clusters and associated credentials from the plugin's kubeconfig file. +func (k *KubectlPlugin) Logout() error { + _, err := checkPluginExistsInPath() + if err != nil { + return fmt.Errorf("failed to find kubectl or kubectl-vsphere in PATH: %+w", err) + } + + args := []string{"logout"} + cmd := exec.Command("kubectl-vsphere", args...) + + return cmd.Run() +} + +// KubeconfigPath returns the path to the kubeconfig file that the plugin is configured with. +func (k *KubectlPlugin) KubeconfigPath() string { + return k.kubeconfigPath +} diff --git a/test/e2e/infrastructure/vsphere/ssh/command_runner.go b/test/e2e/infrastructure/vsphere/ssh/command_runner.go new file mode 100644 index 000000000..8e4b17fa0 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/ssh/command_runner.go @@ -0,0 +1,205 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ssh + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "strings" + "time" + + "golang.org/x/crypto/ssh" + "k8s.io/apimachinery/pkg/util/wait" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" +) + +const ( + sshRetryDuration = time.Minute + sshRetryInterval = 5 * time.Second +) + +// SSHCommandRunner knows how to run SSH commands on an arbitrary host. +// +// Useful if users want to directly run commands, eg. on the VC or a GC node. +// +// RunCommand runs the given command on the host the helper is configured with, returning its STDOUT. +type SSHCommandRunner interface { + // Run a command and return its STDOUT. + RunCommand(cmd string) ([]byte, error) + RunCommandWindows(cmd string) ([]byte, error) + Close() error +} + +// sshCommandRunnerImpl implements the SSHCommandRunner interface, using the Golang ssh library. +type sshCommandRunnerImpl struct { + client *ssh.Client +} + +type Gateway struct { + Hostname string + Username string + Port int + AuthMethods []ssh.AuthMethod +} + +// NewSSHCommandRunner returns a helper configured based on the HTTP_PROXY set in the env. +func NewSSHCommandRunnerFromHTTPProxy() (SSHCommandRunner, error) { + gateway, err := GetGateway() + if err != nil { + return nil, err + } + + return NewSSHCommandRunner(gateway.Hostname, gateway.Port, gateway.Username, gateway.AuthMethods) +} + +func newSSHDialerWithRetries(hostname string, port int, config *ssh.ClientConfig, interval time.Duration, timeout time.Duration) (*ssh.Client, error) { + var newClient *ssh.Client + + //nolint:staticcheck // E2E SSH dial retries; migration to PollUntilContextTimeout is tracked separately. + err := wait.PollWithContext(context.Background(), interval, timeout, func(ctx context.Context) (bool, error) { + var err error + if newClient, err = ssh.Dial("tcp", fmt.Sprintf("%s:%d", hostname, port), config); err != nil { + return false, err + } + + return true, nil + }) + + return newClient, err +} + +// NewSSHCommandRunner returns a helper configured with the given host, for the given username and auth methods. +func NewSSHCommandRunner(hostname string, port int, user string, authMethods []ssh.AuthMethod) (SSHCommandRunner, error) { + config := &ssh.ClientConfig{ + User: user, + Auth: authMethods, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec // G106: E2E test clusters use host key validation disabled. + } + + newClient, err := newSSHDialerWithRetries(hostname, port, config, sshRetryInterval, sshRetryDuration) + if err != nil { + return nil, err + } + + return &sshCommandRunnerImpl{ + client: newClient, + }, nil +} + +// NewSSHCommandRunnerWithinGateway uses the gateway as a jump box to create and turn a helper configured with the given host, for the given username and auth methods,. +func NewSSHCommandRunnerWithinGateway(hostname string, port int, user string, authMethods []ssh.AuthMethod, gw Gateway) (SSHCommandRunner, error) { + gwConfig := &ssh.ClientConfig{ + User: gw.Username, + Auth: gw.AuthMethods, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec // G106: E2E test clusters use host key validation disabled. + } + + gatewayClient, err := newSSHDialerWithRetries(gw.Hostname, gw.Port, gwConfig, sshRetryInterval, sshRetryDuration) + if err != nil { + return nil, err + } + + targetHost := fmt.Sprintf("%s:%d", hostname, port) + + conn, err := gatewayClient.Dial("tcp", targetHost) + if err != nil { + return nil, err + } + + config := &ssh.ClientConfig{ + User: user, + Auth: authMethods, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec // G106: E2E test clusters use host key validation disabled. + } + + ncc, chans, reqs, err := ssh.NewClientConn(conn, targetHost, config) + if err != nil { + return nil, err + } + + newClient := ssh.NewClient(ncc, chans, reqs) + + return &sshCommandRunnerImpl{ + client: newClient, + }, nil +} + +func (s *sshCommandRunnerImpl) RunCommand(cmd string) ([]byte, error) { + e2eframework.Logf("Running command via SSH; remote: %s, command: %s", s.client.RemoteAddr(), cmd) + + if s.client == nil { + return nil, fmt.Errorf("s.client is nil") + } + + session, err := s.client.NewSession() + if err != nil { + return nil, err + } + // Ignore python warnings if the env var is not specified in the command. + // This avoid parsing errors when the command output contains warnings. + if !strings.Contains(cmd, "PYTHONWARNINGS") { + cmd = fmt.Sprintf("PYTHONWARNINGS=ignore %s", cmd) + } + + combinedOutput, err := session.CombinedOutput(cmd) + + e2eframework.Logf("Ran command via SSH; remote: %s, command: %s, output: %s, error: %v", s.client.RemoteAddr(), cmd, string(combinedOutput), err) + + return combinedOutput, err +} + +func (s *sshCommandRunnerImpl) RunCommandWindows(cmd string) ([]byte, error) { + e2eframework.Logf("Running command via SSH; remote: %s, command: %s", s.client.RemoteAddr(), cmd) + + if s.client == nil { + return nil, fmt.Errorf("s.client is nil") + } + + session, err := s.client.NewSession() + if err != nil { + return nil, err + } + + combinedOutput, err := session.CombinedOutput(cmd) + + e2eframework.Logf("Ran command via SSH; remote: %s, command: %s, output: %s, error: %v", s.client.RemoteAddr(), cmd, string(combinedOutput), err) + + return combinedOutput, err +} + +func (s *sshCommandRunnerImpl) Close() error { + err := s.client.Close() + if err != nil { + return err + } + + return nil +} + +func GetGateway() (*Gateway, error) { + httpProxy := os.Getenv("HTTP_PROXY" /*consts.HttpProxyEnv*/) + if httpProxy == "" { + return nil, errors.New("HTTP_PROXY environment variable should be set") + } + + parsedURL, err := url.Parse("http://" + httpProxy) + if err != nil { + return nil, err + } + + gatewayIP := parsedURL.Hostname() + gw := &Gateway{ + Hostname: gatewayIP, + Username: testbed.GatewayUsername, + Port: 22, // consts.DefaultSSHPort, + AuthMethods: []ssh.AuthMethod{ssh.Password(testbed.GatewayPassword)}, + } + + return gw, nil +} diff --git a/test/e2e/infrastructure/vsphere/supervisor/apiserver.go b/test/e2e/infrastructure/vsphere/supervisor/apiserver.go new file mode 100644 index 000000000..a08b845e6 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/supervisor/apiserver.go @@ -0,0 +1,166 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package supervisor + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "regexp" + + "golang.org/x/crypto/ssh" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/kubectl" + e2essh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" +) + +const ( + APIServerSSHPort = 22 + APIServerSSHUser = "root" +) + +var ( + managementAPIServerIP = os.Getenv("MANAGEMENT_API_SERVER_IP") +) + +// GetAPIServerCommandRunner returns SSH command runner configured with the SV cluster API server as root user. +func GetAPIServerCommandRunner(ctx context.Context, path string) (e2essh.SSHCommandRunner, error) { + apiServerURL := kubectl.GetKubectlClusterForCurrentContext(ctx, path) + + parsedAPIServerURL, err := url.Parse(apiServerURL) + if err != nil { + return nil, err + } + + apiServerIP := parsedAPIServerURL.Hostname() + if apiServerIP == "" { + return nil, errors.New("unable to get supervisor cluster API server hostname") + } + + apiServerCredentials, err := GetControlPlaneVMConnectionDetails(ctx, path, apiServerIP) + if err != nil { + return nil, err + } + + apiServerSSHCommandRunner, err := e2essh.NewSSHCommandRunner(apiServerCredentials.ManagementAPIServerIP, APIServerSSHPort, apiServerCredentials.Username, []ssh.AuthMethod{ssh.Password(apiServerCredentials.Password)}) + if err != nil { + return nil, err + } + + return apiServerSSHCommandRunner, nil +} + +// ControlPlaneVMConnectionDetails contains connection details to SSH into a control plane VM in the Supervisor Cluster. +// Within the control plane VM, SSO roles can be disregarded to test more granular functionality. +type ControlPlaneVMConnectionDetails struct { + + // ManagementAPIServerIP is the floating IP used internally in a Supervisor Cluster, used by wcpsvc to connect to + // the kube-apiserver. It can also be used to connect for elevated kubectl privileges, or general SSH access to the + // control plane VMs. + ManagementAPIServerIP string + + // Username is the SSH username to connect to the control plane VM. + Username string + + // Password is the SSH password (in conjunction with the Username) to the control plane VM. + Password string +} + +// GetControlPlaneVMConnectionDetails fetches the internal server IP and root credentials. +func GetControlPlaneVMConnectionDetails(ctx context.Context, path string, apiServerIP string) (ControlPlaneVMConnectionDetails, error) { + vCenterHostname := vcenter.GetVCPNIDFromKubeconfig(ctx, path) + e2eframework.Logf("VC: %s", vCenterHostname) + + // Correlate the kubeconfig cluster IP with the API server management endpoint, which is what wcpsvc uses internally. + dcliClient := wcp.NewClientUsingKubeconfig(ctx, path) + + clusters, err := dcliClient.ListClusters() + if err != nil { + return ControlPlaneVMConnectionDetails{}, fmt.Errorf("error listing clusters: %w", err) + } + + if managementAPIServerIP == "" { + for _, cluster := range clusters { + clusterInfo, err := dcliClient.GetCluster(cluster.MoID) + if err != nil { + return ControlPlaneVMConnectionDetails{}, fmt.Errorf("error fetching cluster: %w", err) + } + + // In most cases, the kubeconfig server IP would be the virtual IP (`clusterInfo.APIServerClusterEndpoint`). + // However, to run gce2e, some consumers may use a proxy instead, which allows them to connect directly over + // the floating IP (`clusterInfo.APIServerManagementEndpoint`). This case handles both checks to allow both + // methods of deployment. Both IPs are unique to a single cluster, so this is a safe check. + if apiServerIP == clusterInfo.APIServerManagementEndpoint || apiServerIP == clusterInfo.APIServerClusterEndpoint { + managementAPIServerIP = clusterInfo.APIServerManagementEndpoint + break + } + } + } + + if managementAPIServerIP == "" { + e2eframework.Logf("No cluster matching cluster endpoint %s at VC %s", apiServerIP, vCenterHostname) + return ControlPlaneVMConnectionDetails{}, fmt.Errorf("no cluster matching cluster endpoint %s", apiServerIP) + } + + vcSSHCommandRunner, err := e2essh.NewSSHCommandRunner(vCenterHostname, vcenter.VCSSHPort, testbed.RootUsername, []ssh.AuthMethod{ssh.Password(testbed.RootPassword)}) + if err != nil { + return ControlPlaneVMConnectionDetails{}, fmt.Errorf("error getting ssh runner: %w", err) + } + + cmd := "python3 /usr/lib/vmware-wcp/decryptK8Pwd.py" + + out, err := vcSSHCommandRunner.RunCommand(cmd) + if err != nil { + return ControlPlaneVMConnectionDetails{}, fmt.Errorf("error running command over ssh: %w", err) + } + + re := regexp.MustCompile(fmt.Sprintf("IP:\\s*%s\\nPWD:\\s*(.*)\\n", managementAPIServerIP)) + + matches := re.FindStringSubmatch(string(out)) + if matches == nil { + return ControlPlaneVMConnectionDetails{}, fmt.Errorf("unable to find the api server info from output: %v", string(out)) + } + + apiServerPassword := matches[1] + + return ControlPlaneVMConnectionDetails{ + ManagementAPIServerIP: managementAPIServerIP, + Username: APIServerSSHUser, + Password: apiServerPassword, + }, nil +} + +func GetSvAPIServerAsGateway(ctx context.Context, kubeconfigPath string) (*e2essh.Gateway, error) { + apiServerURL := kubectl.GetKubectlClusterForCurrentContext(ctx, kubeconfigPath) + + parsedAPIServerURL, err := url.Parse(apiServerURL) + if err != nil { + return nil, err + } + + apiServerIP := parsedAPIServerURL.Hostname() + if apiServerIP == "" { + return nil, errors.New("unable to get supervisor cluster API server hostname") + } + + apiServerCredentials, err := GetControlPlaneVMConnectionDetails(ctx, kubeconfigPath, apiServerIP) + if err != nil { + return nil, err + } + + fmt.Print("The API server pwd is " + apiServerCredentials.Password) + + return &e2essh.Gateway{ + Hostname: apiServerCredentials.ManagementAPIServerIP, + Username: apiServerCredentials.Username, + Port: 22, + AuthMethods: []ssh.AuthMethod{ssh.Password(apiServerCredentials.Password)}, + }, nil +} diff --git a/test/e2e/infrastructure/vsphere/testbed/creds.go b/test/e2e/infrastructure/vsphere/testbed/creds.go new file mode 100644 index 000000000..afbe4665e --- /dev/null +++ b/test/e2e/infrastructure/vsphere/testbed/creds.go @@ -0,0 +1,31 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package testbed + +import ( + "os" +) + +// Hopefully it's okay to put standard constants in the test framework. +// In the future, we may want the ability to override these constants for different environments. +// Of course, that begs the question of whether 'testbed' is even the right package for these constants. +var ( + GatewayUsername = getFromEnvWithDefault("GATEWAY_VM_USERNAME", "root") + GatewayPassword = getFromEnvWithDefault("GATEWAY_VM_PASSWORD", "vmware") + RootUsername = getFromEnvWithDefault("SSH_USERNAME", "root") + RootPassword = getFromEnvWithDefault("SSH_PASSWORD", "vmware") + AdminUsername = getFromEnvWithDefault("VCSA_USERNAME", "Administrator@vsphere.local") + AdminPassword = getFromEnvWithDefault("VCSA_PASSWORD", "Admin!23") + NSXAdminUsername = getFromEnvWithDefault("NSX_MANAGER_USERNAME", "admin") + NSXAdminPassword = getFromEnvWithDefault("NSX_MANAGER_PASSWORD", "Admin!23") +) + +func getFromEnvWithDefault(key, defaultVal string) string { + val, ok := os.LookupEnv(key) + if !ok { + return defaultVal + } + + return val +} diff --git a/test/e2e/infrastructure/vsphere/vcenter/client.go b/test/e2e/infrastructure/vsphere/vcenter/client.go new file mode 100644 index 000000000..539760c0c --- /dev/null +++ b/test/e2e/infrastructure/vsphere/vcenter/client.go @@ -0,0 +1,187 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/antchfx/htmlquery" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/session/cache" + vapirest "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/methods" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" +) + +const ( + h5LoginURLFormat = "https://%s/ui/login" + ssoLoginURLFormat = "https://%s/ui/saml/websso/sso" + vcsimURLFormat = "https://user:pass@%s:8989" + + castleAuthorizationKey = "CastleAuthorization" + castleAuthorizationFormat = "Basic %s" + samlResponseKey = "SAMLResponse" + relayStateKey = "RelayState" +) + +// NewVimClientFromKubeConfig uses a kubeconfig pointed at a supervisor cluster context with an +// +// administrator user to determine the vCenter instance and setup an authenticated vim client. +func NewVimClientFromKubeconfig(ctx context.Context, path string) *vim25.Client { + return NewVimClientFromKubeconfigFile(ctx, path) +} + +func NewVimClientFromKubeconfigFile(ctx context.Context, path string) *vim25.Client { + vCenterHostname := GetVCPNIDFromKubeconfigFile(ctx, path) + Expect(vCenterHostname).NotTo(Equal(""), "Unable to determine VC PNID") + + // TODO (GCM-3023) Figure out how to get these from E2E test config. + // Hardcoding is a short term fix until then. + vCenterAdminUser := testbed.AdminUsername + vCenterAdminPassword := testbed.AdminPassword + + vimClient, err := NewVimClient(vCenterHostname, vCenterAdminUser, vCenterAdminPassword) + Expect(err).NotTo(HaveOccurred()) + + return vimClient +} + +// Constructs a new vim client authenticated with the given user credentials. +func NewVimClient(vCenterHost string, username string, password string) (*vim25.Client, error) { + clientURL := url.URL{ + Scheme: "https", + Host: vCenterHost, + Path: "sdk", + } + ctx := context.Background() + sc := soap.NewClient(&clientURL, true) + + client, err := vim25.NewClient(ctx, sc) + if err != nil { + return nil, err + } + + loginRequest := types.Login{ + This: *client.ServiceContent.SessionManager, + UserName: username, + Password: password, + } + + _, err = methods.Login(ctx, sc, &loginRequest) + if err != nil { + return nil, err + } + + return client, nil +} + +// LogoutVimClient sends a logout request using the given client. +func LogoutVimClient(client *vim25.Client) { + if client == nil || client.RoundTripper == nil { + return + } + + logoutRequest := types.Logout{ + This: *client.ServiceContent.SessionManager, + } + _, err := methods.Logout(context.Background(), client.RoundTripper, &logoutRequest) + Expect(err).NotTo(HaveOccurred()) +} + +// GetH5SessionCookie gets the session cookie by logging into the H5 client with given user credentials. +// For further H5 requests, attach this session cookie in order to authenticate. +func GetH5SessionCookie(client *http.Client, hostname, username, password string) *http.Cookie { + // 1. Starts the login process in the H5 Client. Returns a correctly formed redirection link to the WebSSO + // that contains the required URL parameters such as SAMLRequest, signature and signature algorithm. + redirectToSso := getWebSsoLocation(client, hostname) + + // 2. Login in the Web SSO with the supplied username and password. Returns a SAMLResponse that packs the + // SAML token issued from the SSO and also the RelayState. + // Note: The step of retrieving the HTML that is needed for rendering the Login Page is skipped. + samlResponse, relayState := loginToWebSso(client, redirectToSso, username, password) + + // 3. Finish the login process by passing the returned SAMLResponse and RelayState back to the H5 Client. + cookie := loginByTokenToH5(client, hostname, relayState, samlResponse) + + return cookie +} + +func getWebSsoLocation(client *http.Client, hostname string) string { + resp, err := client.Get(fmt.Sprintf(h5LoginURLFormat, hostname)) + Expect(err).NotTo(HaveOccurred()) + + defer func() { _ = resp.Body.Close() }() + + return resp.Request.URL.String() +} + +func loginToWebSso(client *http.Client, location, username, password string) (string, string) { + auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + resp, err := client.PostForm(location, url.Values{castleAuthorizationKey: {fmt.Sprintf(castleAuthorizationFormat, auth)}}) + Expect(err).NotTo(HaveOccurred()) + + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + + doc, err := htmlquery.Parse(strings.NewReader(string(body))) + Expect(err).NotTo(HaveOccurred()) + + samlResponseNode := htmlquery.FindOne(doc, "/html/body/form/input[@name=\"SAMLResponse\"]") + relayStateNode := htmlquery.FindOne(doc, "/html/body/form/input[@name=\"RelayState\"]") + + return htmlquery.SelectAttr(samlResponseNode, "value"), htmlquery.SelectAttr(relayStateNode, "value") +} + +func loginByTokenToH5(client *http.Client, hostname, relayState, samlResponse string) *http.Cookie { + resp, err := client.PostForm(fmt.Sprintf(ssoLoginURLFormat, hostname), url.Values{samlResponseKey: {samlResponse}, relayStateKey: {relayState}}) + Expect(err).NotTo(HaveOccurred()) + + defer func() { _ = resp.Body.Close() }() + + cookie := strings.Split(resp.Request.Response.Header.Get("Set-Cookie"), ";") + + return &http.Cookie{ + Name: strings.Split(cookie[0], "=")[0], + Value: strings.Split(cookie[0], "=")[1], + Path: strings.Split(cookie[1], "=")[1], + } +} + +// NewVCSimClient creates a vcsim vim25.Client for using in the kind environment. +func NewVcSimClient(ctx context.Context, vcip string) *vim25.Client { + u, err := soap.ParseURL(fmt.Sprintf(vcsimURLFormat, vcip)) + Expect(err).NotTo(HaveOccurred(), "Should be able to parse the url.") + + s := &cache.Session{ + URL: u, + Insecure: true, + } + + c := new(vim25.Client) + err = s.Login(ctx, c, nil) + Expect(err).NotTo(HaveOccurred(), "Should be able to login to the vcsim") + + return c +} + +// Constructs a new rest client authenticated with the given user credentials. +func NewRestClient(ctx context.Context, c *vim25.Client, userName string, password string) (*vapirest.Client, error) { + restClient := vapirest.NewClient(c) + userInfo := url.UserPassword(userName, password) + err := restClient.Login(ctx, userInfo) + + return restClient, err +} diff --git a/test/e2e/infrastructure/vsphere/vcenter/info.go b/test/e2e/infrastructure/vsphere/vcenter/info.go new file mode 100644 index 000000000..f1e05aa63 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/vcenter/info.go @@ -0,0 +1,213 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "context" + "fmt" + "os" + "time" + + . "github.com/onsi/gomega" + "gopkg.in/yaml.v2" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/dcli" + e2essh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" +) + +const ( + clusterMoIDKey = "vc_cluster" + fullClusterID = "cluster_id_full" + supervisorID = "supervisor_id" + vCenterPNIDKey = "vc_pnid" + vCenterIPKey = "VCSA_IP" + clusterConfigMapName = "wcp-cluster-config" + kubeSystemNamespace = "kube-system" + clusterConfigJSONPathFilter = "jsonpath={.data.wcp-cluster-config\\.yaml}" + VCSSHPort = 22 +) + +// GetClusterMoIDFromKubeconfig uses the framework's current kubeconfig context to determine the MoID of the WCP enabled cluster. +// This relies on the present of the wcp-cluster-config ConfigMap in kube-system +// Only users in the Administrators@ group can access this. +func GetClusterMoIDFromKubeconfig(ctx context.Context, path string) string { + yamlData := getWCPClusterConfig(ctx, path) + return yamlData[clusterMoIDKey] +} + +// GetSupervisorIDFromKubeconfig uses the framework's current kubeconfig context to determine the supervisor ID. +// This relies on the presence of the wcp-cluster-config ConfigMap in kube-system +// Only users in the Administrators@ group can access this. +func GetSupervisorIDFromKubeconfig(ctx context.Context, path string) string { + yamlData := getWCPClusterConfig(ctx, path) + return yamlData[supervisorID] +} + +// GetFullClusterIDFromKubeConfig uses the framework's current kubeconfig context to determine the full ID of the WCP enabled cluster. +// This relies on the presence of the wcp-cluster-config ConfigMap in kube-system +// Only users in the Administrators@ group can access this. +func GetFullClusterIDFromKubeConfig(ctx context.Context, path string) string { + yamlData := getWCPClusterConfig(ctx, path) + return yamlData[fullClusterID] +} + +// GetVCPNIDFromKubeconfig uses the framework's current kubeconfig context to determine the PNID of the VC that the WCP enabled cluster was deployed by. +// This primarily relies of the VCSA_IP env variable that is setup during the gce2e runs +// As a fallback strategy it relies on the vc_pnid key present in the wcp-cluster-config ConfigMap in kube-system +// Only users in the Administrators@ group can access this. +func GetVCPNIDFromKubeconfig(ctx context.Context, path string) string { + val, ok := os.LookupEnv(vCenterIPKey) + if !ok { + // Useful when the current context does not point to a kubeconfig, eg. during GC app tests. + yamlData := getWCPClusterConfig(ctx, path) + return yamlData[vCenterPNIDKey] + } + + return val +} + +// GetVCPNIDFromKubeconfigFile (See [GetVCPNIDFromKubeconfig]). +func GetVCPNIDFromKubeconfigFile(ctx context.Context, path string) string { + return GetVCPNIDFromKubeconfig(ctx, path) +} + +// GetClusterMoIDFromKubeconfigFile returns the clusterMoID from a kubeconfig file currently pointing to a supervisor cluster. +func GetClusterMoIDFromKubeconfigFile(ctx context.Context, path string) string { + // Useful when the current context does not point to a kubeconfig, eg. during GC app tests. + yamlData := getWCPClusterConfig(ctx, path) + return yamlData[clusterMoIDKey] +} + +// CreateUserAndAssignToGrp creates user with given username and password, and assigns it to the given group. +func CreateUserAndAssignToGrp(ctx context.Context, vimClient *vim25.Client, sshCommandRunner e2essh.SSHCommandRunner, userName, password, group string) (*User, error) { + // Create the non-admin VC user via admin + user := NewUser(userName, password). + WithAdminCreds(dcli.VCenterUserCredentials{ + Username: testbed.AdminUsername, + Password: testbed.AdminPassword, + }). + WithSSHCommandRunner(sshCommandRunner) + + CreateUserOrFail(user) + + // Add user to SupervisorProviderAdministrators via admin vimClient + err := AddToGroup(ctx, vimClient, user.Credentials.Username, group) + if err != nil { + return nil, err + } + + return user, err +} + +// getWCPClusterConfig is helper method to get the WCP cluster config. +func getWCPClusterConfig(ctx context.Context, path string) map[string]string { + applyCmd := framework.NewCommand( + framework.WithCommand("cat"), + framework.WithArgs(path), + ) + + stdout, stderr, err := applyCmd.Run(ctx) + if err != nil { + fmt.Printf("stderr getting kubeconfig file: %s", string(stderr)) + fmt.Printf("err getting kubeconfig file: %v", err) + } + // TODO: kubectl GET returns "Unable to connect to the server: Service Unavailable" sometimes. Add retry to further debugging. + var configYAML []byte + + Eventually(func() error { + configYAML, err = framework.KubectlGet(ctx, path, "configmap", clusterConfigMapName, "-n", kubeSystemNamespace, "-o", clusterConfigJSONPathFilter) + return err + }, 5*time.Minute, 10*time.Second).ShouldNot(HaveOccurred(), "failed to get configmap %q/%q with jsonpath=%q. Kubeconfig file: %s", kubeSystemNamespace, clusterConfigMapName, clusterConfigJSONPathFilter, string(stdout)) + + yamlData := map[string]string{} + err = yaml.Unmarshal(configYAML, yamlData) + Expect(err).NotTo(HaveOccurred(), "should be able to parse the WCP cluster config as YAML: %s", string(configYAML)) + + return yamlData +} + +// NewVcsimFinder returns a vcsim finder. +func NewVcsimFinder(ctx context.Context, vimClient *vim25.Client) *find.Finder { + finder := find.NewFinder(vimClient, false) + dc, err := finder.DefaultDatacenter(ctx) + Expect(err).NotTo(HaveOccurred(), "Should be able to get the default DC.") + finder.SetDatacenter(dc) + + return finder +} + +// GetClusterRefs returns available clusters in the vcsim. +func GetClusterRefs(ctx context.Context, finder *find.Finder) []*object.ClusterComputeResource { + clusters, err := finder.ClusterComputeResourceList(ctx, "*") + Expect(err).NotTo(HaveOccurred(), "Should be able to find clusters.") + Expect(clusters).ShouldNot(BeEmpty(), "Should be able to find at least one cluster.") + + return clusters +} + +func GetClusterResourcePool(ctx context.Context, finder *find.Finder) types.ManagedObjectReference { + clusters := GetClusterRefs(ctx, finder) + Expect(len(clusters)).To(BeNumerically(">", 0)) + rp, err := clusters[0].ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + + rpRef := rp.Reference() + + return rpRef +} + +func GetClusterIDToRefMapping(clusterRefs []*object.ClusterComputeResource) map[string]*object.ClusterComputeResource { + refMapping := make(map[string]*object.ClusterComputeResource) + for _, ref := range clusterRefs { + refMapping[ref.Reference().Value] = ref + } + + return refMapping +} + +// SimulateFindClusterByMoid returns a cluster reference in the vcsim by the given cluster moid. +func SimulateFindClusterByMoid(ctx context.Context, vcip, moid string) *object.ClusterComputeResource { + vimClient := NewVcSimClient(ctx, vcip) + finder := NewVcsimFinder(ctx, vimClient) + + clusterRefs := GetClusterRefs(ctx, finder) + refMapping := GetClusterIDToRefMapping(clusterRefs) + Expect(refMapping).Should(HaveKey(moid)) + + return refMapping[moid] +} + +// GetResourcePoolAndFolder returns associated resource pool and folder for a cluster. +// Note: the folder here is the root folder of datacenter DC0 in the vcsim. +func GetResourcePoolAndFolder(ctx context.Context, finder *find.Finder, + cluster *object.ClusterComputeResource) (rp, folder string) { + rpRef, err := cluster.ResourcePool(ctx) + Expect(err).NotTo(HaveOccurred(), "Should return the cluster's resource pool.") + + rp = rpRef.Reference().Value + + folderRef, err := finder.DefaultFolder(ctx) + Expect(err).NotTo(HaveOccurred(), "Should return default folder.") + + folder = folderRef.Reference().Value + + return rp, folder +} + +// DeleteFolder deletes a folder and all its contents (VMs/templates/folders). +func DeleteFolder(ctx context.Context, folder *object.Folder) { + task, err := folder.Destroy(ctx) + Expect(err).ToNot(HaveOccurred()) + + // Wait for task completion + err = task.Wait(ctx) + Expect(err).ToNot(HaveOccurred()) +} diff --git a/test/e2e/infrastructure/vsphere/vcenter/invsvc/client.go b/test/e2e/infrastructure/vsphere/vcenter/invsvc/client.go new file mode 100644 index 000000000..0ead6640c --- /dev/null +++ b/test/e2e/infrastructure/vsphere/vcenter/invsvc/client.go @@ -0,0 +1,223 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package invsvc + +import ( + "context" + "net/url" + "strings" + + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/soap" + vimtypes "github.com/vmware/govmomi/vim25/types" +) + +// The Global Permissions API is still internal-only. Some MOB based workarounds exist, see: +// https://williamlam.com/2017/03/automating-vsphere-global-permissions-with-powercli.html +// https://github.com/vmware/govmomi/pull/2485 +// This client uses the inventory service API directly, rather than via the MOB. + +const ( + Namespace = "inventoryservice" + Path = "/invsvc/vmomi/sdk" + Version = "4.0" +) + +var ( + AuthorizationService = vimtypes.ManagedObjectReference{ + Type: "AuthorizationService", + Value: "authorizationService", + } + + SessionManager = vimtypes.ManagedObjectReference{ + Type: "InventoryServiceSessionManager", + Value: "sessionManager", + } +) + +type Principal struct { + Name string `xml:"name"` + Group bool `xml:"group"` +} + +type AccessControl struct { + Principal Principal `xml:"principal"` + Roles []int64 `xml:"roles,omitempty"` + Propagate bool `xml:"propagate"` + Version int64 `xml:"version"` +} + +type Client struct { + *soap.Client +} + +func NewClient(ctx context.Context, c *vim25.Client) *Client { + // TODO: should use soap.NewServiceClient, but vcSessionCookie SOAP header causes InventoryServiceLogin() to fail + u := c.URL() + u.Path = Path + sc := soap.NewClient(u, c.DefaultTransport().TLSClientConfig.InsecureSkipVerify) + sc.Namespace = "urn:" + Namespace + sc.Version = Version + + return &Client{sc} +} + +func (c *Client) Login(ctx context.Context, u *url.Userinfo) error { + var reqBody, resBody InventoryServiceLoginBody + + p, _ := u.Password() + + reqBody.Req = &InventoryServiceLoginRequest{ + This: SessionManager, + Username: u.Username(), + Password: p, + } + + return c.RoundTrip(ctx, &reqBody, &resBody) +} + +func (c *Client) Logout(ctx context.Context) error { + var reqBody, resBody InventoryServiceLogoutBody + + reqBody.Req = &InventoryServiceLogoutRequest{ + This: SessionManager, + } + + return c.RoundTrip(ctx, &reqBody, &resBody) +} + +func (c *Client) GetGlobalAccessControlList(ctx context.Context) ([]AccessControl, error) { + var reqBody, resBody GetGlobalAccessControlListBody + + reqBody.Req = &GetGlobalAccessControlListRequest{ + This: AuthorizationService, + } + + err := c.RoundTrip(ctx, &reqBody, &resBody) + if err != nil { + return nil, err + } + + return resBody.Res.Returnval, nil +} + +func (p *Principal) format() Principal { + name := strings.SplitN(p.Name, "@", 2) + if len(name) == 2 { + // Convert from UPN to domain/username format + p.Name = name[1] + `\` + name[0] + } + + return *p +} + +func (c *Client) AddGlobalAccessControlList(ctx context.Context, permissions ...AccessControl) error { + for i, p := range permissions { + permissions[i].Principal = p.Principal.format() + } + + var reqBody, resBody AddGlobalAccessControlListBody + + reqBody.Req = &AddGlobalAccessControlListRequest{ + This: AuthorizationService, + Permissions: permissions, + } + + return c.RoundTrip(ctx, &reqBody, &resBody) +} + +func (c *Client) RemoveGlobalAccess(ctx context.Context, principals ...Principal) error { + for i, p := range principals { + principals[i] = p.format() + } + + var reqBody, resBody RemoveGlobalAccessBody + + reqBody.Req = &RemoveGlobalAccessRequest{ + This: AuthorizationService, + Principals: principals, + } + + return c.RoundTrip(ctx, &reqBody, &resBody) +} + +type InventoryServiceLoginRequest struct { + This vimtypes.ManagedObjectReference `xml:"_this"` + Username string `xml:"username"` + Password string `xml:"password"` +} + +type InventoryServiceLoginResponse struct { +} + +type InventoryServiceLoginBody struct { + Req *InventoryServiceLoginRequest `xml:"urn:inventoryservice InventoryServiceLogin,omitempty"` + Res *InventoryServiceLoginResponse `xml:"urn:inventoryservice InventoryServiceLoginResponse,omitempty"` + Fault_ *soap.Fault `xml:"http://schemas.xmlsoap.org/soap/envelope/ Fault,omitempty"` +} + +func (b *InventoryServiceLoginBody) Fault() *soap.Fault { return b.Fault_ } + +type InventoryServiceLogoutRequest struct { + This vimtypes.ManagedObjectReference `xml:"_this"` +} + +type InventoryServiceLogoutResponse struct { +} + +type InventoryServiceLogoutBody struct { + Req *InventoryServiceLogoutRequest `xml:"urn:inventoryservice InventoryServiceLogout,omitempty"` + Res *InventoryServiceLogoutResponse `xml:"urn:inventoryservice InventoryServiceLogoutResponse,omitempty"` + Fault_ *soap.Fault `xml:"http://schemas.xmlsoap.org/soap/envelope/ Fault,omitempty"` +} + +func (b *InventoryServiceLogoutBody) Fault() *soap.Fault { return b.Fault_ } + +type GetGlobalAccessControlListRequest struct { + This vimtypes.ManagedObjectReference `xml:"_this"` +} + +type GetGlobalAccessControlListResponse struct { + Returnval []AccessControl `xml:"returnval"` +} + +type GetGlobalAccessControlListBody struct { + Req *GetGlobalAccessControlListRequest `xml:"urn:inventoryservice AuthorizationService.GetGlobalAccessControlList,omitempty"` + Res *GetGlobalAccessControlListResponse `xml:"urn:inventoryservice AuthorizationService.GetGlobalAccessControlListResponse,omitempty"` + Fault_ *soap.Fault `xml:"http://schemas.xmlsoap.org/soap/envelope/ Fault,omitempty"` +} + +func (b *GetGlobalAccessControlListBody) Fault() *soap.Fault { return b.Fault_ } + +type AddGlobalAccessControlListRequest struct { + This vimtypes.ManagedObjectReference `xml:"_this"` + Permissions []AccessControl `xml:"permissions"` +} + +type AddGlobalAccessControlListResponse struct { +} + +type AddGlobalAccessControlListBody struct { + Req *AddGlobalAccessControlListRequest `xml:"urn:inventoryservice AuthorizationService.AddGlobalAccessControlList,omitempty"` + Res *AddGlobalAccessControlListResponse `xml:"urn:inventoryservice AuthorizationService.AddGlobalAccessControlListResponse,omitempty"` + Fault_ *soap.Fault `xml:"http://schemas.xmlsoap.org/soap/envelope/ Fault,omitempty"` +} + +func (b *AddGlobalAccessControlListBody) Fault() *soap.Fault { return b.Fault_ } + +type RemoveGlobalAccessRequest struct { + This vimtypes.ManagedObjectReference `xml:"_this"` + Principals []Principal `xml:"principals"` +} + +type RemoveGlobalAccessResponse struct { +} + +type RemoveGlobalAccessBody struct { + Req *RemoveGlobalAccessRequest `xml:"urn:inventoryservice AuthorizationService.RemoveGlobalAccess,omitempty"` + Res *RemoveGlobalAccessResponse `xml:"urn:inventoryservice AuthorizationService.RemoveGlobalAccessResponse,omitempty"` + Fault_ *soap.Fault `xml:"http://schemas.xmlsoap.org/soap/envelope/ Fault,omitempty"` +} + +func (b *RemoveGlobalAccessBody) Fault() *soap.Fault { return b.Fault_ } diff --git a/test/e2e/infrastructure/vsphere/vcenter/roles.go b/test/e2e/infrastructure/vsphere/vcenter/roles.go new file mode 100644 index 000000000..a25258ac1 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/vcenter/roles.go @@ -0,0 +1,181 @@ +// Copyright (c) 2020-2024 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "context" + "fmt" + "log" + "net/url" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/ssoadmin" + "github.com/vmware/govmomi/sts" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter/invsvc" +) + +// roles contains helpers to manage roles with privilege on a vCenter instance. +// This can be useful when testing against authorization. + +// CreateOrUpdateRole adds a new role or updates an existing role in VC with the specified privileges. Returns the role id. +func CreateOrUpdateRole(ctx context.Context, vimClient *vim25.Client, roleName string, privilegeIDs []string) (int32, error) { + role, err := GetRoleByName(ctx, vimClient, roleName) + if err != nil { + return 0, err + } + + if role == nil { + return CreateRole(ctx, vimClient, roleName, privilegeIDs) + } else { + err = UpdateRole(ctx, vimClient, role.RoleId, roleName, privilegeIDs) + return role.RoleId, err + } +} + +// GetRoleByName returns the AuthorizationRole with the given name or nil if the role is not found. +func GetRoleByName(ctx context.Context, vimClient *vim25.Client, roleName string) (*types.AuthorizationRole, error) { + authzManager := object.NewAuthorizationManager(vimClient) + + roleList, err := authzManager.RoleList(ctx) + if err != nil { + return nil, err + } + + role := roleList.ByName(roleName) + + return role, nil +} + +// CreateRole adds a new role with the specified privileges in VC, and returns the id of the role. +func CreateRole(ctx context.Context, vimClient *vim25.Client, roleName string, privilegeIDs []string) (int32, error) { + authzManager := object.NewAuthorizationManager(vimClient) + + roleID, err := authzManager.AddRole(ctx, roleName, privilegeIDs) + if err != nil { + return 0, err + } + + return roleID, nil +} + +// UpdateRole updated the specified role with specified privileges in VC. +func UpdateRole(ctx context.Context, vimClient *vim25.Client, roleID int32, roleName string, privilegeIDs []string) error { + authzManager := object.NewAuthorizationManager(vimClient) + + err := authzManager.UpdateRole(ctx, roleID, roleName, privilegeIDs) + if err != nil { + log.Printf("Failed to update the role, newer privileges might not be present: %v", err) + return err + } + + return nil +} + +// RemoveRole removes the role with specified id in VC. +func RemoveRole(ctx context.Context, vimClient *vim25.Client, roleID int32) error { + authzManager := object.NewAuthorizationManager(vimClient) + + err := authzManager.RemoveRole(ctx, roleID, false) + if err != nil { + return err + } + + return nil +} + +func AddToGroup(ctx context.Context, vimClient *vim25.Client, userName, groupName string) error { + return withSSO(ctx, vimClient, func(c *ssoadmin.Client) error { + user, err := c.FindUser(ctx, userName) + if err != nil { + return err + } + + if user == nil { + return fmt.Errorf("user %q not found", userName) + } + + if err = c.AddUsersToGroup(ctx, groupName, user.Id); err != nil { + return err + } + + return nil + }) +} + +func withSSO(ctx context.Context, vc *vim25.Client, f func(*ssoadmin.Client) error) error { + c, err := ssoadmin.NewClient(ctx, vc) + if err != nil { + return err + } + + token, err := sts.NewClient(ctx, vc) + if err != nil { + return err + } + + req := sts.TokenRequest{ + Userinfo: url.UserPassword(testbed.AdminUsername, testbed.AdminPassword), + } + + header := soap.Header{} + + header.Security, err = token.Issue(ctx, req) + if err != nil { + return err + } + + if err = c.Login(c.WithHeader(ctx, header)); err != nil { + return err + } + + defer func() { + err := c.Logout(ctx) + if err != nil { + log.Printf("user logout error: %v", err) + } + }() + + return f(c) +} + +func withInvSvc(ctx context.Context, vc *vim25.Client, f func(*invsvc.Client) error) error { + c := invsvc.NewClient(ctx, vc) + + user := url.UserPassword(testbed.AdminUsername, testbed.AdminPassword) + + err := c.Login(ctx, user) + if err != nil { + return err + } + + defer func() { + err := c.Logout(ctx) + if err != nil { + log.Printf("user logout error: %v", err) + } + }() + + return f(c) +} + +func SetGlobalPermission(ctx context.Context, vimClient *vim25.Client, roleID int32, user string) error { + return withInvSvc(ctx, vimClient, func(c *invsvc.Client) error { + return c.AddGlobalAccessControlList(ctx, invsvc.AccessControl{ + Principal: invsvc.Principal{Name: user}, + Roles: []int64{int64(roleID)}, + Propagate: true, + }) + }) +} + +func RemoveGlobalPermission(ctx context.Context, vimClient *vim25.Client, user string) error { + return withInvSvc(ctx, vimClient, func(c *invsvc.Client) error { + return c.RemoveGlobalAccess(ctx, invsvc.Principal{Name: user}) + }) +} diff --git a/test/e2e/infrastructure/vsphere/vcenter/sso.go b/test/e2e/infrastructure/vsphere/vcenter/sso.go new file mode 100644 index 000000000..4709c8cec --- /dev/null +++ b/test/e2e/infrastructure/vsphere/vcenter/sso.go @@ -0,0 +1,165 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "fmt" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/dcli" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" +) + +// sso contains helpers to manage users on a vCenter instance. +// This can be useful when testing RBAC, for instance. + +const ( + noSuchUserErrorString = "ERROR_NO_SUCH_USER" +) + +// Use dir-cli to create commands +// /usr/lib/vmware-vmafd/bin/dir-cli +// Example for user creation +// user create --account +//--user-password +//--first-name +//--last-name +// [ --login ] +// [ --password ] + +// User is a basic type used to create vSphere SSO users. +type User struct { + // Credentials for the user. + Credentials dcli.VCenterUserCredentials + // Credentials to be used to create the user. + adminCreds dcli.VCenterUserCredentials + cmdRunner ssh.SSHCommandRunner +} + +// Helper method to return the path to the vmware-vmafd binary. +func getDirCLIBinaryPath() string { + return filepath.Join("/usr", "lib", "vmware-vmafd", "bin", "dir-cli") +} + +func addAdminCredentialsToCommand(cmd string, creds dcli.VCenterUserCredentials) string { + return fmt.Sprintf("%s --login '%s' --password '%s'", cmd, creds.Username, creds.Password) +} + +// NewUser creates a basic user that should have the given username and password. +func NewUser(username, password string) *User { + return &User{Credentials: dcli.VCenterUserCredentials{Username: username, Password: password}} +} + +// WithAdminCreds configures the credentials to be used to create this user. +func (u *User) WithAdminCreds(creds dcli.VCenterUserCredentials) *User { + u.adminCreds = creds + return u +} + +// WithSSHCommandRunner configures the SSH helper to be used to run the commands to create the user. +func (u *User) WithSSHCommandRunner(cmdRunner ssh.SSHCommandRunner) *User { + u.cmdRunner = cmdRunner + return u +} + +func (u *User) checkIfUserExists() (bool, error) { + binaryPath := getDirCLIBinaryPath() + + // Check if the user already exists. + cmd := fmt.Sprintf("%s user find-by-name --account '%s' ", binaryPath, u.Credentials.Username) + cmd = addAdminCredentialsToCommand(cmd, u.adminCreds) + fmt.Printf("Running command %s\n", cmd) + result, err := u.cmdRunner.RunCommand(cmd) + // This command can fail either due to intermittent issues, or because + // the user does not exist. Either way, return false, and the + // underlying error. + if err != nil { + // Check if the user didn't exist. dir-cli logs a specific + // error message when this happens. Look for that string to + // differentiate between 'something went wrong running the + // command' and 'the user does not exist' + if strings.Contains(string(result), noSuchUserErrorString) { + return false, nil + } + // Lookup may have failed intermittently. Return an error. + fmt.Printf("Command output: %s\n", string(result)) + + return false, err + } + // At this point, the user must exist. Return true. + return true, nil +} + +// Create runs dir-cli on the VC to create the given user. +// +// It is idempotent in that if the given user already exists, it will not +// attempt to create them again (and simply succeed). This allows it to be +// retried in an Eventually() block safely. +func (u *User) Create() error { + // TODO You might have a bad time if you run this on Windows. + // Then again, if you're developing on Windows, maybe you're already having a bad time. + if exists, err := u.checkIfUserExists(); err == nil && exists { + // NOOP if the user definitely exists. Otherwise, retry. Worst + // case, we intermittently failed to check and it'll error out + // anyway. + return nil + } + + binaryPath := getDirCLIBinaryPath() + + cmd := fmt.Sprintf("%s user create --account '%s' --user-password '%s' --first-name '%s First name' --last-name '%s Last name'", binaryPath, u.Credentials.Username, u.Credentials.Password, u.Credentials.Username, u.Credentials.Username) + cmd = addAdminCredentialsToCommand(cmd, u.adminCreds) + fmt.Printf("Running command %s\n", cmd) + result, err := u.cmdRunner.RunCommand(cmd) + fmt.Printf("Command output: %s\n", string(result)) + + return err +} + +// user delete --account +// [ --login ] +// [ --password ] + +// Delete runs dir-cli on the VC to delete the given user. +// This method is idempotent, in that if the user is not found in dir-cli, it +// simply exits as a NOOP without any error. +func (u *User) Delete() error { + if exists, err := u.checkIfUserExists(); err == nil && !exists { + // NOOP if the user definitely does not exist. Otherwise, retry. Worst + // case, we intermittently failed to check and it'll error out + // anyway. + return nil + } + + binaryPath := getDirCLIBinaryPath() + cmd := fmt.Sprintf("%s user delete --account '%s'", binaryPath, u.Credentials.Username) + cmd = addAdminCredentialsToCommand(cmd, u.adminCreds) + _, err := u.cmdRunner.RunCommand(cmd) + + return err +} + +// CreateUserOrFail is a convenience method that retries user creation until it +// succeeds or times out. +func CreateUserOrFail(user *User) { + Eventually( + // user.Create() returns an error if it failed to create the + // user, and is meant to be idempotent. + user.Create, 1*time.Minute, 5*time.Second).Should(Succeed(), "User should have eventually been created", user.Credentials.Username) +} + +// DeleteUserOrFail is a convenience method that retries user deletion until it +// succeeds or times out. +func DeleteUserOrFail(user *User) { + Eventually( + // user.Delete() returns an error if it failed to delete the + // user, and is meant to be idempotent. + user.Delete, 1*time.Minute, 5*time.Second).Should(Succeed(), "User should have eventually been deleted", user.Credentials.Username) +} + +// TODO Similar abstractions for groups. diff --git a/test/e2e/infrastructure/vsphere/vcenter/storage.go b/test/e2e/infrastructure/vsphere/vcenter/storage.go new file mode 100644 index 000000000..a309d5f26 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/vcenter/storage.go @@ -0,0 +1,313 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "context" + "errors" + "strings" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/pbm" + "github.com/vmware/govmomi/pbm/types" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + vimtypes "github.com/vmware/govmomi/vim25/types" +) + +const ( + encryptionCapabilityID = "ad5a249d-cbc2-43af-9366-694d7664fa52" + encryptionCapabilityNamespace = "com.vmware.storageprofile.dataservice" + vSANDirectTypeID = "vSANDirectType" + vSANDirect = "vSANDirect" + volumeAllocationNamespace = "com.vmware.storage.volumeallocation" + volumeAllocationTypeID = "VolumeAllocationType" + fullyInitializedValue = "Fully initialized" +) + +// GetStoragePolicyIDFromName looks up a storage profile by name and returns its ID. +// Useful when configuring WCP namespaces with storage profiles. +func GetStoragePolicyIDFromName(client *vim25.Client, profileName string) (string, error) { + pbmClient, err := pbm.NewClient(context.Background(), client) + if err != nil { + return "", err + } + + return pbmClient.ProfileIDByName(context.Background(), profileName) +} + +// GetOrCreateEncryptionStoragePolicy Gets already created Encryption Storage Policy ID or creates one if not found. +func GetOrCreateEncryptionStoragePolicy(ctx context.Context, client *vim25.Client, profileName, wcpProfileID string) (string, error) { + pbmClient, err := pbm.NewClient(ctx, client) + if err != nil { + return "", err + } + + policyID, err := pbmClient.ProfileIDByName(ctx, profileName) + if err == nil { + return policyID, nil + } + + if !strings.Contains(err.Error(), "no pbm profile found") { + return "", err + } + + m, err := pbmClient.ProfileMap(ctx, wcpProfileID) + if err != nil { + return "", err + } + + wcpProfile := m.Profile[0] + + createSpec, err := pbm.CreateCapabilityProfileSpec(pbm.CapabilityProfileCreateSpec{ + Name: profileName, + SubProfileName: "Host based services", + Description: "Encryption storage profile + " + wcpProfile.GetPbmProfile().Description, + CapabilityList: []pbm.Capability{{ + ID: encryptionCapabilityID, + Namespace: encryptionCapabilityNamespace, + PropertyList: []pbm.Property{{ + ID: encryptionCapabilityID, + Value: encryptionCapabilityID, // Value is same as ID in this case + DataType: "string", + }}, + }}, + Category: string(types.PbmProfileCategoryEnumREQUIREMENT), + }) + if err != nil { + return "", err + } + + // Add wcpProfileID's capabilities - tagged shared datastore (sharedVmfs-0 / vsanDatastore) + // To see the result: govc storage.policy.info -dump "VM Service Encryption Policy" + subProfile := &createSpec.Constraints.(*types.PbmCapabilitySubProfileConstraints).SubProfiles[0] + if p, ok := wcpProfile.(*types.PbmCapabilityProfile); ok { + if c, ok := p.Constraints.(*types.PbmCapabilitySubProfileConstraints); ok { + subProfile.Capability = append(subProfile.Capability, c.SubProfiles[0].Capability...) + } + } + + profile, err := pbmClient.CreateProfile(ctx, *createSpec) + if err != nil { + return "", err + } + + return profile.UniqueId, nil +} + +// GetOrCreateEZTStoragePolicy Gets already created EZT (Eager Zeroed Thick) +// Storage Policy ID or creates one if not found. This creates a storage policy +// with VMFS "Fully initialized" volume allocation and inherits datastore placement +// tags from the base WCP profile. +func GetOrCreateEZTStoragePolicy(ctx context.Context, client *vim25.Client, profileName, wcpProfileID string) (string, error) { + pbmClient, err := pbm.NewClient(ctx, client) + if err != nil { + return "", err + } + + // Check if policy already exists. + policyID, err := pbmClient.ProfileIDByName(ctx, profileName) + if err == nil { + return policyID, nil + } + + // Check for not found error only then proceed for create. + if !strings.Contains(err.Error(), "no pbm profile found") { + return "", err + } + + // Get the base WCP profile to inherit datastore tag/category capabilities. + m, err := pbmClient.ProfileMap(ctx, wcpProfileID) + if err != nil { + return "", err + } + + wcpProfile := m.Profile[0] + + // Create storage policy with VMFS "Fully initialized" (EZT) volume allocation. + createSpec, err := pbm.CreateCapabilityProfileSpec(pbm.CapabilityProfileCreateSpec{ + Name: profileName, + SubProfileName: "VMFS rules", + Description: "EZT storage profile + " + wcpProfile.GetPbmProfile().Description, + CapabilityList: []pbm.Capability{{ + ID: volumeAllocationTypeID, + Namespace: volumeAllocationNamespace, + PropertyList: []pbm.Property{{ + ID: volumeAllocationTypeID, + Value: fullyInitializedValue, // "Fully initialized" = Eager Zeroed Thick + DataType: "string", + }}, + }}, + Category: string(types.PbmProfileCategoryEnumREQUIREMENT), + }) + if err != nil { + return "", err + } + + // Add wcpProfileID's capabilities - tagged shared datastore (for placement). + // This inherits the tag/category from the base WCP profile + // (e.g., wcpglobal_tag/wcpglobal_tag_category). + subProfile := &createSpec.Constraints.(*types.PbmCapabilitySubProfileConstraints).SubProfiles[0] + if p, ok := wcpProfile.(*types.PbmCapabilityProfile); ok { + if c, ok := p.Constraints.(*types.PbmCapabilitySubProfileConstraints); ok { + // Copy all capabilities from the base profile (includes datastore tags) + subProfile.Capability = append(subProfile.Capability, c.SubProfiles[0].Capability...) + } + } + + profile, err := pbmClient.CreateProfile(ctx, *createSpec) + if err != nil { + return "", err + } + + return profile.UniqueId, nil +} + +// GetOrCreateWorkerStoragePolicy returns an existing vSphere profile ID for profileName, or creates +// a policy that duplicates the supervisor primary (WCP global) storage policy placement rules. +// After creation, WCPSVC syncs a Kubernetes StorageClass; callers should wait for that object. +// Pattern matches vks-gce2e EZT helper (duplicate base capabilities under a new name). +func GetOrCreateWorkerStoragePolicy(ctx context.Context, client *vim25.Client, profileName, wcpProfileID string) (string, error) { + pbmClient, err := pbm.NewClient(ctx, client) + if err != nil { + return "", err + } + + policyID, err := pbmClient.ProfileIDByName(ctx, profileName) + if err == nil { + return policyID, nil + } + + if !strings.Contains(err.Error(), "no pbm profile found") { + return "", err + } + + m, err := pbmClient.ProfileMap(ctx, wcpProfileID) + if err != nil { + return "", err + } + + wcpProfile := m.Profile[0] + + createSpec, err := pbm.CreateCapabilityProfileSpec(pbm.CapabilityProfileCreateSpec{ + Name: profileName, + SubProfileName: "Datastore placement", + Description: "Worker storage profile (clone of WCP global capabilities) — " + wcpProfile.GetPbmProfile().Description, + CapabilityList: []pbm.Capability{}, + Category: string(types.PbmProfileCategoryEnumREQUIREMENT), + }) + if err != nil { + return "", err + } + + subProfile := &createSpec.Constraints.(*types.PbmCapabilitySubProfileConstraints).SubProfiles[0] + if p, ok := wcpProfile.(*types.PbmCapabilityProfile); ok { + if c, ok := p.Constraints.(*types.PbmCapabilitySubProfileConstraints); ok { + subProfile.Capability = append(subProfile.Capability, c.SubProfiles[0].Capability...) + } + } + + if len(subProfile.Capability) == 0 { + return "", errors.New("could not copy capability constraints from base WCP storage policy") + } + + profile, err := pbmClient.CreateProfile(ctx, *createSpec) + if err != nil { + return "", err + } + + return profile.UniqueId, nil +} + +// GetOrCreateVsanDirectStoragePolicyID Gets already created VSAN Direct Storage Policy ID or Creates one if not found. +func GetOrCreateVsanDirectStoragePolicyID(ctx context.Context, client *vim25.Client, profileName string) (string, error) { + pbmClient, err := pbm.NewClient(ctx, client) + if err != nil { + return "", err + } + + policyID, err := pbmClient.ProfileIDByName(ctx, profileName) + if err == nil { + return policyID, nil + } + // Check for not found error only then proceed for create + if !strings.Contains(err.Error(), "no pbm profile found") { + return "", err + } + + createSpec, err := pbm.CreateCapabilityProfileSpec(pbm.CapabilityProfileCreateSpec{ + Name: profileName, + SubProfileName: "Storage sub profile", + Description: "vSAN Direct Storage profile", + CapabilityList: []pbm.Capability{{ + ID: vSANDirectTypeID, + Namespace: vSANDirect, + PropertyList: []pbm.Property{{ + ID: vSANDirectTypeID, + Value: vSANDirect, + DataType: "string", + }}, + }}, + Category: string(types.PbmProfileCategoryEnumREQUIREMENT), + }) + if err != nil { + return "", err + } + + profile, err := pbmClient.CreateProfile(ctx, *createSpec) + if err != nil { + return "", err + } + + return profile.UniqueId, nil +} + +// IsVSANDEnabledCluster checks if given wcp enabled cluster is enabled with vsand capability. +func IsVSANDEnabledCluster(ctx context.Context, client *vim25.Client, kubeconfigPath string) (bool, error) { + clusterMOID := GetClusterMoIDFromKubeconfig(ctx, kubeconfigPath) + if clusterMOID == "" { + return false, errors.New("could not fetch cluster moid from wcp cluster config") + } + + cluster := object.NewClusterComputeResource( + client, + vimtypes.ManagedObjectReference{ + Type: "ClusterComputeResource", + Value: clusterMOID, + }, + ) + + return clusterConfiguredWithVsand(ctx, cluster) +} + +func clusterConfiguredWithVsand(ctx context.Context, cluster *object.ClusterComputeResource) (bool, error) { + var cr mo.ComputeResource + + err := cluster.Properties(ctx, cluster.Reference(), []string{"datastore"}, &cr) + if err != nil { + return false, err + } + + if len(cr.Datastore) == 0 { + return false, errors.New("no datastores in cluster") + } + + var datastores []mo.Datastore + + pc := property.DefaultCollector(cluster.Client()) + + err = pc.Retrieve(ctx, cr.Datastore, []string{"summary"}, &datastores) + if err != nil { + return false, err + } + + for _, d := range datastores { + if d.Summary.Type == "vsanD" { + return true, nil + } + } + + return false, nil +} diff --git a/test/e2e/infrastructure/vsphere/vcenter/tasks.go b/test/e2e/infrastructure/vsphere/vcenter/tasks.go new file mode 100644 index 000000000..b25825607 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/vcenter/tasks.go @@ -0,0 +1,193 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/history" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/methods" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" + + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + maxPageSize = 1000 +) + +// VCTask is a helper struct to inspect and compare vCenter Tasks. +type VCTask struct { + TaskMoid string + Name string + Description string + EntityMoid string + State types.TaskInfoState + Progress int32 + ErrorDesc string +} + +// RecentTasks returns a list of vCenter Tasks fetched using the given Client. The pastDuration argument specifies +// how far back in time to gather Tasks, and any Tasks with older start times will be excluded. The entity argument +// allows optional filtering based on ManagedObjects in the vCenter inventory. +func RecentTasks(client *vim25.Client, pastDuration time.Duration, entity *types.ManagedObjectReference) []VCTask { + now := getCurrentTime(client) + recentTimeDuration := now.Add(-1 * pastDuration) + + return getTasksWithinTimeRange(client, entity, recentTimeDuration, time.Now()) +} + +// LookupTask checks vCenter's Recent Tasks for a match with the given name and description. An optional entity +// parameter supports restricting the Task list to a specific ManagedObject. Returns nil if there are no matches. +// Restricts search to Tasks that have started in the duration between pastDuration and now. +func LookupTask(client *vim25.Client, taskName, taskDescription string, pastDuration time.Duration, + entity *types.ManagedObjectReference) *VCTask { + recentTasks := RecentTasks(client, pastDuration, entity) + framework.Logf("Searching %d recent Tasks for %s with description: %s", len(recentTasks), taskName, + taskDescription) + + for _, task := range recentTasks { + if task.Name == taskName && task.Description == taskDescription { + return &task + } + } + + var taskStr strings.Builder + for _, t := range recentTasks { + fmt.Fprintf(&taskStr, "{Name: %s, Description: %s} \n", t.Name, t.Description) + } + + framework.Logf("No tasks matching description: %s All recent Tasks: %s", taskDescription, taskStr.String()) + + return nil +} + +// WaitForTaskToBeComplete looks up a vCenter Task and waits on it to complete. +func WaitForTaskToBeComplete(client *vim25.Client, targetTask *VCTask) types.TaskInfoState { + Expect(client).NotTo(BeNil()) + Expect(targetTask).NotTo(BeNil()) + + // Construct a ManagedObjectReference for the Task and wait for it to reach terminal state. + obj := types.ManagedObjectReference{ + Type: "Task", + Value: targetTask.TaskMoid, + } + taskObj := object.NewTask(client, obj) + taskInfo, err := taskObj.WaitForResult(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(taskInfo).NotTo(BeNil()) + + return taskInfo.State +} + +// ExpectTaskToSucceed waits on a vCenter Task to complete and verifies that it succeeds. +func ExpectTaskToSucceed(client *vim25.Client, targetTask *VCTask) { + taskState := WaitForTaskToBeComplete(client, targetTask) + Expect(taskState).To(Equal(types.TaskInfoStateSuccess)) +} + +// getCurrentTime fetches the currentTime from the vCenter Server. +func getCurrentTime(client soap.RoundTripper) *time.Time { + res, err := methods.GetCurrentTime(context.Background(), client) + Expect(err).NotTo(HaveOccurred()) + + return res +} + +// getTasksWithinTimeRange uses a TaskHistoryCollector with Filters to narrow the search for recent Tasks. +func getTasksWithinTimeRange(client *vim25.Client, watch *types.ManagedObjectReference, start time.Time, end time.Time) []VCTask { + Expect(client).NotTo(BeNil()) + + // Setup Time filter based on start and end times. + filter := types.TaskFilterSpec{ + Time: &types.TaskFilterSpecByTime{ + TimeType: types.TaskFilterSpecTimeOptionStartedTime, + BeginTime: &start, + EndTime: &end, + }, + } + + // Add Entity filter if watch argument is specified. + if watch != nil { + filter.Entity = &types.TaskFilterSpecByEntity{ + Entity: *watch, + Recursion: types.TaskFilterSpecRecursionOptionSelf, + } + } + + ctx := context.Background() + taskReq := types.CreateCollectorForTasks{ + This: *client.ServiceContent.TaskManager, + Filter: filter, + } + res, err := methods.CreateCollectorForTasks(ctx, client, &taskReq) + Expect(err).NotTo(HaveOccurred()) + + collector := history.NewCollector(client, res.Returnval) + err = collector.Reset(ctx) + Expect(err).NotTo(HaveOccurred()) + + err = collector.SetPageSize(ctx, maxPageSize) + Expect(err).NotTo(HaveOccurred()) + + var ( + taskInfo []types.TaskInfo + taskHistoryCollector mo.TaskHistoryCollector + ) + + err = collector.Properties(ctx, collector.Reference(), []string{"latestPage"}, &taskHistoryCollector) + Expect(err).NotTo(HaveOccurred()) + + taskInfo = append(taskInfo, taskHistoryCollector.LatestPage...) + + tasks := []VCTask{} + + for _, task := range taskInfo { + name := strings.TrimSuffix(task.Name, "_Task") + if task.Entity == nil { + continue + } + + if len(name) == 0 { + name = task.DescriptionId + } + + description := task.DescriptionId + if task.Description != nil { + description = task.Description.Message + } + + var taskErr string + if task.Error != nil { + taskErr = task.Error.LocalizedMessage + } + + tasks = append(tasks, VCTask{ + TaskMoid: task.Task.Value, + Name: name, + Description: description, + EntityMoid: task.Entity.Value, + State: task.State, + Progress: task.Progress, + ErrorDesc: taskErr, + }) + } + + err = collector.Destroy(ctx) + if err != nil { + // Collectors should be cleaned up but an error here is not fatal. + framework.Logf("Failed to Destroy TaskHistoryCollector: %v", err) + } + + return tasks +} diff --git a/test/e2e/infrastructure/vsphere/wcp/api.go b/test/e2e/infrastructure/vsphere/wcp/api.go new file mode 100644 index 000000000..e292b47b0 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/api.go @@ -0,0 +1,240 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +// Missing features tracked here +// - Resource quotas (CPU/memory). Don't think we need them in GC yet, but maybe nice to have. +// - WCP Enablement/disablement. Not something we currently use/intend to test. + +// WorkloadManagementAPI exposes methods to interact with wcpsvc on a vCenter. This is not meant to be a faithful implementation +// +// of all WCP VAPI methods, but a minimal subset to make writing gce2e tests/enhancing the framework easier. +// +// For an exhaustive list of WCP VMODL methods, see https://vcode.eng.vmware.com/home/explorerepo/blob/@latest/bora.main.perforce.1666/bora/main/vpx/wcp/wcpsvc/vmodl +// ListClusters lists all WCP enabled clusters. +// GetClusters gets information about a specific cluster, given the cluster MoID. +// ListNamespaces lists all WCP enabled namespaces. +// GetNamespace returns information about a specific namespace, given the name. +// CreateNamespace creates a namespace in the given cluster (by MoID) with the given name. +// DeleteNamespace deletes the given namespace. +// SetNamespaceStorageSpecs associates a list of storage policies (with optional limits) with a given namespace, +// +// functioning as a PUT/Set() operation (ie. removing any old associations) +// +// CreateNamespacePermissions grants a principal (user/group) access (either edit or view) to a given namespace. +// +// This assumes the user already has access to the namespace. +// RemoveNamespacePermissions removes all privileges the principal has on a given namespace. +type WorkloadManagementAPI interface { + // Supervisor Cluster CRUD + ListSupervisorSummary() (SupervisorSummaryList, error) + + // Namespace CRUD + ListClusters() ([]WCPClusterDetails, error) + GetCluster(string) (WCPClusterDetails, error) + ListNamespaces() ([]NamespaceDetails, error) + GetNamespace(string) (NamespaceDetails, error) + GetNamespaceV2(string) (NamespaceDetails, error) + CreateNamespace(clusterMoid, namespace string) error + CreateNamespaceWithSpecs(clusterMoid, namespace string, storageSpecs []StorageSpec, vmsvcSpec VMServiceSpecDetails) error + CreateNamespaceWithVMReservation(namespace, zone, supervisorID string, storageSpecs []StorageSpec, vmsvcSpec VMServiceSpecDetails, vmClassNameToReservedCount map[string]int) error + CreateNamespaceWithNetwork(clusterMoid, namespace string, storageSpecs []StorageSpec, vmsvcSpec VMServiceSpecDetails, network *NameSpaceNetworkInfo) error + DeleteNamespace(namespace string) error + SetNamespaceStorageSpecs(namespace string, specs []StorageSpec) error + + CreateNamespaceNetwork(config NamespaceNetworkConfig) error + GetNamespaceNetwork(cluster, network string) (map[string]any, error) + UpdateNamespaceWithNetworks(namespaceName, networkProvider, networkToAdd string) error + + GetVirtualMachine(vmMoid string) (VirtualMachineDetails, error) + + // Namespaces Authz + CreateNamespacePermissions(user Principal, namespace string, level AccessType) error + RemoveNamespacePermissions(principal Principal, namespace string) error + + // SupervisorServices + DeleteSupervisorService(serviceID string) error + + // ActivateSupervisorService sets the state of a service, and all its versions, to ACTIVATED state. + ActivateSupervisorService(serviceID string) error + + // DeactivateSupervisorService sets the state of a service, and all its versions, to DEACTIVATED state. + DeactivateSupervisorService(serviceID string) error + + // CreateOrSetClusterSupervisorService creates or sets specified Supervisor Service Version in the cluster + CreateOrSetClusterSupervisorService(op DcliOperationType, cluster, serviceID, version string, serviceConfig map[string]string) error + CreateOrSetClusterSupervisorServiceWithYamlConfig(op DcliOperationType, cluster, serviceID, version, yamlServiceConfig string) error + + // Enables/Disables PSP supervisor service in the cluster using v1 "set" API. + EnableV1SupervisorService(cluster, serviceID string, version *string, serviceConfig map[string]string) error + DisableV1SupervisorService(cluster, serviceID string, version *string, serviceConfig map[string]string) error + + // DeleteClusterSupervisorService deletes specified Supervisor Service from the cluster + DeleteClusterSupervisorService(cluster, serviceID string) error + + // ActivateSupervisorService sets the state of a service version to ACTIVATED state. + ActivateSupervisorServiceVersion(serviceID, version string) error + + // DeactivateSupervisorService sets the state of a service version to DEACTIVATED state. + DeactivateSupervisorServiceVersion(serviceID, version string) error + + // DeleteSupervisorServiceVersion deletes a specific version of a Supervisor Service. + DeleteSupervisorServiceVersion(serviceID, version string) error + + // Carvel Service + RegisterCarvelService(carvelYaml []byte) error + RegisterCarvelServiceVersion(serviceID string, carvelYaml []byte) error + + // ServicePrecheck initiates a pre-check task for a Supervisor Service version against a Supervisor. + ServicePrecheck(serviceID, version, supervisor string) error + + // VMService APIs. + // GetVMClassInfo returns info about a given VMClass. + GetVMClassInfo(vmClass string) (VMClassInfo, error) + + // ListVMClasses lists all the VMClasses in the VC. + ListVMClasses() ([]VMClassInfo, error) + + // CreateVMClass creates VMClass with given spec in the VC. + CreateVMClass(createSpec VMClassSpec) error + + // UpdateVMClass updates an existing VMClass in the VC. + UpdateVMClass(updateSpec VMClassSpec) error + + // DeleteVMClass deletes specified VMClass from the VC. + DeleteVMClass(vmClass string) error + + // UpdateNamespaceVMServiceSpec updates namespace instance for changes made w.r.t. VMService specs. + UpdateNamespaceVMServiceSpec(namespaceName string, updateSpec NamespaceUpdateVMserviceSpec) error + + // ListContentLibraries lists all the content libraries in the VC + ListContentLibraries() ([]string, error) + + // GetContentLibrary returns a ContentLibrary via an ID. + GetContentLibrary(id string) (ContentLibraryInfo, error) + + // FetchContentLibraryIDByName fetches a content library ID given name of CL ID + FetchContentLibraryIDByName(name string, libraries []string) (string, error) + + // ListDatastores lists all datastores in vCenter. + ListDatastores() ([]Datastore, error) + + // AddCLTrustedCertificate adds trusted certificate to Content Library. + AddCLTrustedCertificate(trustedCertificate string) (string, error) + + // DeleteCLTrustedCertificate deletes a trusted certificate from Content Library. + DeleteCLTrustedCertificate(trustedCertificateID string) error + + // ListCLSecurityPolicies lists all security policies in CL. + ListCLSecurityPolicies() ([]SecurityPolicyInfo, error) + + // CreateLocalContentLibrary creates a local Content Library with the associated storage backing info. + CreateLocalContentLibrary(name string, storageBackings StorageBackingInfo) (string, error) + + // DeleteLocalContentLibrary deletes a local Content Library. + DeleteLocalContentLibrary(id string) error + + // DeleteLocalContentLibraryByForce force deletes a local Content Library + DeleteLocalContentLibraryByForce(id string) error + + // UpdateContentLibrary updates common aspects of a Content Library. + UpdateContentLibrary(id, description, securityPolicyID string) error + + // UpdateLocalContentLibrary updates aspects of a local Content Library. + UpdateLocalContentLibrary(id string, enablePublishing bool) error + + // CreateSubscribedContentLibrary creates a subscribed Content Library with the given subscription URL and storage backing info. + CreateSubscribedContentLibrary(name, subscriptionURL, thumbprint string, onDemand bool, storageBackings StorageBackingInfo) (string, error) + + // SyncSubscribedContentLibrary synchronizes the specified content library. + SyncSubscribedContentLibrary(id string) error + + // DeleteSubscribedContentLibrary deletes a subscribed Content Library. + DeleteSubscribedContentLibrary(id string) error + + // DeleteSubscribedContentLibraryByForce force deletes a subscribed Content Library. + DeleteSubscribedContentLibraryByForce(id string) error + + // CreateContentLibraryItem creates a new Content Library item + CreateContentLibraryItem(contentLibraryID, name string, itemType ContentLibraryItemType) (string, error) + + // CreateContentLibraryOVFTemplateItemByPull creates a new content library item in OVF by pulling the given OVF URL. + // It does not wait for the content upload completion, client should wait and check for item ready status. + CreateContentLibraryOVFTemplateItemByPull(contentLibraryID, name, ovfFileURL string) (string, error) + + // ListContentLibraryItems returns a list of library item identifiers. + ListContentLibraryItems(contentLibraryID string) ([]string, error) + + // GetContentLibraryItem returns a Content Library Item + GetContentLibraryItem(id string) (ContentLibraryItemInfo, error) + + // UpdateContentLibraryItem updates aspects of a Content Library Item. + UpdateContentLibraryItem(id, description string) error + + // DeleteContentLibraryItem deletes a Content Library Item + DeleteContentLibraryItem(id string) error + + // AssociateContentLibrariesToCluster associates a list of content libraries to a Supervisor Cluster. + AssociateContentLibrariesToCluster(cluster string, contentLibraries ...ClusterContentLibrarySpec) error + + // DisassociateContentLibrariesFromCluster disassociates the provided list of content libraries from a Supervisor Cluster. + // If they were not already associated, this will be accepted as success. + DisassociateContentLibrariesFromCluster(cluster string, contentLibraryIDs ...string) error + + // AssociateImageRegistryContentLibrariesToNamespace associates a list of Image Registry content libraries to a Supervisor Namespace. + AssociateImageRegistryContentLibrariesToNamespace(namespace string, contentLibraries ...ContentLibrarySpec) error + + // DisassociateImageRegistryContentLibrariesFromNamespace disassociates the provided list of content libraries from a Supervisor namespace. + // If they were not already associated, this will be accepted as success. + DisassociateImageRegistryContentLibrariesFromNamespace(namespace string, contentLibraryIDs ...string) error + + // RegisterVM registers an existing virtual machine as VM Service managed VM. + RegisterVM(namespace string, vmMoID string) (string, error) + + // GetWorkerDNS returns the set worker DNS server IPs + GetWorkerDNS(clusterMoid string) ([]string, error) + + // UpdateWorkerDNS updates the worker DNS server IPs on the cluster. + UpdateWorkerDNS(clusterMoid string, dnsServerIPs ...string) error + + // CRUD APIs for configuring cluster proxy settings on the cluster + // see: https://opengrok2.eng.vmware.com/xref/main.perforce.1666/bora/vpx/wcp/wcpsvc/vmodl/namespace_management/ProxyConfiguration.vmodl + UpdateClusterProxyConfig(clusterMoid string, proxyConfig ClusterProxyConfig) error + GetClusterProxyConfig(clusterMoid string) (ClusterProxyConfig, error) + + // CRUD APIs for configuring container image registries + CreateContainerImageRegistry(supervisorID string, registry ContainerImageRegistry) (ContainerImageRegistryInfo, error) + GetContainerImageRegistry(supervisorID, registryName string) (ContainerImageRegistryInfo, error) + DeleteContainerImageRegistry(supervisorID, registryID string) error + + // CRUD APIs for configuring Key Providers + CreateKeyProvider(provider string) error + DeleteKeyProvider(provider string) error + + // CRUD APIs for Zones + ListVSphereZones() (VSphereZoneList, error) + GetZonesBoundWithSupervisor(supervisorID string) (ZoneList, error) + UpdateNamespaceWithZones(namespace string, zones []ZoneSpec) error + DeleteZoneFromNamespace(namespace string, zone string) error + CreateZoneBindingsWithSupervisor(supervisorID string, zoneBindingSpecs []ZoneBindingSpecs) error + DeleteZoneBindingsWithSupervisor(supervisorID string, zones string) error + UpdateZoneBindingsWithVMReservation(supervisorID, zone string, reservedVMClassToCount map[string]int) error + + // CIS APIs. + AssignLicenseEntitlement(signedEntitlement string) (string, error) + CreateTagCategory(name, description string) (string, error) + CreateTag(name, description, categoryID string) (string, error) + AssignTagsToHost(tagIDs []string, hostID string) error + + // IaaS Policy APIs. + CreateComputePolicy(spec ComputePolicySpec) (string, error) + CreateInfraPolicy(spec InfraPolicySpec) error + UpdateNamespaceWithInfraPolicies(namespace string, policyNames ...string) error + + // NSX APIs + GetSupervisorLoadBalancerProvider(supervisorID string) (string, error) + + // Host APIs. + ListHostIDs() ([]string, error) +} diff --git a/test/e2e/infrastructure/vsphere/wcp/authz.go b/test/e2e/infrastructure/vsphere/wcp/authz.go new file mode 100644 index 000000000..9ed38ac9c --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/authz.go @@ -0,0 +1,38 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +// NamespaceAccess is part of the response returned by wcpsvc when getting the +// +// privileges assigned to a given principal on a namespace. +// +// It's technically used in the VMODL definition to assign privileges as well, but the +// +// WorkloadManagementAPI interface in gce2e simplifies those methods to take in +// the principal, namespace (entity), and access type (privilege), to look more +// like other VC authorization manager methods. +type NamespaceAccess struct { + Role string `json:"role"` + SubjectType string `json:"subject_type"` +} + +// SubjectType defines whether the subject principal privileges is a user or a group. +type SubjectType string + +// AccessType controls the kind of privileges a user receives on a supervisor namespace. +type AccessType string + +const ( + UserSubjectType SubjectType = "USER" + GroupSubjectType SubjectType = "GROUP" + EditAccessType AccessType = "EDIT" + ViewAccessType AccessType = "VIEW" +) + +// Principal represents an entity to be granted privileges (for instance, a user or a group). +type Principal struct { + Type SubjectType `json:"subject_type"` + Name string `json:"name"` + Domain string `json:"domain"` +} diff --git a/test/e2e/infrastructure/vsphere/wcp/cluster.go b/test/e2e/infrastructure/vsphere/wcp/cluster.go new file mode 100644 index 000000000..80803bb37 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/cluster.go @@ -0,0 +1,25 @@ +// Copyright (c) 2020-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +type ClusterContentLibrarySpec struct { + ContentLibrary string `json:"content_library"` + SupervisorServices []string `json:"supervisor_services"` + ResourceNamingStrategy string `json:"resource_naming_strategy"` +} + +// WCPClusterDetails contains detailed information about a cluster, including IP addresses, content library details, etc. +// +// This is information obtained by getting a specific WCP enabled cluster. +type WCPClusterDetails struct { + MoID string `json:"cluster"` + KubernetesStatus string `json:"kubernetes_status"` + ConfigStatus string `json:"config_status"` + APIServerClusterEndpoint string `json:"api_server_cluster_endpoint"` + APIServerManagementEndpoint string `json:"api_server_management_endpoint"` + DefaultContentLibraryID string `json:"default_kubernetes_service_content_library"` + ContentLibraries []ClusterContentLibrarySpec `json:"content_libraries"` + ClusterProxyConfig ClusterProxyConfig `json:"cluster_proxy_config"` + NetworkProvider string `json:"network_provider"` +} diff --git a/test/e2e/infrastructure/vsphere/wcp/container_image_registry.go b/test/e2e/infrastructure/vsphere/wcp/container_image_registry.go new file mode 100644 index 000000000..ef1a8e1d7 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/container_image_registry.go @@ -0,0 +1,25 @@ +// Copyright (c) 2020-2024 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +type ImageRegistry struct { + Hostname string `json:"hostname"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password"` + CertificateChain string `json:"certificate_chain"` +} + +type ContainerImageRegistry struct { + Name string `json:"name"` + DefaultRegistry bool `json:"default_registry"` + ImageRegistry ImageRegistry `json:"image_registry"` +} + +type ContainerImageRegistryInfo struct { + ID string `json:"id"` + Name string `json:"name"` + DefaultRegistry bool `json:"default_registry"` + ImageRegistry ImageRegistry `json:"image_registry"` +} diff --git a/test/e2e/infrastructure/vsphere/wcp/contentlibrary.go b/test/e2e/infrastructure/vsphere/wcp/contentlibrary.go new file mode 100644 index 000000000..fb109975b --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/contentlibrary.go @@ -0,0 +1,96 @@ +// Copyright (c) 2021-2024 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +// Datastore represents VMODL output for datastore get on a vCenter. +type Datastore struct { + Datastore string `json:"datastore"` + Name string `json:"name"` + Type string `json:"type"` + FreeSpace int `json:"free_space"` + Capacity int `json:"capacity"` +} + +// BackingInfo represents datastore backing used for storing VMImages of the Content Library. +type BackingInfo struct { + DatastoreID string `json:"datastore_id"` + Type string `json:"type"` +} + +// StorageBackingInfo represents storage backing used for storing VMImages of the Content Library. +type StorageBackingInfo struct { + StorageBackings []BackingInfo +} + +// SecurityPolicyInfo represents datastore backing used for storing VMImages of the Content Library. +type SecurityPolicyInfo struct { + PolicyID string `json:"policy"` + Name string `json:"name"` + ItemTypeRules map[string]string `json:"item_type_rules"` +} + +// SubscriptionInfo represents subscribed Content Library related information. +type SubscriptionInfo struct { + AuthenticationMethod string `json:"authentication_method"` + Password string `json:"password"` + SubscriptionURL string `json:"subscription_url"` + AutomaticSyncEnabled bool `json:"automatic_sync_enabled"` + OnDemand bool `json:"on_demand"` +} + +type PublishInfo struct { + AuthenticationMethod string `json:"authentication_method"` + UserName string `json:"user_name"` + Password string `json:"password"` + Published bool `json:"published"` + PublishURL string `json:"publish_url"` + PersistJSONEnabled bool `json:"persist_json_enabled"` +} + +// ContentLibraryInfo represents VMODL output for content library get on a vCenter. +type ContentLibraryInfo struct { + StorageBackingInfo + + CreationTime string `json:"creation_time"` + LastModifiedTime string `json:"last_modified_time"` + ServerGUID string `json:"server_guid"` + Description string `json:"description"` + Type string `json:"type"` + Version string `json:"version"` + SubscriptionInfo SubscriptionInfo `json:"subscription_info"` + PublishInfo PublishInfo `json:"publish_info"` + Name string `json:"name"` + ID string `json:"id"` +} + +// ContentLibraryItemInfo represents VMODL output for a Content Library Item on a vCenter. +type ContentLibraryItemInfo struct { + CreationTime string `json:"creation_time"` + LastModifiedTime string `json:"last_modified_time"` + Description string `json:"description"` + Type ContentLibraryItemType `json:"type"` + Version string `json:"version"` + ContentVersion string `json:"content_version"` + LibraryID string `json:"library_id"` + Size int `json:"size"` + Cached bool `json:"cached"` + Name string `json:"name"` + ID string `json:"id"` + SourceID string `json:"source_id"` + SecurityCompliance bool `json:"security_compliance"` +} + +type ProbeResult struct { + Status string `json:"status"` + SSLThumbprint string `json:"ssl_thumbprint"` + ErrorMessages []string `json:"error_messages"` +} + +// ContentLibraryItemType represents different types of supported Content Library Objects. +type ContentLibraryItemType string + +const ( + OVF ContentLibraryItemType = "ovf" + ISO ContentLibraryItemType = "iso" +) diff --git a/test/e2e/infrastructure/vsphere/wcp/dcli_client.go b/test/e2e/infrastructure/vsphere/wcp/dcli_client.go new file mode 100644 index 000000000..37c392157 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/dcli_client.go @@ -0,0 +1,2224 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +import ( + "bytes" + "context" + b64 "encoding/base64" + "encoding/json" + "fmt" + "path" + "slices" + "strings" + "time" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/dcli" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/vim25/types" +) + +// wcpDcliClient of WorkloadManagementAPI that uses DCLI + SSH to achieve the desired result. +// This is a short term workaround until a better REST client is implemented (which would do +// +// a better job of catching breaking API contracts) while also decoupling us from generated vapi-go code. +// (which is potentially a pain to consume from bora/VMODL) +type wcpDcliClient struct { + dcliClient dcli.DCLICommandRunner +} + +const ( + dcliVCenterPrefix = "com vmware vcenter " + dcliNamespaces = dcliVCenterPrefix + "namespaces " + namespaceManagement = dcliVCenterPrefix + "namespacemanagement " + supervisors = namespaceManagement + "supervisors" + namespacesInstances = dcliNamespaces + "instances " + namespacesAccess = dcliNamespaces + "access " + namespaceServices = dcliNamespaces + "supervisorservices " // for supervisor service v1 APIs + namespaceManagementServices = namespaceManagement + " +show-unreleased-apis supervisorservices " + namespaceManagementClusterServices = namespaceManagementServices + " clustersupervisorservices " + namespaceManagementSupervisors = namespaceManagement + " supervisors" + namespaceMgmtSupervisorsSvServices = namespaceManagementSupervisors + " supervisorservices +show-unreleased" + namespaceManagementVMClassServices = namespaceManagement + "virtualmachineclasses +show-unreleased" + virtualmachine = dcliVCenterPrefix + "vm" + consumptiondomainsPrefix = "consumptiondomains " + zones = dcliVCenterPrefix + consumptiondomainsPrefix + "zones" + cryptoManager = dcliVCenterPrefix + "cryptomanager" + dcliContentPrefix = "com vmware content library" + dcliContentLocalPrefix = "com vmware content locallibrary" + dcliContentSubscribedPrefix = "com vmware content subscribedlibrary" + dcliCISLicenseEntitlementPrefix = "com vmware cis license subscription entitlement" + dcliSupervisorSummary = namespaceManagement + "supervisors summary list" + dcliSupervisorNetworkEdges = namespaceManagement + "supervisors networks edges list " + jsonFormatter = "+formatter json" + showUnreleased = "+show-unreleased" + namespaceManagementNetworks = namespaceManagement + "networks +show-unreleased" + namespacesInstancesUpdate = dcliNamespaces + "instances update +show-unreleased" +) + +type DcliOperationType int + +const ( + Create DcliOperationType = iota + Set +) + +func (op DcliOperationType) String() string { + return [...]string{"create", "set"}[op] +} + +// DcliError is an error type to help wrap errors that get returned from DCLI commands, +// to be able to help pass along a regular error that might have strings in the response +// we want to verify. +type DcliError struct { + rawResponse string + baseErr error +} + +func (e DcliError) Error() string { + return e.baseErr.Error() +} + +func (e DcliError) Unwrap() error { + return e.baseErr +} + +func (e DcliError) Response() string { + return e.rawResponse +} + +// Add these structs for network creation input. +type NamespaceNetworkConfig struct { + Cluster string `json:"cluster"` + Network string `json:"network"` + NetworkProvider string `json:"network_provider"` + + // vSphere Network specific + VsphereNetworkMode string `json:"vsphere_network_mode"` + VsphereNetworkIPAssignmentMode string `json:"vsphere_network_ip_assignment_mode"` + VsphereNetworkPortgroup string `json:"vsphere_network_portgroup"` + VsphereNetworkGateway string `json:"vsphere_network_gateway"` + VsphereNetworkSubnetMask string `json:"vsphere_network_subnet_mask"` +} + +// NewWCPAPIClient is a public method meant to get a WorkloadManagementAPI, regardless of the backing implementation. +// The idea is to first implement a DCLI based one and then transition to a VAPI backed one, while +// +// consumers can rely on the interface exposed by NewWCPAPIClient/WorkloadManagementAPI. +func NewWCPAPIClient(vCenterHostname string, username, password, sshUsername, sshPassword string) (WorkloadManagementAPI, error) { + return newWcpDcliClient(vCenterHostname, username, password, sshUsername, sshPassword) +} + +func newWcpDcliClient(vCenterHostname, username, password, sshUsername, sshPassword string) (WorkloadManagementAPI, error) { + dcliClient, err := dcli.NewDCLICommandRunner(vCenterHostname, vcenter.VCSSHPort, username, password, sshUsername, sshPassword) + if err != nil { + return nil, err + } + + return &wcpDcliClient{ + dcliClient: dcliClient, + }, nil +} + +func (d *wcpDcliClient) ListSupervisorSummary() (SupervisorSummaryList, error) { + cmd := fmt.Sprintf("%s summary list %s", supervisors, jsonFormatter) + retVal := SupervisorSummaryList{} + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +func (d *wcpDcliClient) ListClusters() ([]WCPClusterDetails, error) { + cmd := fmt.Sprintf("%s clusters list +formatter json", namespaceManagement) + retVal := []WCPClusterDetails{} + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +func (d *wcpDcliClient) GetCluster(moID string) (WCPClusterDetails, error) { + cmd := fmt.Sprintf("%s clusters get --cluster %s +formatter json", namespaceManagement, moID) + retVal := WCPClusterDetails{} + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +func (d *wcpDcliClient) ListNamespaces() ([]NamespaceDetails, error) { + cmd := fmt.Sprintf("%s list +formatter json", namespacesInstances) + retVal := []NamespaceDetails{} + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +func (d *wcpDcliClient) GetNamespace(name string) (NamespaceDetails, error) { + cmd := fmt.Sprintf("%s get --namespace %s +formatter json", namespacesInstances, name) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return NamespaceDetails{}, DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + retVal := NamespaceDetails{} + if err = json.Unmarshal(resp, &retVal); err != nil { + return NamespaceDetails{}, err + } + + return retVal, err +} + +func (d *wcpDcliClient) GetNamespaceV2(name string) (NamespaceDetails, error) { + cmd := fmt.Sprintf("%s getv2 --namespace %s +formatter json", namespacesInstances, name) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return NamespaceDetails{}, DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + retVal := NamespaceDetails{} + if err = json.Unmarshal(resp, &retVal); err != nil { + return NamespaceDetails{}, err + } + + return retVal, nil +} + +func (d *wcpDcliClient) CreateNamespacePermissions(principal Principal, namespace string, level AccessType) error { + cmd := fmt.Sprintf("%s create --domain %s --namespace %s --subject %s --role %s --type %s", + namespacesAccess, + principal.Domain, + namespace, + principal.Name, + level, + principal.Type, + ) + + // We don't really care about the STDOUT of this command. + _, err := d.dcliClient.RunDCLICommand(cmd) + + return err +} + +func (d *wcpDcliClient) RemoveNamespacePermissions(principal Principal, namespace string) error { + cmd := fmt.Sprintf("%s delete --domain %s --namespace %s --subject %s --type %s", + namespacesAccess, + principal.Domain, + namespace, + principal.Name, + principal.Type, + ) + _, err := d.dcliClient.RunDCLICommand(cmd) + + return err +} + +func (d *wcpDcliClient) CreateNamespace(clusterMoid, namespace string) error { + cmd := fmt.Sprintf("%s create --cluster %s --namespace %s", + namespacesInstances, + clusterMoid, + namespace, + ) + _, err := d.dcliClient.RunDCLICommand(cmd) + + return err +} + +// buildCreateNamespaceCommand is a helper function to build the command string for creating a namespace. +func (d *wcpDcliClient) buildCreateNamespaceCommand( + clusterMoid, namespace string, + storageSpecs []StorageSpec, + vmsvcSpec VMServiceSpecDetails, + network *NameSpaceNetworkInfo) (string, error) { + var cmd strings.Builder + + baseCmd, err := d.buildCreateNamespaceBaseCommand(clusterMoid, namespace, network) + if err != nil { + return "", fmt.Errorf("failed to build network-specific command: %w", err) + } + + cmd.WriteString(baseCmd) + + if len(storageSpecs) > 0 { + storageSpecBuffer, err := json.Marshal(storageSpecs) + if err != nil { + return "", fmt.Errorf("failed to marshal storage specs: %w", err) + } + + fmt.Fprintf(&cmd, " --storage-specs '%s'", string(storageSpecBuffer)) + } + + for _, vmClass := range vmsvcSpec.VMClasses { + fmt.Fprintf(&cmd, " --vm-service-spec-vm-classes %s", vmClass) + } + + for _, clID := range vmsvcSpec.ContentLibraries { + fmt.Fprintf(&cmd, " --vm-service-spec-content-libraries %s", clID) + } + + return cmd.String(), nil +} + +// buildCreateNamespaceBaseCommand builds the base command based on the network configuration. +func (d *wcpDcliClient) buildCreateNamespaceBaseCommand( + clusterMoid, namespace string, + network *NameSpaceNetworkInfo) (string, error) { + // Handle VDS network + if network != nil { + if network.VDSNetwork != "" { + return fmt.Sprintf( + "%s create --cluster %s --namespace %s --networks %s", + namespacesInstances, + clusterMoid, + namespace, + network.VDSNetwork, + ), nil + } + // Handle VPC network with necessary validation + if network.VPCNetwork != nil { + if network.VPCNetwork.VPCPath == "" || network.VPCNetwork.SupervisorID == "" || network.VPCNetwork.DefaultSubnetSize == 0 { + return "", fmt.Errorf("VPC network requires VPCPath, SupervisorID, and DefaultSubnetSize to be defined") + } + // com vmware vcenter namespaces instances create --nanespace tkgs-test-1 + // --supervisor e12272e9-253d-489f-b078-f3a4a3948dba + // --network-spec-network-provider NSX_VPC + // --network-spec-vpc-network-vpc /orgs/default/projects/project-quality/vpcs/vpc-ext-104 + // --network-spec-vpc-network-default-subnet-size 32 + // --network-spec-vpc-network-shared-subnets /orgs/default/projects/project-quality/vpcs/vpc-ext-104/subnets/vlan-subnet-104 + + // Build the base command + cmd := fmt.Sprintf( + "%s createv2 --namespace %s "+ + "--supervisor %s "+ + "--network-spec-network-provider NSX_VPC "+ + "--network-spec-vpc-network-default-subnet-size %d "+ + "--network-spec-vpc-network-vpc %s", + namespacesInstances, + namespace, + network.VPCNetwork.SupervisorID, + network.VPCNetwork.DefaultSubnetSize, + network.VPCNetwork.VPCPath, + ) + + // Add shared subnets parameter only if VPCSharedSubnetPath is not empty + // The parameter needs to be formatted as a JSON array of objects with "path" field + if network.VPCNetwork.VPCSharedSubnetPath != "" { + sharedSubnets := []map[string]string{ + {"path": network.VPCNetwork.VPCSharedSubnetPath}, + } + + sharedSubnetsJSON, err := json.Marshal(sharedSubnets) + if err != nil { + return "", fmt.Errorf("failed to marshal shared subnets: %w", err) + } + + cmd += fmt.Sprintf(" --network-spec-vpc-network-shared-subnets '%s'", string(sharedSubnetsJSON)) + } + + return cmd, nil + } + } + + // Handle default case when no network is specified + return fmt.Sprintf("%s create --cluster %s --namespace %s", + namespacesInstances, + clusterMoid, + namespace, + ), nil +} + +// CreateNamespaceWithSpecs creates a namespace with the given specs. +func (d *wcpDcliClient) CreateNamespaceWithSpecs( + clusterMoid, namespace string, + storageSpecs []StorageSpec, + vmsvcSpec VMServiceSpecDetails) error { + // Call the common command building function without network + cmd, err := d.buildCreateNamespaceCommand(clusterMoid, namespace, storageSpecs, vmsvcSpec, nil) + if err != nil { + return err + } + + stdout, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + if !strings.Contains(string(stdout), "errors.AlreadyExists") { + return err + } + } + + return nil +} + +// CreateNamespaceWithNetwork creates a namespace with the given specs and network. +func (d *wcpDcliClient) CreateNamespaceWithNetwork( + clusterMoid, namespace string, + storageSpecs []StorageSpec, + vmsvcSpec VMServiceSpecDetails, + network *NameSpaceNetworkInfo) error { + cmd, err := d.buildCreateNamespaceCommand(clusterMoid, namespace, storageSpecs, vmsvcSpec, network) + if err != nil { + return err + } + + _, err = d.dcliClient.RunDCLICommand(cmd) + + return err +} + +// CreateNamespaceWithVMReservation creates a namespace with the given specs and VM reservations. +func (d *wcpDcliClient) CreateNamespaceWithVMReservation(namespace, zone, supervisorID string, storageSpecs []StorageSpec, vmsvcSpec VMServiceSpecDetails, vmClassNameToReservedCount map[string]int) error { + var cmd strings.Builder + fmt.Fprintf(&cmd, "%s %s createv2 --namespace %s --supervisor %s --zones '[{\"name\": \"%s\", \"vm_reservations\": [", + namespacesInstances, + showUnreleased, + namespace, + supervisorID, + zone) + + for vmClass, count := range vmClassNameToReservedCount { + fmt.Fprintf(&cmd, "{\"reserved_vm_class\": \"%s\", \"count\": %d},", vmClass, count) + } + + // Remove the last comma in "vm_reservations" if exists. + cmdStr := cmd.String() + if cmdStr[len(cmdStr)-1] == ',' { + cmd.Reset() + cmd.WriteString(cmdStr[:len(cmdStr)-1]) + } + + cmd.WriteString("]}]'") + + // Add storage specs. + storageSpecBuffer, err := json.Marshal(storageSpecs) + if err != nil { + return err + } + + storageSpecString := string(storageSpecBuffer) + fmt.Fprintf(&cmd, " --storage-specs '%s'", storageSpecString) + + // Add vm service specs (VM Class and Content Library). + for _, vmClass := range vmsvcSpec.VMClasses { + fmt.Fprintf(&cmd, " --vm-service-spec-vm-classes %s", vmClass) + } + + for _, clID := range vmsvcSpec.ContentLibraries { + fmt.Fprintf(&cmd, " --vm-service-spec-content-libraries %s", clID) + } + + resp, err := d.dcliClient.RunDCLICommand(cmd.String()) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// ListVSphereZones list all vSphere zones. +func (d *wcpDcliClient) ListVSphereZones() (VSphereZoneList, error) { + cmd := fmt.Sprintf("%s list +formatter json", zones) + retVal := VSphereZoneList{} + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +// UpdateNamespaceWithZones binds a namespace with the given namespace scoped Zones. +func (d *wcpDcliClient) UpdateNamespaceWithZones( + namespace string, + zones []ZoneSpec) error { + var cmd strings.Builder + fmt.Fprintf(&cmd, "%s %s update --namespace %s", + namespacesInstances, + showUnreleased, + namespace) + + zoneSpec, err := json.Marshal(zones) + if err != nil { + return err + } + + zoneSpecString := string(zoneSpec) + fmt.Fprintf(&cmd, " --zones '%s'", zoneSpecString) + + _, err = d.dcliClient.RunDCLICommand(cmd.String()) + + return err +} + +// DeleteZoneFromNamespace delete a zone from namespace (Mark Zone for removal). +func (d *wcpDcliClient) DeleteZoneFromNamespace( + namespace string, + zone string) error { + var cmd strings.Builder + fmt.Fprintf(&cmd, "%s %s zones delete --namespace %s --zone %s", + namespacesInstances, + showUnreleased, + namespace, + zone) + + _, err := d.dcliClient.RunDCLICommand(cmd.String()) + + return err +} + +// GetZonesBoundWithSupervisor gets all zones bound with a supervisor. +func (d *wcpDcliClient) GetZonesBoundWithSupervisor( + supervisorID string) (ZoneList, error) { + var cmd strings.Builder + fmt.Fprintf(&cmd, "%s %s zones bindings list --supervisor %s %s", + namespaceManagementSupervisors, + showUnreleased, + supervisorID, + jsonFormatter) + + resp, err := d.dcliClient.RunDCLICommand(cmd.String()) + if err != nil { + return ZoneList{}, DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + retVal := ZoneList{} + if err = json.Unmarshal(resp, &retVal); err != nil { + return ZoneList{}, err + } + + return retVal, err +} + +func (d *wcpDcliClient) CreateZoneBindingsWithSupervisor(supervisorID string, zoneBindingSpecs []ZoneBindingSpecs) error { + specs, err := json.Marshal(zoneBindingSpecs) + Expect(err).NotTo(HaveOccurred()) + + zoneBindingSpecString := string(specs) + + var cmd strings.Builder + fmt.Fprintf(&cmd, "%s %s zones bindings create --supervisor %s --specs '%s'", + namespaceManagementSupervisors, + showUnreleased, + supervisorID, + zoneBindingSpecString) + + resp, err := d.dcliClient.RunDCLICommand(cmd.String()) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DeleteZoneBindingsWithSupervisor(supervisorID string, zone string) error { + var cmd strings.Builder + fmt.Fprintf(&cmd, "%s %s zones bindings delete --supervisor %s %s", + namespaceManagementSupervisors, + showUnreleased, + supervisorID, + jsonFormatter) + + fmt.Fprintf(&cmd, " --zone '%s'", zone) + + resp, err := d.dcliClient.RunDCLICommand(cmd.String()) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// UpdateZoneBindingsWithVMReservation updates the zone bindings with the given VM reservations +// and waits for the zone to be ready after the update. +func (d *wcpDcliClient) UpdateZoneBindingsWithVMReservation(supervisorID, zoneID string, reservedVMClassToCount map[string]int) error { + var cmd strings.Builder + fmt.Fprintf(&cmd, "%s %s zones bindings update --supervisor %s --zone %s --resource-allocation-vm-reservations '[", + namespaceManagementSupervisors, + showUnreleased, + supervisorID, + zoneID) + + for vmClass, count := range reservedVMClassToCount { + fmt.Fprintf(&cmd, "{\"reserved_vm_class\": \"%s\", \"count\": %d},", vmClass, count) + } + + // Remove the last comma in "resource-allocation-vm-reservations" if exists. + cmdStr := cmd.String() + cmdStr = strings.TrimSuffix(cmdStr, ",") + cmdStr += "]'" + + resp, err := d.dcliClient.RunDCLICommand(cmdStr) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + timeout := time.Now().Add(60 * time.Second) + for time.Now().Before(timeout) { + zList, err := d.GetZonesBoundWithSupervisor(supervisorID) + if err != nil { + return fmt.Errorf("failed to get zones bound with supervisor %s: %w", supervisorID, err) + } + + for _, z := range zList.Zones { + if z.Zone == zoneID && z.Status == "READY" { + return nil + } + } + + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("zone %s is not ready after update", zoneID) +} + +func (d *wcpDcliClient) DeleteNamespace(namespace string) error { + cmd := fmt.Sprintf("%s delete --namespace %s", + namespacesInstances, + namespace, + ) + _, err := d.dcliClient.RunDCLICommand(cmd) + + return err +} + +func (d *wcpDcliClient) GetVirtualMachine(vmMoid string) (VirtualMachineDetails, error) { + cmd := fmt.Sprintf("%s get --vm %s +formatter json", virtualmachine, vmMoid) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return VirtualMachineDetails{}, DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + retVal := VirtualMachineDetails{} + if err = json.Unmarshal(resp, &retVal); err != nil { + return VirtualMachineDetails{}, err + } + + return retVal, err +} + +type StorageSpec struct { + // This is the ID of the storage policy to use. + Policy string `json:"policy"` + Limit int64 `json:"limit,omitempty"` +} + +type ZoneSpec struct { + Name string `json:"name"` +} + +type ZoneBindingSpecs struct { + Zone string `json:"zone"` + Type string `json:"type"` +} + +func (d *wcpDcliClient) SetNamespaceStorageSpecs(namespace string, specs []StorageSpec) error { + storageSpecBuffer, err := json.Marshal(specs) + Expect(err).NotTo(HaveOccurred()) + + storageSpecString := string(storageSpecBuffer) + cmd := fmt.Sprintf("%s update --namespace %s --storage-specs '%s'", + namespacesInstances, + namespace, + storageSpecString, + ) + _, err = d.dcliClient.RunDCLICommand(cmd) + + return err +} + +var svcInstallTimeout = 120 * time.Second +var svcInstallInterval = 20 * time.Second + +// CreateOrSetClusterSupervisorService creates or sets the Supervisor Service Version in the cluster. +// The function retries on failures until timeout. Service installation is performed by wcpsvc, which +// depends on resources asynchronously reconciled by appplatform-operator. Retries handle the transient +// failures that occur before those resources become ready. +func (d *wcpDcliClient) CreateOrSetClusterSupervisorService(op DcliOperationType, cluster, serviceID, version string, serviceConfig map[string]string) error { + cmd := fmt.Sprintf("%s %s --cluster %s --supervisor-service %s --version %s", + namespaceManagementClusterServices, op.String(), cluster, serviceID, version) + + if serviceConfig != nil { + configJSON, err := json.Marshal(serviceConfig) + if err != nil { + return fmt.Errorf("error converting service_config to json %w", err) + } + + cmd = fmt.Sprintf("%s --service-config '%s'", cmd, string(configJSON)) + } + + var ( + resp []byte + lastErr error + ) + + timeout := time.Now().Add(svcInstallTimeout) + for time.Now().Before(timeout) { + resp, lastErr = d.dcliClient.RunDCLICommand(cmd) + if lastErr != nil { + // Continue retrying on error + time.Sleep(svcInstallInterval) + continue + } + // Success, stop retrying + return nil + } + + if lastErr != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: lastErr, + } + } + + return nil +} + +// CreateOrSetClusterSupervisorServiceWithYamlConfig creates or sets the Supervisor Service Version in the cluster with YAML Config. +// The function retries on failures until timeout. Service installation is performed by wcpsvc, which +// depends on resources asynchronously reconciled by appplatform-operator. Retries handle the transient +// failures that occur before those resources become ready. +func (d *wcpDcliClient) CreateOrSetClusterSupervisorServiceWithYamlConfig(op DcliOperationType, cluster, serviceID, version, yamlServiceConfig string) error { + cmd := fmt.Sprintf("%s %s --cluster %s --supervisor-service %s --version %s --yaml-service-config %s", + namespaceManagementClusterServices, op.String(), cluster, serviceID, version, yamlServiceConfig) + + var ( + resp []byte + lastErr error + ) + + timeout := time.Now().Add(svcInstallTimeout) + for time.Now().Before(timeout) { + resp, lastErr = d.dcliClient.RunDCLICommand(cmd) + if lastErr != nil { + // Continue retrying on error + time.Sleep(svcInstallInterval) + continue + } + // Success, stop retrying + return nil + } + + if lastErr != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: lastErr, + } + } + + return nil +} + +// EnableV1SupervisorService enables PSP supervisor service in the cluster using v1 "set" API. +func (d *wcpDcliClient) EnableV1SupervisorService(cluster, serviceID string, version *string, serviceConfig map[string]string) error { + return setV1SupervisorService(true, cluster, serviceID, version, serviceConfig, d.dcliClient) +} + +// DisableV1SupervisorService disables PSP supervisor service in the cluster using v1 "set" API. +func (d *wcpDcliClient) DisableV1SupervisorService(cluster, serviceID string, version *string, serviceConfig map[string]string) error { + return setV1SupervisorService(false, cluster, serviceID, version, serviceConfig, d.dcliClient) +} + +// setV1SupervisorService enables or disables PSP supervisor service. +func setV1SupervisorService(enabled bool, cluster, serviceID string, version *string, serviceConfig map[string]string, dcliClient dcli.DCLICommandRunner) error { + cmd := fmt.Sprintf("%s %s set --enabled %t --cluster %s --service-id %s", + namespaceServices, showUnreleased, enabled, cluster, serviceID) // the v1 "set" API is an internal API, thus requires +show + + // v1alpha1 PSP services are *known* to only include a single version, so the version parameter can be omitted. + if version != nil { + cmd = fmt.Sprintf("%s --version '%s'", cmd, *version) + } + + if serviceConfig != nil { + configJSON, err := json.Marshal(serviceConfig) + if err != nil { + return fmt.Errorf("error converting service_config to json %w", err) + } + + cmd = fmt.Sprintf("%s --service-config '%s'", cmd, string(configJSON)) + } + + resp, err := dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// DeleteClusterSupervisorService deletes the Supervisor Service from the cluster. +func (d *wcpDcliClient) DeleteClusterSupervisorService(cluster, serviceID string) error { + cmd := fmt.Sprintf("%s delete --cluster %s --supervisor-service %s", + namespaceManagementClusterServices, cluster, serviceID) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// DeleteSupervisorService removes a Service. +func (d *wcpDcliClient) DeleteSupervisorService(serviceID string) error { + cmd := fmt.Sprintf("%s delete --supervisor-service %s", namespaceManagementServices, serviceID) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// DeleteSupervisorServiceVersion deletes a specific version of a Supervisor Service. +func (d *wcpDcliClient) DeleteSupervisorServiceVersion(serviceID, version string) error { + cmd := fmt.Sprintf("%s versions delete --supervisor-service %s --version %s", namespaceManagementServices, serviceID, version) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) ActivateSupervisorService(serviceID string) error { + cmd := fmt.Sprintf("%s activate --supervisor-service %s", namespaceManagementServices, serviceID) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DeactivateSupervisorService(serviceID string) error { + cmd := fmt.Sprintf("%s deactivate --supervisor-service %s", namespaceManagementServices, serviceID) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) ActivateSupervisorServiceVersion(serviceID, versionID string) error { + cmd := fmt.Sprintf("%s versions activate --supervisor-service %s --version %s", namespaceManagementServices, serviceID, versionID) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DeactivateSupervisorServiceVersion(serviceID, versionID string) error { + cmd := fmt.Sprintf("%s versions deactivate --supervisor-service %s --version %s", namespaceManagementServices, serviceID, versionID) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// RegisterCarvelService creates a Carvel service with one version. +func (d *wcpDcliClient) RegisterCarvelService(carvelYaml []byte) error { + content := b64.StdEncoding.EncodeToString(carvelYaml) + cmd := fmt.Sprintf("%s create --carvel-spec-version-spec-content %s "+ + " +formatter json", namespaceManagementServices, content) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// RegisterCarvelServiceVersion creates a Carvel service version. +func (d *wcpDcliClient) RegisterCarvelServiceVersion(serviceID string, carvelYaml []byte) error { + content := b64.StdEncoding.EncodeToString(carvelYaml) + cmd := fmt.Sprintf("%s versions create --supervisor-service %s "+ + " --carvel-spec-content %s "+ + " +formatter json ", namespaceManagementServices, serviceID, content) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// Precheck initiates a pre-check task for a Supervisor Service version against a Supervisor. +func (d *wcpDcliClient) ServicePrecheck(serviceID, version, supervisor string) error { + cmd := fmt.Sprintf("%s precheck --supervisor %s --supervisor-service %s --target-version %s"+ + " +formatter json ", namespaceMgmtSupervisorsSvServices, supervisor, serviceID, version) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// NewClientUsingKubeconfig uses a kubeconfig pointed at a supervisor cluster context with an +// +// administrator user to return a WCP client that points at the VC corresponding to that +// supervisor cluster. +func NewClientUsingKubeconfig(ctx context.Context, kubeconfig string) WorkloadManagementAPI { + return NewClientUsingKubeconfigFile(ctx, kubeconfig) +} + +func NewClientUsingKubeconfigFile(ctx context.Context, path string) WorkloadManagementAPI { + vCenterHostname := vcenter.GetVCPNIDFromKubeconfigFile(ctx, path) + Expect(vCenterHostname).NotTo(Equal(""), "Unable to determine VC PNID") + + // TODO (GCM-3023) Figure out how to get these from E2E test config. + // Hardcoding is a short term fix until then. + vCenterAdminUser := testbed.AdminUsername + vCenterAdminPassword := testbed.AdminPassword + + wcpClient, err := NewWCPAPIClient(vCenterHostname, vCenterAdminUser, vCenterAdminPassword, testbed.RootUsername, testbed.RootPassword) + Expect(err).NotTo(HaveOccurred()) + + return wcpClient +} + +// NewClientUsingKubeconfigWithCredentials uses a kubeconfig pointed at a supervisor cluster context with an +// +// administrator user to return a WCP client that points at the VC corresponding to that +// supervisor cluster. +func NewClientUsingKubeconfigWithCredentials(ctx context.Context, kubeconfig string, user string, password string) WorkloadManagementAPI { + return NewClientUsingKubeconfigFileWithCredentials(ctx, kubeconfig, user, password) +} + +func NewClientUsingKubeconfigFileWithCredentials(ctx context.Context, path string, sshUser string, sshPassword string) WorkloadManagementAPI { + vCenterHostname := vcenter.GetVCPNIDFromKubeconfigFile(ctx, path) + Expect(vCenterHostname).NotTo(Equal(""), "Unable to determine VC PNID") + wcpClient, err := NewWCPAPIClient(vCenterHostname, testbed.AdminUsername, testbed.AdminPassword, sshUser, sshPassword) + Expect(err).NotTo(HaveOccurred()) + + return wcpClient +} + +func generateVirtualDevicesCommand(devices VirtualDevices) string { + var result strings.Builder + if devices.VGPUDevices != nil { + result.WriteString(" --devices-vgpu-devices '[") + for index, val := range devices.VGPUDevices { + _, _ = fmt.Fprintf(&result, `{"profile_name":%q}`, val.ProfileName) + if index != len(devices.VGPUDevices)-1 { + result.WriteString(",") + } + } + + result.WriteString("]' ") + } + + if devices.DynamicDirectPathIODevices != nil { + result.WriteString(" --devices-dynamic-direct-path-io-devices '[") + for index, val := range devices.DynamicDirectPathIODevices { + _, _ = fmt.Fprintf( + &result, + `{"vendor_id":%d,"device_id":%d,"custom_label":%q}`, + val.VendorID, val.DeviceID, val.CustomLabel, + ) + if index != len(devices.DynamicDirectPathIODevices)-1 { + result.WriteString(",") + } + } + + result.WriteString("]' ") + } + + return result.String() +} + +func generateInstanceStorageCommand(instanceStorage InstanceStorage) (string, error) { + var result string + + if len(instanceStorage.Volumes) > 0 && instanceStorage.StoragePolicy != "" { + volumes, err := json.Marshal(instanceStorage.Volumes) + if err != nil { + return "", err + } + + result = fmt.Sprintf(" --instance-storage-volumes '%s' --instance-storage-policy \"%s\"", + string(volumes), instanceStorage.StoragePolicy) + } + + return result, nil +} + +// GenerateVMClassSpecCmd is a helper function to create cmd for VMClass spec fields. +func GenerateVMClassSpecCmd(vmClassSpec VMClassSpec) (string, error) { + var cmd string + if vmClassSpec.CPUCount != nil { + cmd += fmt.Sprintf(" --cpu-count %d", *vmClassSpec.CPUCount) + } + + if vmClassSpec.MemoryMB != nil { + cmd += fmt.Sprintf(" --memory-mb %d", *vmClassSpec.MemoryMB) + } + + if vmClassSpec.CPUReservation != nil { + cmd += fmt.Sprintf(" --cpu-reservation %d", *vmClassSpec.CPUReservation) + } + + if vmClassSpec.MemoryReservation != nil { + cmd += fmt.Sprintf(" --memory-reservation %d", *vmClassSpec.MemoryReservation) + } + + if vmClassSpec.Description != nil { + cmd += fmt.Sprintf(" --description \"%s\"", *vmClassSpec.Description) + } + + if vmClassSpec.ConfigSpec != nil { + var w bytes.Buffer + + enc := types.NewJSONEncoder(&w) + + err := enc.Encode(vmClassSpec.ConfigSpec) + if err != nil { + return "", err + } + + cmd += fmt.Sprintf( + ` --config-spec '%s'`, + strings.TrimSuffix(w.String(), "\n"), + ) + } + + cmd += generateVirtualDevicesCommand(vmClassSpec.Devices) + + commandIS, err := generateInstanceStorageCommand(vmClassSpec.InstanceStorage) + if err != nil { + return "", err + } + + cmd += commandIS + + return cmd, nil +} + +func (d *wcpDcliClient) GetVMClassInfo(vmClass string) (VMClassInfo, error) { + operation := fmt.Sprintf("get --vm-class %s", vmClass) + s := []string{namespaceManagementVMClassServices, operation, jsonFormatter} + cmd := strings.Join(s, " ") + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return VMClassInfo{}, DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + dec := types.NewJSONDecoder(bytes.NewReader(resp)) + + var obj VMClassInfo + if err := dec.Decode(&obj); err != nil { + return VMClassInfo{}, err + } + + return obj, nil +} + +func (d *wcpDcliClient) ListContentLibraries() ([]string, error) { + operation := "list" + s := []string{dcliContentPrefix, operation, jsonFormatter} + cmd := strings.Join(s, " ") + + var retval []string + + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retval) + + return retval, err +} + +func (d *wcpDcliClient) GetContentLibrary(id string) (ContentLibraryInfo, error) { + cmd := fmt.Sprintf("%s %s get --library-id \"%s\" %s", showUnreleased, dcliContentPrefix, id, jsonFormatter) + + retVal := ContentLibraryInfo{} + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +func (d *wcpDcliClient) FetchContentLibraryIDByName(name string, libraries []string) (string, error) { + operation := "get" + for _, libraryID := range libraries { + s := []string{dcliContentPrefix, operation, "--library-id", libraryID, jsonFormatter} + cmd := strings.Join(s, " ") + retVal := ContentLibraryInfo{} + + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + if err == nil { + if retVal.Name == name { + return libraryID, nil + } + } else { + return "", err + } + } + + return "", nil +} + +func (d *wcpDcliClient) ListVMClasses() ([]VMClassInfo, error) { + operation := "list" + s := []string{namespaceManagementVMClassServices, operation, jsonFormatter} + cmd := strings.Join(s, " ") + + var retVal []VMClassInfo + + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +func (d *wcpDcliClient) CreateVMClass(createSpec VMClassSpec) error { + // CPUCount and MemoryMB are compulsory fields for VMClass creation. + Expect(createSpec.CPUCount).NotTo(Equal(BeNil())) + Expect(createSpec.MemoryMB).NotTo(Equal(BeNil())) + operation := fmt.Sprintf("create --id %s", createSpec.ID) + + vmClassSpecCmd, err := GenerateVMClassSpecCmd(createSpec) + if err != nil { + return err + } + + s := []string{namespaceManagementVMClassServices, operation, vmClassSpecCmd, jsonFormatter} + cmd := strings.Join(s, " ") + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) UpdateVMClass(updateSpec VMClassSpec) error { + operation := fmt.Sprintf("update --vm-class %s", updateSpec.ID) + + vmClassSpecCmd, err := GenerateVMClassSpecCmd(updateSpec) + if err != nil { + return err + } + + s := []string{namespaceManagementVMClassServices, operation, vmClassSpecCmd, jsonFormatter} + cmd := strings.Join(s, " ") + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DeleteVMClass(vmClass string) error { + operation := fmt.Sprintf("delete --vm-class %s", vmClass) + s := []string{namespaceManagementVMClassServices, operation, jsonFormatter} + cmd := strings.Join(s, " ") + _, err := d.dcliClient.RunDCLICommand(cmd) + + return err +} + +func (d *wcpDcliClient) UpdateNamespaceVMServiceSpec(namespaceName string, updateSpec NamespaceUpdateVMserviceSpec) error { + var ( + vmClassSpec string + contentLibrariesSpec strings.Builder + ) + + if updateSpec.VMClasses != nil { + for _, vmClass := range *updateSpec.VMClasses { + vmClassSpec += fmt.Sprintf("--vm-service-spec-vm-classes %s ", vmClass) + } + } + + if updateSpec.ContentLibraries != nil { + for _, cl := range *updateSpec.ContentLibraries { + _, _ = fmt.Fprintf(&contentLibrariesSpec, "--vm-service-spec-content-libraries %s ", cl) + } + } + + operation := fmt.Sprintf("update --namespace %s +show-unreleased", namespaceName) + s := []string{namespacesInstances, operation, vmClassSpec, contentLibrariesSpec.String()} + cmd := strings.Join(s, " ") + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) ListDatastores() ([]Datastore, error) { + cmd := fmt.Sprintf("%s %s datastore list", jsonFormatter, dcliVCenterPrefix) + retVal := []Datastore{} + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +func (d *wcpDcliClient) AddCLTrustedCertificate(trustedCertificate string) (string, error) { + cmd := fmt.Sprintf("%s com vmware content trustedcertificates create --cert-text '%s'", jsonFormatter, trustedCertificate) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return strings.TrimSpace(string(resp)), err +} + +func (d *wcpDcliClient) DeleteCLTrustedCertificate(trustedCertificateID string) error { + cmd := fmt.Sprintf("%s com vmware content trustedcertificates delete --certificate %s", jsonFormatter, trustedCertificateID) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) ListCLSecurityPolicies() ([]SecurityPolicyInfo, error) { + cmd := fmt.Sprintf("%s com vmware content securitypolicies list", jsonFormatter) + retVal := []SecurityPolicyInfo{} + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +func (d *wcpDcliClient) CreateLocalContentLibrary(name string, storageBackings StorageBackingInfo) (string, error) { + storageBackingsJSON, err := json.Marshal(storageBackings.StorageBackings) + Expect(err).NotTo(HaveOccurred()) + + cmd := fmt.Sprintf("%s %s create --name \"%s\" --storage-backings '%s'", showUnreleased, dcliContentLocalPrefix, name, storageBackingsJSON) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return strings.TrimSpace(string(resp)), nil +} + +func (d *wcpDcliClient) DeleteLocalContentLibrary(id string) error { + cmd := fmt.Sprintf("%s %s delete --library-id %s", showUnreleased, dcliContentLocalPrefix, id) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DeleteLocalContentLibraryByForce(id string) error { + cmd := fmt.Sprintf("%s %s forcedelete --library-id %s", showUnreleased, dcliContentLocalPrefix, id) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) UpdateContentLibrary(contentLibraryID, description string, securityPolicyID string) error { + cmd := fmt.Sprintf("%s %s update --library-id %s --description \"%s\"", showUnreleased, dcliContentPrefix, contentLibraryID, description) + if securityPolicyID != "" { + cmd = fmt.Sprintf("%s --security-policy-id %s", cmd, securityPolicyID) + } + + if resp, err := d.dcliClient.RunDCLICommand(cmd); err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) UpdateLocalContentLibrary(contentLibraryID string, enablePublishing bool) error { + cmd := fmt.Sprintf("%s %s update --library-id %s --publish-info-published %t", showUnreleased, dcliContentLocalPrefix, contentLibraryID, enablePublishing) + + if resp, err := d.dcliClient.RunDCLICommand(cmd); err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) CreateSubscribedContentLibrary(name, subscriptionURL, thumbprint string, onDemand bool, storageBackings StorageBackingInfo) (string, error) { + storageBackingsJSON, err := json.Marshal(storageBackings.StorageBackings) + if err != nil { + return "", err + } + + cmd := fmt.Sprintf("%s %s create --name \"%s\" --subscription-info-subscription-url \"%s\" --subscription-info-on-demand %t --subscription-info-authentication-method %s --subscription-info-automatic-sync-enabled %t --storage-backings '%s'", + showUnreleased, dcliContentSubscribedPrefix, name, subscriptionURL, onDemand, "NONE", false, storageBackingsJSON) + if thumbprint != "" { + cmd += fmt.Sprintf(" --subscription-info-ssl-thumbprint \"%s\"", thumbprint) + } + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return strings.TrimSpace(string(resp)), nil +} + +func (d *wcpDcliClient) SyncSubscribedContentLibrary(id string) error { + cmd := fmt.Sprintf("%s %s sync --library-id %s", showUnreleased, dcliContentSubscribedPrefix, id) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DeleteSubscribedContentLibrary(id string) error { + cmd := fmt.Sprintf("%s %s delete --library-id %s", showUnreleased, dcliContentSubscribedPrefix, id) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DeleteSubscribedContentLibraryByForce(id string) error { + cmd := fmt.Sprintf("%s %s forcedelete --library-id %s", showUnreleased, dcliContentSubscribedPrefix, id) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) CreateContentLibraryItem(contentLibraryID, name string, itemType ContentLibraryItemType) (string, error) { + cmd := fmt.Sprintf("%s %s item create --library-id %s --name %s --type %s", showUnreleased, dcliContentPrefix, contentLibraryID, name, string(itemType)) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return strings.TrimSpace(string(resp)), nil +} + +func (d *wcpDcliClient) CreateContentLibraryOVFTemplateItemByPull(contentLibraryID, name, ovfFileURL string) (string, error) { + cmd := fmt.Sprintf("%s %s item create --library-id %s --name %s --type %s", showUnreleased, dcliContentPrefix, contentLibraryID, name, "ovf") + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + libraryItemID := strings.TrimSpace(string(resp)) + + // create update session to add OVF template files by URL + updateCmd := fmt.Sprintf("%s %s item updatesession create --library-item-id %s", showUnreleased, dcliContentPrefix, libraryItemID) + + resp, err = d.dcliClient.RunDCLICommand(updateCmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + updateSessionID := strings.TrimSpace(string(resp)) + + probeCmd := fmt.Sprintf("%s %s item updatesession file probe --uri %s +formatter json", showUnreleased, dcliContentPrefix, ovfFileURL) + probeResult := ProbeResult{} + + err = d.dcliClient.RunCommandAndUnmarshalJSONResult(probeCmd, &probeResult) + if err != nil { + return "", DcliError{ + rawResponse: "", + baseErr: err, + } + } + + sslThumbprint := strings.TrimSpace(probeResult.SSLThumbprint) + + fileName := path.Base(ovfFileURL) + fileCmd := fmt.Sprintf("%s %s item updatesession file add --name %s --source-type %s --update-session-id %s --source-endpoint-uri %s --source-endpoint-ssl-certificate-thumbprint %s", + showUnreleased, dcliContentPrefix, fileName, "PULL", updateSessionID, ovfFileURL, sslThumbprint) + + resp, err = d.dcliClient.RunDCLICommand(fileCmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + completeCmd := fmt.Sprintf("%s %s item updatesession complete --update-session-id %s", showUnreleased, dcliContentPrefix, updateSessionID) + + resp, err = d.dcliClient.RunDCLICommand(completeCmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return libraryItemID, nil +} + +func (d *wcpDcliClient) ListContentLibraryItems(contentLibraryID string) ([]string, error) { + cmd := fmt.Sprintf("%s %s item list --library-id %s", showUnreleased, dcliContentPrefix, contentLibraryID) + cmd += " " + jsonFormatter + + var retVal []string + + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +func (d *wcpDcliClient) GetContentLibraryItem(id string) (ContentLibraryItemInfo, error) { + cmd := fmt.Sprintf("%s %s item get --library-item-id %s", showUnreleased, dcliContentPrefix, id) + cmd += " " + jsonFormatter + + retVal := ContentLibraryItemInfo{} + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &retVal) + + return retVal, err +} + +func (d *wcpDcliClient) UpdateContentLibraryItem(id, description string) error { + cmd := fmt.Sprintf("%s %s item update --library-item-id %s --description \"%s\"", showUnreleased, dcliContentPrefix, id, description) + if resp, err := d.dcliClient.RunDCLICommand(cmd); err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DeleteContentLibraryItem(id string) error { + cmd := fmt.Sprintf("%s %s item delete --library-item-id %s", showUnreleased, dcliContentPrefix, id) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) AssociateImageRegistryContentLibrariesToNamespace(namespace string, contentLibraries ...ContentLibrarySpec) error { + if len(contentLibraries) == 0 { + return fmt.Errorf("expected at least one content library") + } + + namespaceDetails, err := d.GetNamespaceV2(namespace) + if err != nil { + return fmt.Errorf("error getting namespace %s: %w", namespace, err) + } + + contentLibraries = append(contentLibraries, namespaceDetails.ContentLibraries...) + + cmd := fmt.Sprintf("%s %s update --namespace %s --content-libraries '[", showUnreleased, namespacesInstances, namespace) + + uniqueCLIDs := make(map[string]struct{}) + for _, cl := range contentLibraries { + if _, ok := uniqueCLIDs[cl.ContentLibrary]; !ok { + uniqueCLIDs[cl.ContentLibrary] = struct{}{} + cmd += fmt.Sprintf("{\"content_library\": \"%s\", \"writable\": %t, \"allow_import\": %t, \"resource_naming_strategy\": \"%s\"},", cl.ContentLibrary, cl.Writable, cl.AllowImport, cl.ResourceNamingStrategy) + } + } + + cmd = cmd[:len(cmd)-1] + cmd += "]'" + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DisassociateImageRegistryContentLibrariesFromNamespace(namespace string, contentLibraryIDs ...string) error { + if len(contentLibraryIDs) == 0 { + return fmt.Errorf("expected at least one content library ID") + } + + namespaceDetails, err := d.GetNamespaceV2(namespace) + if err != nil { + return fmt.Errorf("error getting namespace %s: %w", namespace, err) + } + + // Run update with *all existing content libraries, except the provided ones. + clsToKeep := []ContentLibrarySpec{} + + for _, cl := range namespaceDetails.ContentLibraries { + match := slices.Contains(contentLibraryIDs, cl.ContentLibrary) + + if !match { + clsToKeep = append(clsToKeep, cl) + } + } + + cmd := fmt.Sprintf("%s %s update --namespace %s --content-libraries '[", showUnreleased, namespacesInstances, namespace) + + if len(clsToKeep) > 0 { + for _, cl := range clsToKeep { + cmd += fmt.Sprintf("{\"content_library\": \"%s\", \"writable\": %t, \"allow_import\": %t, \"resource_naming_strategy\": \"%s\"},", cl.ContentLibrary, cl.Writable, cl.AllowImport, cl.ResourceNamingStrategy) + } + + cmd = cmd[:len(cmd)-1] + } + + cmd += "]'" + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) AssociateContentLibrariesToCluster(cluster string, contentLibraries ...ClusterContentLibrarySpec) error { + if len(contentLibraries) == 0 { + return fmt.Errorf("expected at least one content library ID") + } + + // Get cluster details to get the existing content libraries, so we ensure we retain them after update. + clusterDetails, err := d.GetCluster(cluster) + if err != nil { + return fmt.Errorf("error getting current content libraries on cluster %s: %w", cluster, err) + } + + // Only retain existing CLs. + for _, clSpec := range clusterDetails.ContentLibraries { + _, err := d.GetContentLibrary(clSpec.ContentLibrary) + if err == nil { + contentLibraries = append(contentLibraries, clSpec) + } + } + + cmd := fmt.Sprintf("%s %s clusters update --cluster %s --content-libraries '[", showUnreleased, namespaceManagement, cluster) + + uniqueCLIDs := make(map[string]struct{}) + for _, cl := range contentLibraries { + // Avoid duplicate content library error in running the dcli command. + if _, ok := uniqueCLIDs[cl.ContentLibrary]; !ok { + uniqueCLIDs[cl.ContentLibrary] = struct{}{} + + supervisorServices := "[]" + if len(cl.SupervisorServices) > 0 { + supervisorServices = fmt.Sprintf("[\"%s\"]", strings.Join(cl.SupervisorServices, "\",\"")) + } + + cmd += fmt.Sprintf("{\"content_library\": \"%s\", \"supervisor_services\": %s, \"resource_naming_strategy\": \"%s\"},", cl.ContentLibrary, supervisorServices, cl.ResourceNamingStrategy) + } + } + + cmd = cmd[:len(cmd)-1] + cmd += "]'" + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DisassociateContentLibrariesFromCluster(cluster string, contentLibraryIDs ...string) error { + if len(contentLibraryIDs) == 0 { + return fmt.Errorf("expected at least one content library ID") + } + + // Get cluster details to get the existing content libraries, so we ensure we retain them after update. + clusterDetails, err := d.GetCluster(cluster) + if err != nil { + return fmt.Errorf("error getting current content libraries on cluster %s: %w", cluster, err) + } + + // Run update with all existing content libraries, except the provided ones. + var clsToKeep []ClusterContentLibrarySpec + + for _, clSpec := range clusterDetails.ContentLibraries { + match := false + + for _, delID := range contentLibraryIDs { + if clSpec.ContentLibrary == delID { + match = true + } + } + + if !match { + // Make sure that the ContentLibrary *exists* to update with. + _, err := d.GetContentLibrary(clSpec.ContentLibrary) + if err == nil { + clsToKeep = append(clsToKeep, clSpec) + } + } + } + + // Run an update, omitting the passed in contentLibraryIDs, but keeping other added content libraries + // (to preserve state away from tests). + cmd := fmt.Sprintf("%s %s clusters update --cluster %s --content-libraries '[", showUnreleased, namespaceManagement, cluster) + + if len(clsToKeep) > 0 { + for _, cl := range clsToKeep { + supervisorServices := "[]" + if len(cl.SupervisorServices) > 0 { + supervisorServices = fmt.Sprintf("[\"%s\"]", strings.Join(cl.SupervisorServices, "\",\"")) + } + + cmd += fmt.Sprintf("{\"content_library\": \"%s\", \"supervisor_services\": %s, \"resource_naming_strategy\": \"%s\"},", cl.ContentLibrary, supervisorServices, cl.ResourceNamingStrategy) + } + + cmd = cmd[:len(cmd)-1] + } + + cmd += "]'" + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) RegisterVM(namespace, vmMoID string) (string, error) { + cmd := fmt.Sprintf("%s %s registervm --namespace %s --vm %s", showUnreleased, namespacesInstances, namespace, vmMoID) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return strings.TrimSpace(string(resp)), nil +} + +func (d *wcpDcliClient) UpdateClusterProxyConfig(clusterMoid string, proxyConfig ClusterProxyConfig) error { + flagMap := map[string]string{ + "cluster-proxy-config-http-proxy-config": proxyConfig.HTTPProxyConfig, + "cluster-proxy-config-https-proxy-config": proxyConfig.HTTPSProxyConfig, + "cluster-proxy-config-no-proxy-config": strings.Join(proxyConfig.NoProxyConfig, ","), + "cluster-proxy-config-tls-root-ca-bundle": proxyConfig.TLSRootCABundle, + } + + var cmd strings.Builder + _, _ = fmt.Fprintf(&cmd, "%s clusters update --cluster %v --cluster-proxy-config-proxy-settings-source %v", namespaceManagement, clusterMoid, proxyConfig.ProxySettingsSource) + + switch proxyConfig.ProxySettingsSource { + case VcInherited: + // For VcInherited, we don't need to pass any other flags + case ClusterConfigured: + // The only time we need to pass flags is when the source is ClusterConfigured + for flag, value := range flagMap { + // Filter out empty proxies, they cause an error in dcli + if value != "" { + flagStr := fmt.Sprintf(" --%s '%s'", flag, value) + cmd.WriteString(flagStr) + } + } + case None: + // For None, we don't need to pass any other flags + default: + return fmt.Errorf("invalid proxy source: %s", proxyConfig.ProxySettingsSource) + } + + if resp, err := d.dcliClient.RunDCLICommand(cmd.String()); err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) GetClusterProxyConfig(clusterMoid string) (ClusterProxyConfig, error) { + details, err := d.GetCluster(clusterMoid) + if err != nil { + return ClusterProxyConfig{}, err + } + + return details.ClusterProxyConfig, nil +} + +func (d *wcpDcliClient) CreateContainerImageRegistry(supervisorID string, registry ContainerImageRegistry) (ContainerImageRegistryInfo, error) { + cmd := fmt.Sprintf("%s containerimageregistries create --name '%v' --supervisor '%v' ", namespaceManagementSupervisors, registry.Name, supervisorID) + + if registry.ImageRegistry.CertificateChain != "" { + cmd += fmt.Sprintf("--image-registry-certificate-chain '%v' ", registry.ImageRegistry.CertificateChain) + } + + if registry.ImageRegistry.Password != "" { + cmd += fmt.Sprintf("--image-registry-password '%v' ", registry.ImageRegistry.Password) + } + + if registry.ImageRegistry.Username != "" { + cmd += fmt.Sprintf("--image-registry-username '%v' ", registry.ImageRegistry.Username) + } + + if registry.ImageRegistry.Port != 0 { + cmd += fmt.Sprintf("--image-registry-port %v ", registry.ImageRegistry.Port) + } + + cmd += fmt.Sprintf("--image-registry-hostname '%v' ", registry.ImageRegistry.Hostname) + cmd += fmt.Sprintf("--default-registry '%v' ", registry.DefaultRegistry) + + cmd += jsonFormatter + + result := ContainerImageRegistryInfo{} + + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &result) + if err != nil { + return result, DcliError{ + rawResponse: "", + baseErr: err, + } + } + + return result, nil +} + +func (d *wcpDcliClient) GetContainerImageRegistry(supervisorID, registryName string) (ContainerImageRegistryInfo, error) { + registries, err := d.listContainerImageRegistries(supervisorID) + if err != nil { + return ContainerImageRegistryInfo{}, err + } + + for _, registry := range registries { + if registry.Name == registryName { + return registry, nil + } + } + + return ContainerImageRegistryInfo{}, fmt.Errorf("image registry with name %s not found", registryName) +} + +func (d *wcpDcliClient) listContainerImageRegistries(supervisorID string) ([]ContainerImageRegistryInfo, error) { + cmd := fmt.Sprintf("%s containerimageregistries list --supervisor '%v' +formatter json", namespaceManagementSupervisors, supervisorID) + + result := []ContainerImageRegistryInfo{} + + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &result) + if err != nil { + return result, DcliError{ + rawResponse: "", + baseErr: err, + } + } + + return result, nil +} + +func (d *wcpDcliClient) DeleteContainerImageRegistry(supervisorID, registryID string) error { + cmd := fmt.Sprintf("%s containerimageregistries delete --container-image-registry '%v' --supervisor '%v' ", namespaceManagementSupervisors, registryID, supervisorID) + if resp, err := d.dcliClient.RunDCLICommand(cmd); err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) CreateKeyProvider(provider string) error { + cmd := fmt.Sprintf("%s kms providers create --provider %s", cryptoManager, provider) + if resp, err := d.dcliClient.RunDCLICommand(cmd); err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) DeleteKeyProvider(provider string) error { + cmd := fmt.Sprintf("%s kms providers delete --provider %s", cryptoManager, provider) + if resp, err := d.dcliClient.RunDCLICommand(cmd); err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// AssignLicenseEntitlement assigns the given license entitlement to the cluster using the dcli cis command. +func (d *wcpDcliClient) AssignLicenseEntitlement(signedEntitlement string) (string, error) { + cmd := fmt.Sprintf("%s %s entitlements update-task --other-vc-usages '[]' --configuration '%s'", showUnreleased, dcliCISLicenseEntitlementPrefix, signedEntitlement) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return strings.TrimSpace(string(resp)), nil +} + +func (d *wcpDcliClient) GetSupervisorLoadBalancerProvider(supervisorID string) (string, error) { + cmd := fmt.Sprintf("%s --supervisor %s %s", dcliSupervisorNetworkEdges, supervisorID, jsonFormatter) + + var result struct { + Edges []struct { + Provider string `json:"provider"` + } `json:"edges"` + } + + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &result) + if err != nil { + return "", DcliError{ + rawResponse: "", + baseErr: fmt.Errorf("failed to run DCLI command or parse result: %w", err), + } + } + + if len(result.Edges) == 0 || result.Edges[0].Provider == "" { + return "", fmt.Errorf("no provider found in command output") + } + + return result.Edges[0].Provider, nil +} + +func (d *wcpDcliClient) UpdateWorkerDNS(clusterMoid string, dnsServerIPs ...string) error { + if len(dnsServerIPs) == 0 { + // nothing to update + return nil + } + + var dnsServerIPArgs strings.Builder + + for _, dnsServerIP := range dnsServerIPs { + _, _ = fmt.Fprintf(&dnsServerIPArgs, " --worker-dns %s", dnsServerIP) + } + + cmd := fmt.Sprintf("%s clusters update --cluster %s%s", namespaceManagement, clusterMoid, dnsServerIPArgs.String()) + + if resp, err := d.dcliClient.RunDCLICommand(cmd); err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +func (d *wcpDcliClient) GetWorkerDNS(clusterMoid string) ([]string, error) { + cmd := fmt.Sprintf("%s clusters get --cluster %s %s", namespaceManagement, clusterMoid, jsonFormatter) + + var result struct { + WorkerDNS []string `json:"worker_DNS"` + } + + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &result) + if err != nil { + return nil, DcliError{ + rawResponse: "", + baseErr: fmt.Errorf("failed to run DCLI command or parse result: %w", err), + } + } + + return result.WorkerDNS, nil +} + +// CreateNamespaceNetwork creates a new vSphere network for use in WCP namespaces. +// If the network already exists, this function returns nil (idempotent operation). +func (d *wcpDcliClient) CreateNamespaceNetwork(config NamespaceNetworkConfig) error { + cmd := fmt.Sprintf( + "%s create "+ + "--cluster %s "+ + "--network %s "+ + "--network-provider %s "+ + "--vsphere-network-mode %s "+ + "--vsphere-network-ip-assignment-mode %s "+ + "--vsphere-network-portgroup %s "+ + "--vsphere-network-gateway '%s' "+ + "--vsphere-network-subnet-mask '%s' %s", + namespaceManagementNetworks, + config.Cluster, + config.Network, + config.NetworkProvider, + config.VsphereNetworkMode, + config.VsphereNetworkIPAssignmentMode, + config.VsphereNetworkPortgroup, + config.VsphereNetworkGateway, + config.VsphereNetworkSubnetMask, + jsonFormatter, + ) + + stdout, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + // If the network already exists, treat it as success (idempotent operation) + if strings.Contains(string(stdout), "errors.AlreadyExists") { + return nil + } + + return err + } + + return nil +} + +// GetNamespaceNetwork retrieves details about a vSphere network configured for WCP. +func (d *wcpDcliClient) GetNamespaceNetwork(cluster, network string) (map[string]any, error) { + cmd := fmt.Sprintf( + "%s get --cluster %s --network %s %s", + namespaceManagementNetworks, + cluster, + network, + jsonFormatter, + ) + + var result map[string]any + + err := d.dcliClient.RunCommandAndUnmarshalJSONResult(cmd, &result) + + return result, err +} + +// UpdateNamespaceWithNetworks adds one or more networks to an existing namespace. +func (d *wcpDcliClient) UpdateNamespaceWithNetworks(namespaceName, networkProvider, networkToAdd string) error { + cmd := fmt.Sprintf( + "%s --namespace %s "+ + "--network-spec-network-provider %s "+ + "--network-spec-vsphere-network-config-networks-to-add %s %s", + namespacesInstancesUpdate, + namespaceName, + networkProvider, + networkToAdd, + jsonFormatter, + ) + + _, err := d.dcliClient.RunDCLICommand(cmd) + + return err +} + +// CreateTagCategory creates a new tag category with the given name and description. +func (d *wcpDcliClient) CreateTagCategory(name, description string) (string, error) { + cmd := fmt.Sprintf("com vmware cis tagging category create --cardinality MULTIPLE --name %s --description %s", name, description) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return strings.TrimSpace(string(resp)), nil +} + +// CreateTag creates a new tag with the given name, description and category ID. +func (d *wcpDcliClient) CreateTag(name, description, categoryID string) (string, error) { + cmd := fmt.Sprintf("com vmware cis tagging tag create --name %s --description %s --category-id %s", name, description, categoryID) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return strings.TrimSpace(string(resp)), nil +} + +// AssignTagsToHost assigns multiple tags to a host. +func (d *wcpDcliClient) AssignTagsToHost(tagIDs []string, hostID string) error { + if len(tagIDs) == 0 { + return nil + } + + var cmd strings.Builder + cmd.WriteString("com vmware cis tagging tagassociation attachmultipletagstoobject") + + for _, tagID := range tagIDs { + fmt.Fprintf(&cmd, " --tag-ids %s", tagID) + } + + fmt.Fprintf(&cmd, " --id %s --type HostSystem", hostID) + + resp, err := d.dcliClient.RunDCLICommand(cmd.String()) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// CreateComputePolicy creates a compute policy with the given spec and returns the created compute policy ID. +func (d *wcpDcliClient) CreateComputePolicy(spec ComputePolicySpec) (string, error) { + cmd := fmt.Sprintf("%s %s compute policies create --capability %s --name %s --description \"%s\" --host-tag %s --vm-tag %s", + dcliVCenterPrefix, + showUnreleased, + spec.Capability, + spec.Name, + spec.Description, + spec.HostTagID, + spec.VMTagID, + ) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return "", DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return strings.TrimSpace(string(resp)), nil +} + +// CreateInfraPolicy creates an infrastructure policy with the given spec. +func (d *wcpDcliClient) CreateInfraPolicy(spec InfraPolicySpec) error { + var cmd strings.Builder + fmt.Fprintf(&cmd, "%s %s infrastructurepolicies create --policy %s --description \"%s\" --compute-policy-id %s ", + namespaceManagement, + showUnreleased, + spec.Name, + spec.Description, + spec.ComputePolicyID) + + if spec.EnforcementMode != "" { + fmt.Fprintf(&cmd, " --enforcement-mode %s", spec.EnforcementMode) + } + + if spec.MatchGuestIDValue != "" { + fmt.Fprintf(&cmd, " --match-workload-guest-guest-id-value %s", spec.MatchGuestIDValue) + } + + if len(spec.MatchWorkloadLabel) > 0 { + var labels []string + + for key, value := range spec.MatchWorkloadLabel { + if value == "" { + labels = append(labels, fmt.Sprintf(`{"key":"%s","operator":"EXISTS"}`, key)) + } else { + labels = append(labels, fmt.Sprintf(`{"key":"%s","operator":"IS_IN","values":["%s"]}`, key, value)) + } + } + + fmt.Fprintf(&cmd, " --match-workload-labels '[%s]'", strings.Join(labels, ",")) + } + + resp, err := d.dcliClient.RunDCLICommand(cmd.String()) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// UpdateNamespaceWithInfraPolicies updates a namespace with the given infrastructure policies. +func (d *wcpDcliClient) UpdateNamespaceWithInfraPolicies(namespace string, policyNames ...string) error { + if len(policyNames) == 0 { + return nil + } + + var cmd strings.Builder + fmt.Fprintf(&cmd, "%s %s update --namespace %s", namespacesInstances, showUnreleased, namespace) + + for _, policyName := range policyNames { + fmt.Fprintf(&cmd, " --infrastructure-policies %s", policyName) + } + + resp, err := d.dcliClient.RunDCLICommand(cmd.String()) + if err != nil { + return DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + return nil +} + +// ListHostIDs lists all host IDs in the vCenter. +func (d *wcpDcliClient) ListHostIDs() ([]string, error) { + cmd := fmt.Sprintf("%s %s host list +formatter json", dcliVCenterPrefix, showUnreleased) + + resp, err := d.dcliClient.RunDCLICommand(cmd) + if err != nil { + return nil, DcliError{ + rawResponse: string(resp), + baseErr: err, + } + } + + var hosts []struct { + Host string `json:"host"` + } + if err = json.Unmarshal(resp, &hosts); err != nil { + return nil, err + } + + hostIDs := make([]string, len(hosts)) + for i, h := range hosts { + hostIDs[i] = h.Host + } + + return hostIDs, nil +} diff --git a/test/e2e/infrastructure/vsphere/wcp/iaas_policies.go b/test/e2e/infrastructure/vsphere/wcp/iaas_policies.go new file mode 100644 index 000000000..a253d49cc --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/iaas_policies.go @@ -0,0 +1,35 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +type ComputePolicyCapability string + +const ( + ComputePolicyCapabilityVMHostAffinity ComputePolicyCapability = "com.vmware.vcenter.compute.policies.capabilities.vm_host_affinity" + ComputePolicyCapabilityVMHostAntiAffinity ComputePolicyCapability = "com.vmware.vcenter.compute.policies.capabilities.vm_host_anti_affinity" +) + +type ComputePolicySpec struct { + Name string + Description string + HostTagID string + VMTagID string + Capability ComputePolicyCapability +} + +type InfraPolicyEnforcementMode string + +const ( + InfraPolicyEnforcementModeMandatory InfraPolicyEnforcementMode = "MANDATORY" + InfraPolicyEnforcementModeOptional InfraPolicyEnforcementMode = "OPTIONAL" +) + +type InfraPolicySpec struct { + Name string + Description string + ComputePolicyID string + MatchGuestIDValue string + MatchWorkloadLabel map[string]string + EnforcementMode InfraPolicyEnforcementMode +} diff --git a/test/e2e/infrastructure/vsphere/wcp/namespaces.go b/test/e2e/infrastructure/vsphere/wcp/namespaces.go new file mode 100644 index 000000000..a991e1187 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/namespaces.go @@ -0,0 +1,340 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +import ( + "context" + "errors" + "strings" + "time" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" +) + +type ContentLibrarySpec struct { + ContentLibrary string `json:"content_library"` + Writable bool `json:"writable"` + AllowImport bool `json:"allow_import"` + ResourceNamingStrategy string `json:"resource_naming_strategy"` +} + +type VMServiceSpecDetails struct { + VMClasses []string `json:"vm_classes"` + ContentLibraries []string `json:"content_libraries"` +} + +type VPCNetworkInfo struct { + VPCPath string `json:"vpc_path"` + VPCSharedSubnetPath string `json:"vpc_shared_subnet_path"` + DefaultSubnetSize int64 `json:"default_subnet_size,omitempty"` + SupervisorID string `json:"supervisor_id"` +} + +type NameSpaceNetworkInfo struct { + VDSNetwork string `json:"vds_network,omitempty"` + VPCNetwork *VPCNetworkInfo `json:"vpc_network,omitempty"` +} + +func NewVMServiceSpecDetails(vmClasses []string, contentLibraries []string) VMServiceSpecDetails { + return VMServiceSpecDetails{ + VMClasses: vmClasses, + ContentLibraries: contentLibraries, + } +} + +type NamespaceDetails struct { + ClusterMoID string `json:"cluster"` + Name string `json:"namespace"` + ConfigStatus string `json:"config_status"` + VMServiceSpec VMServiceSpecDetails `json:"vm_service_spec"` + VMStorageSpec []StorageSpec `json:"storage_specs"` + ContentLibraries []ContentLibrarySpec `json:"content_libraries"` +} + +type NamespaceGetInput struct { + NamespaceName string + Kubeconfig string + WCPClient WorkloadManagementAPI + Client ctrlclient.Client +} + +type NamespaceCreateInput struct { + SpecName string + Kubeconfig string + ArtifactFolder string + ClientSet *kubernetes.Clientset + Client ctrlclient.Client + WCPClient WorkloadManagementAPI + StorageClassName string + WorkerStorageClassName string + // Config supplies loaded e2e intervals (wcp.yaml); nil uses built-in defaults for worker StorageClass wait only. + Config framework.ConfigInterface + VMServiceSpec VMServiceSpecDetails + Zone string + SupervisorID string + ReservedVMClassToCount map[string]int + Network *NameSpaceNetworkInfo +} + +type NamespaceDeleteInput struct { + WCPClient WorkloadManagementAPI + Namespace *corev1.Namespace + CancelWatches context.CancelFunc +} + +// NamespaceNetworkCreateInput contains input parameters for creating a vSphere network for WCP. +type NamespaceNetworkCreateInput struct { + Cluster string + NetworkName string + PortGroupKey string // e.g., dvportgroup-90 + Gateway string // can be empty + SubnetMask string + SupervisorID string // i.e., cluster MoID (same as Cluster) + WCPClient WorkloadManagementAPI +} + +func WaitForNamespaceReady(wcpClient WorkloadManagementAPI, namespaceForTest string) { + Eventually(func() bool { + details, err := wcpClient.GetNamespace(namespaceForTest) + if err != nil { + return false + } + // TODO Make this an enum. + return details.ConfigStatus == "RUNNING" + }, 300*time.Second, 5*time.Second).Should(BeTrue(), "Namespace", namespaceForTest, " did not become READY in time") +} + +func GetNamespace(ctx context.Context, input NamespaceGetInput) (*corev1.Namespace, context.CancelFunc) { + var ( + wcpnamespace NamespaceDetails + ns *corev1.Namespace + err error + ) + + wcpClient := input.WCPClient + wcpnamespace, err = wcpClient.GetNamespace(input.NamespaceName) + + Expect(err).NotTo(HaveOccurred(), "get wcp namespace failed from wcp api %q", input.NamespaceName) + + WaitForNamespaceReady(wcpClient, wcpnamespace.Name) + + _, cancelWatches := context.WithCancel(ctx) + + ns, err = GetCorev1Namespace(ctx, input.Client, wcpnamespace.Name) + + Expect(err).NotTo(HaveOccurred(), "get wcp namespace failed from wcp cluster %q", wcpnamespace.Name) + + return ns, cancelWatches +} + +// defaultNamespaceStorageQuotaMiB is the default per-policy quota (MiB) used when associating storage with a WCP namespace. +const defaultNamespaceStorageQuotaMiB = int64(1024 * 500) + +// EnsureWorkerKubernetesStorageClassIfMissing duplicates the primary WCP storage policy on vSphere when the +// worker-named StorageClass is missing from the supervisor, then waits for WCPSVC to sync the StorageClass. +// Wait/poll intervals use built-in defaults (NamespaceCreateInput.Config drives config when creating a namespace). +// +// When namespace and wcpClient are set, the worker storage policy is also associated with that supervisor namespace +// (via SetNamespaceStorageSpecs) if it is not already present. Use this for pre-provisioned namespaces that were +// created before the worker StorageClass existed. +func EnsureWorkerKubernetesStorageClassIfMissing( + ctx context.Context, + kubeconfigPath string, + client kubernetes.Interface, + primaryStorageClass *storagev1.StorageClass, + workerStorageClassName string, + cfg framework.ConfigInterface, + namespace string, + wcpClient WorkloadManagementAPI) *storagev1.StorageClass { + if workerStorageClassName == "" { + return nil + } + + sc, err := client.StorageV1().StorageClasses().Get(ctx, workerStorageClassName, metav1.GetOptions{}) + if err == nil { + Expect(sc.Parameters).NotTo(BeNil()) + policyID, ok := sc.Parameters["storagePolicyID"] + Expect(ok).To(BeTrue(), "worker storage class must expose storagePolicyID for namespace association") + ensureNamespaceAssociatesWorkerStoragePolicyByID(wcpClient, namespace, policyID) + + return sc + } + + if !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } + + Expect(primaryStorageClass.Parameters).NotTo(BeNil()) + basePolicyID, ok := primaryStorageClass.Parameters["storagePolicyID"] + Expect(ok).To(BeTrue(), "primary storage class must expose storagePolicyID for cloning worker policy") + Expect(basePolicyID).NotTo(BeEmpty()) + + vimClient := vcenter.NewVimClientFromKubeconfig(ctx, kubeconfigPath) + defer vcenter.LogoutVimClient(vimClient) + + workerPolicyID, createErr := vcenter.GetOrCreateWorkerStoragePolicy(ctx, vimClient, workerStorageClassName, basePolicyID) + Expect(createErr).NotTo(HaveOccurred(), "failed to create or resolve worker storage policy %q", workerStorageClassName) + ensureNamespaceAssociatesWorkerStoragePolicyByID(wcpClient, namespace, workerPolicyID) + + sc = waitForWorkerStorageClass(ctx, client, workerStorageClassName, cfg) + + return sc +} + +// ensureNamespaceAssociatesWorkerStoragePolicyByID appends the worker policy to the namespace storage +// specs when missing. The policy ID must be passed directly so this can be called before the +// Kubernetes StorageClass object exists (WCPSVC only syncs the StorageClass after the association). +func ensureNamespaceAssociatesWorkerStoragePolicyByID( + wcpClient WorkloadManagementAPI, + namespace string, + workerPolicyID string) { + if namespace == "" || wcpClient == nil || workerPolicyID == "" { + return + } + + details, err := wcpClient.GetNamespace(namespace) + Expect(err).NotTo(HaveOccurred(), "get wcp namespace failed for storage policy sync %q", namespace) + + for _, s := range details.VMStorageSpec { + if s.Policy == workerPolicyID { + return + } + } + + limit := defaultNamespaceStorageQuotaMiB + if len(details.VMStorageSpec) > 0 && details.VMStorageSpec[0].Limit > 0 { + limit = details.VMStorageSpec[0].Limit + } + + merged := append(append([]StorageSpec(nil), details.VMStorageSpec...), + StorageSpec{Policy: workerPolicyID, Limit: limit}) + Expect(wcpClient.SetNamespaceStorageSpecs(namespace, merged)).NotTo(HaveOccurred(), + "failed to associate worker storage policy with namespace %q", namespace) + WaitForNamespaceReady(wcpClient, namespace) +} + +// waitForWorkerStorageClass polls until the StorageClass appears after vSphere policy creation (WCPSVC sync). +func waitForWorkerStorageClass(ctx context.Context, client kubernetes.Interface, name string, cfg framework.ConfigInterface) *storagev1.StorageClass { + var sc *storagev1.StorageClass + + Eventually(func() error { + var err error + + sc, err = client.StorageV1().StorageClasses().Get(ctx, name, metav1.GetOptions{}) + + return err + }, cfg.GetIntervals("default", "wait-storage-class-ready")...).Should(Succeed(), + "timed out waiting for StorageClass %q after ensuring worker storage policy on vCenter", name) + Expect(sc).NotTo(BeNil()) + + return sc +} + +// CreateNamespace creates a WCP kubernetes namespace object using DCLI client. +func CreateNamespace(ctx context.Context, input NamespaceCreateInput) (*corev1.Namespace, context.CancelFunc) { + wcpClient := input.WCPClient + svKubeConfig := input.Kubeconfig + // Create a new cluster, either because we were asked to, or because no running cluster was found. + vCenterHostname := vcenter.GetVCPNIDFromKubeconfig(ctx, svKubeConfig) + Expect(vCenterHostname).NotTo(BeEmpty(), "Unable to determine VC PNID") + + clusterID := vcenter.GetClusterMoIDFromKubeconfig(ctx, svKubeConfig) + Expect(clusterID).NotTo(BeEmpty(), "Unable to determine cluster MoID") + + // Check the supervisor storage class, get the storage policy from it. + // Avoid name based lookups as the VC storage policy name can be transformed when it's being synced to the supervisor cluster. + svClientSet := input.ClientSet + storageClass, err := svClientSet.StorageV1().StorageClasses().Get(ctx, input.StorageClassName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(storageClass).NotTo(BeNil()) + Expect(storageClass.Parameters).NotTo(BeNil()) + policyID, ok := storageClass.Parameters["storagePolicyID"] + Expect(ok).To(BeTrue(), "supervisor storage class must have a corresponding storage policy ID") + + storageSpec := []StorageSpec{{Policy: policyID, Limit: defaultNamespaceStorageQuotaMiB}} + + namespaceForTest := input.SpecName + + network := input.Network + switch { + case network != nil: + err = wcpClient.CreateNamespaceWithNetwork(clusterID, namespaceForTest, storageSpec, input.VMServiceSpec, network) + case len(input.ReservedVMClassToCount) > 0: + err = wcpClient.CreateNamespaceWithVMReservation(namespaceForTest, input.Zone, input.SupervisorID, storageSpec, input.VMServiceSpec, input.ReservedVMClassToCount) + default: + err = wcpClient.CreateNamespaceWithSpecs(clusterID, namespaceForTest, storageSpec, input.VMServiceSpec) + } + + Expect(err).NotTo(HaveOccurred()) + WaitForNamespaceReady(wcpClient, namespaceForTest) + + if input.WorkerStorageClassName != "" { + workerStorageClass := EnsureWorkerKubernetesStorageClassIfMissing(ctx, input.Kubeconfig, svClientSet, + storageClass, input.WorkerStorageClassName, input.Config, namespaceForTest, wcpClient) + + Expect(workerStorageClass).NotTo(BeNil()) + Expect(workerStorageClass.Parameters).NotTo(BeNil()) + _, ok = workerStorageClass.Parameters["storagePolicyID"] + Expect(ok).To(BeTrue(), "supervisor storage class must have a corresponding storage policy ID") + } + + _, cancelWatches := context.WithCancel(ctx) + ns, err := GetCorev1Namespace(ctx, input.Client, namespaceForTest) + Expect(err).NotTo(HaveOccurred()) + + return ns, cancelWatches +} + +func DeleteNamespace(input NamespaceDeleteInput) { + if input.CancelWatches != nil { + input.CancelWatches() + } + + if input.Namespace == nil || input.Namespace.Name == "" { + return + } + + Expect(input.WCPClient).NotTo(BeNil()) + Expect(input.WCPClient.DeleteNamespace(input.Namespace.Name)).NotTo(HaveOccurred()) +} + +// TODO: Timeout time can be reduced after this issue is fixed. +// https://bugzilla.eng.vmware.com/show_bug.cgi?id=3432165 +func WaitForNamespaceDeleted(wcpClient WorkloadManagementAPI, namespaceForTest string) { + Eventually(func() bool { + _, err := wcpClient.GetNamespace(namespaceForTest) + + var dcliErr DcliError + if err != nil && errors.As(err, &dcliErr) { + return strings.Contains(dcliErr.Response(), "com.vmware.vapi.std.errors.NotFound") + } + + return false + }, 6*time.Minute, 30*time.Second).Should(BeTrue(), "Namespace %s did not get DELETED in time", namespaceForTest) +} + +func GetCorev1Namespace(ctx context.Context, client ctrlclient.Client, name string) (*corev1.Namespace, error) { + ns := &corev1.Namespace{} + + key := types.NamespacedName{ + Name: name, + } + + err := client.Get(ctx, key, ns) + if err != nil { + return nil, err + } + + return ns, nil +} diff --git a/test/e2e/infrastructure/vsphere/wcp/privateregistry.go b/test/e2e/infrastructure/vsphere/wcp/privateregistry.go new file mode 100644 index 000000000..b40586e97 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/privateregistry.go @@ -0,0 +1,87 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +import ( + "context" + + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" +) + +type PrivateRegistryInput struct { + Kubeconfig string + WCPClient WorkloadManagementAPI + RegistryName string + Hostname string + Port int + Username string + Password string + CertificateChain string + DefaultRegistry bool +} + +func CreatePrivateRegistry(ctx context.Context, input PrivateRegistryInput) ContainerImageRegistryInfo { + wcpClient := input.WCPClient + svKubeConfig := input.Kubeconfig + + // Get supervisor ID from kubeconfig + supervisorID := vcenter.GetSupervisorIDFromKubeconfig(ctx, svKubeConfig) + Expect(supervisorID).NotTo(BeEmpty(), "Unable to determine supervisor ID") + + // Delete existing registry if it exists + existingRegistry, err := wcpClient.GetContainerImageRegistry(supervisorID, input.RegistryName) + if err == nil { + // Registry exists, delete it first + err = wcpClient.DeleteContainerImageRegistry(supervisorID, existingRegistry.ID) + Expect(err).NotTo(HaveOccurred(), "Failed to delete existing container image registry %s", input.RegistryName) + } + + registry := ContainerImageRegistry{ + Name: input.RegistryName, + DefaultRegistry: input.DefaultRegistry, + ImageRegistry: ImageRegistry{ + Hostname: input.Hostname, + Port: input.Port, + Username: input.Username, + Password: input.Password, + CertificateChain: input.CertificateChain, + }, + } + + registryInfo, err := wcpClient.CreateContainerImageRegistry(supervisorID, registry) + Expect(err).NotTo(HaveOccurred(), "Failed to create container image registry %s", input.RegistryName) + + return registryInfo +} + +func GetPrivateRegistry(ctx context.Context, input PrivateRegistryInput) ContainerImageRegistryInfo { + wcpClient := input.WCPClient + svKubeConfig := input.Kubeconfig + + // Get supervisor ID from kubeconfig + supervisorID := vcenter.GetSupervisorIDFromKubeconfig(ctx, svKubeConfig) + Expect(supervisorID).NotTo(BeEmpty(), "Unable to determine supervisor ID") + + registryInfo, err := wcpClient.GetContainerImageRegistry(supervisorID, input.RegistryName) + Expect(err).NotTo(HaveOccurred(), "Failed to get container image registry %s", input.RegistryName) + + return registryInfo +} + +func DeletePrivateRegistry(ctx context.Context, input PrivateRegistryInput) error { + wcpClient := input.WCPClient + svKubeConfig := input.Kubeconfig + + // Get supervisor ID from kubeconfig + supervisorID := vcenter.GetSupervisorIDFromKubeconfig(ctx, svKubeConfig) + + registryInfo, err := wcpClient.GetContainerImageRegistry(supervisorID, input.RegistryName) + if err != nil { + return err + } + + return wcpClient.DeleteContainerImageRegistry(supervisorID, registryInfo.ID) +} diff --git a/test/e2e/infrastructure/vsphere/wcp/proxy.go b/test/e2e/infrastructure/vsphere/wcp/proxy.go new file mode 100644 index 000000000..644c7c265 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/proxy.go @@ -0,0 +1,23 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +type ClusterProxySource string + +const ( + VcInherited ClusterProxySource = "VC_INHERITED" + ClusterConfigured ClusterProxySource = "CLUSTER_CONFIGURED" + None ClusterProxySource = "NONE" +) + +// ClusterProxyConfig contains the proxy configuration for a cluster. The +// JSON values are retrieved from running dcli commands with the +formatter=json +// option. +type ClusterProxyConfig struct { + TLSRootCABundle string `json:"tls_root_ca_bundle"` + HTTPProxyConfig string `json:"http_proxy_config"` + HTTPSProxyConfig string `json:"https_proxy_config"` + ProxySettingsSource ClusterProxySource `json:"proxy_settings_source"` + NoProxyConfig []string `json:"no_proxy_config"` +} diff --git a/test/e2e/infrastructure/vsphere/wcp/supervisor_cluster.go b/test/e2e/infrastructure/vsphere/wcp/supervisor_cluster.go new file mode 100644 index 000000000..d6a02afec --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/supervisor_cluster.go @@ -0,0 +1,27 @@ +package wcp + +type SupervisorSummaryList struct { + Supervisors []SupervisorSummary `json:"items"` +} + +type SupervisorSummary struct { + Supervisor string `json:"supervisor"` + Info Info `json:"info"` +} + +type Info struct { + Stats Stats `json:"stats"` + KubernetesStatus string `json:"kubernetes_status"` + Name string `json:"name"` + Messages any `json:"messages"` + ConfigStatus string `json:"config_status"` +} + +type Stats struct { + CPUUsed int64 `json:"cpu_used"` + StorageCapacity int64 `json:"storage_capacity"` + MemoryUsed int64 `json:"memory_used"` + CPUCapacity int64 `json:"cpu_capacity"` + MemoryCapacity int64 `json:"memory_capacity"` + StorageUsed int64 `json:"storage_used"` +} diff --git a/test/e2e/infrastructure/vsphere/wcp/tanzu_topology.go b/test/e2e/infrastructure/vsphere/wcp/tanzu_topology.go new file mode 100644 index 000000000..194e06fb6 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/tanzu_topology.go @@ -0,0 +1,24 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +import ( + "context" + + topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func ListZonesByNamespace(ctx context.Context, client ctrlclient.Client, ns string) (*topologyv1.ZoneList, error) { + listOptions := &ctrlclient.ListOptions{Namespace: ns} + + zoneList := &topologyv1.ZoneList{} + + err := client.List(ctx, zoneList, listOptions) + if err != nil { + return nil, err + } + + return zoneList, nil +} diff --git a/test/e2e/infrastructure/vsphere/wcp/vmservice.go b/test/e2e/infrastructure/vsphere/wcp/vmservice.go new file mode 100644 index 000000000..2c6a13318 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/vmservice.go @@ -0,0 +1,78 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +import ( + "github.com/vmware/govmomi/vim25/types" +) + +// VGPUDevice contains the configuration corresponding to a vGPU device. +type VGPUDevice struct { + ProfileName string `json:"profile_name"` +} + +// DynamicDirectPathIODevice contains the configuration corresponding to a Dynamic DirectPath I/O device. +type DynamicDirectPathIODevice struct { + VendorID int `json:"vendor_id"` + DeviceID int `json:"device_id"` + CustomLabel string `json:"custom_label,omitempty"` +} + +// InstanceStorageVolume contains instance volume associated with VirtualMachineClass. +type InstanceStorageVolume struct { + Size int `json:"size"` +} + +// InstanceStorage contains instance storage configurations associated with VirtualMachineClass. +type InstanceStorage struct { + StoragePolicy string `json:"storage_policy,omitempty"` + Volumes []InstanceStorageVolume `json:"volumes,omitempty"` +} + +// VirtualDevices contains information about the virtual devices associated with a VirtualMachineClass. +type VirtualDevices struct { + VGPUDevices []VGPUDevice `json:"vgpu_devices,omitempty"` + DynamicDirectPathIODevices []DynamicDirectPathIODevice `json:"dynamic_direct_path_IO_devices,omitempty"` +} + +// VMClassSpec represents the input structure of a VMClass required to create/update VMClass in VC. +type VMClassSpec struct { + ID string `json:"id"` + CPUCount *int `json:"cpu_count,omitempty"` + MemoryMB *int `json:"memory_mb,omitempty"` + CPUReservation *int `json:"cpu_reservation,omitempty"` + MemoryReservation *int `json:"memory_reservation,omitempty"` + Description *string `json:"description,omitempty"` + Devices VirtualDevices `json:"devices,omitempty"` + InstanceStorage InstanceStorage `json:"instance_storage,omitempty"` + ConfigSpec *types.VirtualMachineConfigSpec `json:"config_spec,omitempty"` +} + +// VMClassInfo represents the output structure. +type VMClassInfo struct { + VMClassSpec + + VMs []string `json:"vms"` + Namespaces []string `json:"namespaces"` + ConfigStatus string `json:"config_status"` +} + +// NamespaceUpdateVMserviceSpec represent structure for only VMService specs required to update namespace instance. +type NamespaceUpdateVMserviceSpec struct { + VMClasses *[]string `json:"vmClasses,omitempty"` + ContentLibraries *[]string `json:"contentLibraries,omitempty"` +} + +type DeviceBacking struct { + VMDKFile string `json:"vmdk_file"` +} + +type DeviceInfo struct { + Backing DeviceBacking `json:"backing"` + Capacity int64 `json:"capacity"` +} + +type VirtualMachineDetails struct { + Disks map[string]DeviceInfo `json:"disks"` +} diff --git a/test/e2e/infrastructure/vsphere/wcp/vmservice_test.go b/test/e2e/infrastructure/vsphere/wcp/vmservice_test.go new file mode 100644 index 000000000..03c715ab4 --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/vmservice_test.go @@ -0,0 +1,566 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package wcp + +import ( + "strings" + "testing" + + "github.com/vmware/govmomi/vim25/types" +) + +func TestGetVMClassInfo(t *testing.T) { + dec := types.NewJSONDecoder(strings.NewReader(getVMClassJSON)) + + var obj VMClassInfo + + err := dec.Decode(&obj) + if err != nil { + t.Error(err) + } + + if obj.ConfigSpec == nil { + t.Errorf("VM Class %s has an empty ConfigSpec", obj.ID) + } + + if e, a := 1, len(obj.ConfigSpec.ExtraConfig); e != a { + t.Errorf("should be %d extra config options: %d", e, a) + } +} + +func TestListVMClasses(t *testing.T) { + dec := types.NewJSONDecoder(strings.NewReader(listVMClassesJSON)) + + var obj []VMClassInfo + + err := dec.Decode(&obj) + if err != nil { + t.Error(err) + } + + if e, a := 20, len(obj); e != a { + t.Errorf("should be %d vm classes: %d", e, a) + } + + for _, obj := range obj { + if obj.ConfigSpec == nil { + t.Errorf("VM Class %s has an empty ConfigSpec", obj.ID) + } + + if obj.ID == "e2etest-vmclass-config-gpuddpio" { + if e, a := 1, len(obj.ConfigSpec.ExtraConfig); e != a { + t.Errorf("should be %d extra config options: %d", e, a) + } + } + } +} + +const getVMClassJSON = `{ + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": 100, + "config_spec": { + "extraConfig": [ + { + "_typeName": "OptionValue", + "value": { + "_typeName": "string", + "_value": "hello-test-value" + }, + "key": "hello-test-key" + } + ], + "_typeName": "VirtualMachineConfigSpec", + "deviceChange": [ + { + "_typeName": "VirtualDeviceConfigSpec", + "device": { + "backing": { + "vgpu": "mockup-vmiop", + "_typeName": "VirtualPCIPassthroughVmiopBackingInfo" + }, + "_typeName": "VirtualPCIPassthrough", + "key": -20000 + }, + "operation": "add" + }, + { + "_typeName": "VirtualDeviceConfigSpec", + "device": { + "backing": { + "_typeName": "VirtualPCIPassthroughDynamicBackingInfo", + "customLabel": "SampleLabel2", + "deviceName": "", + "allowedDevice": [ + { + "_typeName": "VirtualPCIPassthroughAllowedDevice", + "vendorId": 52, + "deviceId": 53 + } + ] + }, + "_typeName": "VirtualPCIPassthrough", + "key": -30000 + }, + "operation": "add" + } + ] + }, + "memory_MB": 4096, + "messages": [], + "id": "e2etest-vmclass-config-gpuddpio", + "memory_reservation": 100, + "vms": [], + "namespaces": [] +}` + +const listVMClassesJSON = `[ + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 8, + "cpu_reservation": null, + "config_spec": { + "memoryMB": 65536, + "numCPUs": 8, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 65536, + "messages": [], + "id": "best-effort-2xlarge", + "memory_reservation": null, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 32, + "cpu_reservation": 100, + "config_spec": { + "memoryMB": 131072, + "numCPUs": 32, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 131072, + "messages": [], + "id": "guaranteed-8xlarge", + "memory_reservation": 100, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": null, + "config_spec": { + "memoryMB": 64, + "numCPUs": 2, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 64, + "messages": [], + "id": "test-vm-class-1674599315", + "memory_reservation": null, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": null, + "config_spec": { + "memoryMB": 8192, + "numCPUs": 2, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 8192, + "messages": [], + "id": "best-effort-medium", + "memory_reservation": null, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 4, + "cpu_reservation": null, + "config_spec": { + "memoryMB": 16384, + "numCPUs": 4, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 16384, + "messages": [], + "id": "best-effort-large", + "memory_reservation": null, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 4, + "cpu_reservation": 100, + "config_spec": { + "memoryMB": 16384, + "numCPUs": 4, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 16384, + "messages": [], + "id": "guaranteed-large", + "memory_reservation": 100, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 4, + "cpu_reservation": null, + "config_spec": { + "memoryMB": 32768, + "numCPUs": 4, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 32768, + "messages": [], + "id": "best-effort-xlarge", + "memory_reservation": null, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": null, + "config_spec": { + "memoryMB": 64, + "numCPUs": 2, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 64, + "messages": [], + "id": "test-vm-class-1674599917", + "memory_reservation": null, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 16, + "cpu_reservation": 100, + "config_spec": { + "memoryMB": 131072, + "numCPUs": 16, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 131072, + "messages": [], + "id": "guaranteed-4xlarge", + "memory_reservation": 100, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 32, + "cpu_reservation": null, + "config_spec": { + "memoryMB": 131072, + "numCPUs": 32, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 131072, + "messages": [], + "id": "best-effort-8xlarge", + "memory_reservation": null, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": null, + "config_spec": { + "_typeName": "VirtualMachineConfigSpec", + "deviceChange": [ + { + "_typeName": "VirtualDeviceConfigSpec", + "device": { + "_typeName": "VirtualE1000", + "key": -10000 + }, + "operation": "add" + } + ] + }, + "memory_MB": 4096, + "messages": [], + "id": "e2etest-vmclass-config-e1000", + "memory_reservation": null, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": 100, + "config_spec": { + "memoryMB": 2048, + "numCPUs": 2, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 2048, + "messages": [], + "id": "guaranteed-xsmall", + "memory_reservation": 100, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": 100, + "config_spec": { + "memoryMB": 8192, + "numCPUs": 2, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 8192, + "messages": [], + "id": "guaranteed-medium", + "memory_reservation": 100, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 4, + "cpu_reservation": 100, + "config_spec": { + "memoryMB": 32768, + "numCPUs": 4, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 32768, + "messages": [], + "id": "guaranteed-xlarge", + "memory_reservation": 100, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 8, + "cpu_reservation": 100, + "config_spec": { + "memoryMB": 65536, + "numCPUs": 8, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 65536, + "messages": [], + "id": "guaranteed-2xlarge", + "memory_reservation": 100, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": 100, + "config_spec": { + "memoryMB": 4096, + "numCPUs": 2, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 4096, + "messages": [], + "id": "guaranteed-small", + "memory_reservation": 100, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 16, + "cpu_reservation": null, + "config_spec": { + "memoryMB": 131072, + "numCPUs": 16, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 131072, + "messages": [], + "id": "best-effort-4xlarge", + "memory_reservation": null, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": 100, + "config_spec": { + "extraConfig": [ + { + "_typeName": "OptionValue", + "value": { + "_typeName": "string", + "_value": "hello-test-value" + }, + "key": "hello-test-key" + } + ], + "_typeName": "VirtualMachineConfigSpec", + "deviceChange": [ + { + "_typeName": "VirtualDeviceConfigSpec", + "device": { + "backing": { + "vgpu": "mockup-vmiop", + "_typeName": "VirtualPCIPassthroughVmiopBackingInfo" + }, + "_typeName": "VirtualPCIPassthrough", + "key": -20000 + }, + "operation": "add" + }, + { + "_typeName": "VirtualDeviceConfigSpec", + "device": { + "backing": { + "_typeName": "VirtualPCIPassthroughDynamicBackingInfo", + "customLabel": "SampleLabel2", + "deviceName": "", + "allowedDevice": [ + { + "_typeName": "VirtualPCIPassthroughAllowedDevice", + "vendorId": 52, + "deviceId": 53 + } + ] + }, + "_typeName": "VirtualPCIPassthrough", + "key": -30000 + }, + "operation": "add" + } + ] + }, + "memory_MB": 4096, + "messages": [], + "id": "e2etest-vmclass-config-gpuddpio", + "memory_reservation": 100, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": null, + "config_spec": { + "memoryMB": 2048, + "numCPUs": 2, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 2048, + "messages": [], + "id": "best-effort-xsmall", + "memory_reservation": null, + "vms": [], + "namespaces": [] + }, + { + "devices": null, + "instance_storage": null, + "description": "", + "config_status": "READY", + "cpu_count": 2, + "cpu_reservation": null, + "config_spec": { + "memoryMB": 4096, + "numCPUs": 2, + "_typeName": "VirtualMachineConfigSpec" + }, + "memory_MB": 4096, + "messages": [], + "id": "best-effort-small", + "memory_reservation": null, + "vms": [ + "vm-9w0uy3\/vm-nbpp", + "vm-w999eu\/vm-ep0y", + "vm-qqizem\/vm-6arf", + "vm-ppdgsb\/vm-5oz9", + "vmsvc-e2e-vmb7sp\/vmsvc-e2e-vmsvc-e2e-vmb7sp-p6t-255r6-2vjdw", + "vmsvc-e2e-vmb7sp\/vmsvc-e2e-vmsvc-e2e-vmb7sp-p6t-worker-tx5gf-79f5f7f65c-dfjx7" + ], + "namespaces": [ + "vm-ppdgsb", + "vm-qqizem", + "packer-fjdbg5", + "vmsvc-e2e-vmb7sp", + "vm-9w0uy3", + "vm-w999eu" + ] + } +]` diff --git a/test/e2e/infrastructure/vsphere/wcp/zone.go b/test/e2e/infrastructure/vsphere/wcp/zone.go new file mode 100644 index 000000000..2acfcd70c --- /dev/null +++ b/test/e2e/infrastructure/vsphere/wcp/zone.go @@ -0,0 +1,242 @@ +package wcp + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "k8s.io/client-go/kubernetes" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +type BindZonesForNamespaceInput struct { + Namespace string + Zones []string + Kubeconfig string + ArtifactFolder string + ClientSet *kubernetes.Clientset + SvClusterClient ctrlclient.Client + WCPClient WorkloadManagementAPI +} + +type ZonesGetInput struct { + SupervisorID string + WCPClient WorkloadManagementAPI +} + +type ZonesBindingInput struct { + SupervisorID string + Zones []string + WCPClient WorkloadManagementAPI +} + +type ZoneList struct { + Zones []ZoneDetails `json:"zones"` +} + +type ZoneDetails struct { + Zone string `json:"zone"` + Type string `json:"type"` + Namespaces []string `json:"namespaces"` + Status string `json:"status"` + Messages []ZoneMessage `json:"messages"` +} + +type ZoneMessage struct { + Type string `json:"type"` + Status string `json:"status"` +} + +type VSphereZoneList struct { + VSphereZones []VSphereZoneItem `json:"items"` +} + +type VSphereZoneItem struct { + Zone string `json:"zone"` + Info VSphereZoneInfo `json:"info"` +} + +type VSphereZoneInfo struct { + Description string `json:"description"` +} + +func WaitUpdateNamespaceWithZonesReady(svClusterClient ctrlclient.Client, testNamespace string, zones []string) { + Eventually(func() bool { + retZoneList, err := ListZonesByNamespace(context.Background(), svClusterClient, testNamespace) + if err != nil { + return false + } + // By default, namespace is already bound with zone-1 + return len(retZoneList.Items) == len(zones)+1 + }, 180*time.Second, 10*time.Second).Should(BeTrue(), "Zones ", zones, "update namespace with Zones not READY in time") +} + +func WaitForZoneRemovalFromNamespace(svClusterClient ctrlclient.Client, testNamespace string, zone string) { + Eventually(func() bool { + retZoneList, err := ListZonesByNamespace(context.Background(), svClusterClient, testNamespace) + if err != nil { + return false + } + + for _, item := range retZoneList.Items { + if item.Name == zone { + // If the zone is found, it hasn't been removed yet + return false + } + } + + // If the zone is not found, it has been successfully removed + return true + }, 1800*time.Second, 20*time.Second).Should(BeTrue(), "Zone ", zone, "delete zone from namespace failed within the timeout") +} + +func WaitUpdateZoneBindingsSupervisorReady(wcpClient WorkloadManagementAPI, supervisorID string, expectedZones int) { + Eventually(func() bool { + details, err := wcpClient.GetZonesBoundWithSupervisor(supervisorID) + if err != nil { + e2eframework.Logf("Failed to get zones bound with supervisor %s due to %v", supervisorID, err) + return false + } + + // Check if the number of zones matches the expectedZones + if len(details.Zones) != expectedZones { + e2eframework.Logf("Expected %d Zones bound with Supervisor %s, actual %v Zones", expectedZones, supervisorID, len(details.Zones)) + return false + } + + supervisorSummaryList, err := wcpClient.ListSupervisorSummary() + if err != nil { + e2eframework.Logf("Failed to list supervisor Summary %v", err) + return false + } + + for _, supervisorSummary := range supervisorSummaryList.Supervisors { + if supervisorSummary.Supervisor == supervisorID { + return supervisorSummary.Info.ConfigStatus == "RUNNING" + } + } + + return false + }, 600*time.Second, 15*time.Second).Should(BeTrue(), "Expected Zones", expectedZones, "update Zone bindings with supervisor is not READY in time") +} + +// UpdateNamespaceWithZones binds provided Zones for a WCP kubernetes namespace object. +func UpdateNamespaceWithZones(ctx context.Context, input BindZonesForNamespaceInput) (*topologyv1.ZoneList, context.CancelFunc) { + testNamespace := input.Namespace + wcpClient := input.WCPClient + svKubeConfig := input.Kubeconfig + // Get supervisor ID + supervisorID := vcenter.GetSupervisorIDFromKubeconfig(ctx, svKubeConfig) + Expect(supervisorID).NotTo(BeEmpty(), "Unable to get supervisor ID") + + zoneSpecs := make([]ZoneSpec, 0, len(input.Zones)) + for _, z := range input.Zones { + zSpec := ZoneSpec{Name: z} + zoneSpecs = append(zoneSpecs, zSpec) + } + + err := wcpClient.UpdateNamespaceWithZones(testNamespace, zoneSpecs) + Expect(err).NotTo(HaveOccurred()) + WaitUpdateNamespaceWithZonesReady(input.SvClusterClient, testNamespace, input.Zones) + + _, cancelWatches := context.WithCancel(ctx) + retZoneList, err := ListZonesByNamespace(ctx, input.SvClusterClient, input.Namespace) + Expect(err).NotTo(HaveOccurred()) + Expect(len(retZoneList.Items)).To(BeNumerically(">=", 2)) + + return retZoneList, cancelWatches +} + +// DeleteZonesFromNamespace deletes specified Zones for a WCP kubernetes namespace object. +func DeleteZonesFromNamespace(ctx context.Context, input BindZonesForNamespaceInput) error { + testNamespace := input.Namespace + wcpClient := input.WCPClient + + for _, zone := range input.Zones { + err := wcpClient.DeleteZoneFromNamespace(testNamespace, zone) + Expect(err).NotTo(HaveOccurred()) + WaitForZoneRemovalFromNamespace(input.SvClusterClient, testNamespace, zone) + } + + return nil +} + +// GetZonesBoundWithSupervisor gets zones bound with supervisor. +func GetZonesBoundWithSupervisor(input ZonesGetInput) (ZoneList, error) { + wcpClient := input.WCPClient + supervisorID := input.SupervisorID + + zoneList, err := wcpClient.GetZonesBoundWithSupervisor(supervisorID) + if err != nil { + return ZoneList{}, err + } + + return zoneList, nil +} + +// CreateZoneBindingsWithSupervisor creates Zone bindings with supervisor. +func CreateZoneBindingsWithSupervisor(input ZonesBindingInput) error { + wcpClient := input.WCPClient + supervisorID := input.SupervisorID + zones := input.Zones + + currentZones, err := wcpClient.GetZonesBoundWithSupervisor(supervisorID) + if err != nil { + return err + } + + specs := make([]ZoneBindingSpecs, 0, len(zones)) + for _, zone := range zones { + spec := ZoneBindingSpecs{Zone: zone, Type: "WORKLOAD"} + specs = append(specs, spec) + } + + if err = wcpClient.CreateZoneBindingsWithSupervisor(supervisorID, specs); err != nil { + return err + } + + WaitUpdateZoneBindingsSupervisorReady(wcpClient, supervisorID, len(currentZones.Zones)+len(zones)) + + return nil +} + +// DeleteZoneBindingsWithSupervisor delete Zone bindings with supervisor. +func DeleteZoneBindingsWithSupervisor(input ZonesBindingInput) error { + wcpClient := input.WCPClient + supervisorID := input.SupervisorID + zones := input.Zones + + currentZones, err := wcpClient.GetZonesBoundWithSupervisor(supervisorID) + if err != nil { + return err + } + // Supervisor don’t allow > 1 zone to be removed at the same time. If one is being removed, need to wait until the + // first is removed and remove another + for i := 0; i < (len(zones)); i++ { + err = wcpClient.DeleteZoneBindingsWithSupervisor(supervisorID, zones[i]) + if err != nil { + return fmt.Errorf("failed to delete zone binding for %s: %w", zones[i], err) + } + + WaitUpdateZoneBindingsSupervisorReady(wcpClient, supervisorID, len(currentZones.Zones)-i-1) + // Sleep for 30 min to wait for the Zone binding is fully deleted even after it is removed from zone binding list. + // TODO: This is a temporary workaround for wcp issue https://bugzilla-vcf.lvn.broadcom.net/show_bug.cgi?id=3493084. + // Remove it when issue is fixed. + time.Sleep(1800 * time.Second) + } + + return nil +} + +func ListVSphereZones(wcpClient WorkloadManagementAPI) (VSphereZoneList, error) { + vSphereZoneList, err := wcpClient.ListVSphereZones() + if err != nil { + return VSphereZoneList{}, err + } + + return vSphereZoneList, nil +} diff --git a/test/e2e/manifestbuilders/configmap.go b/test/e2e/manifestbuilders/configmap.go new file mode 100644 index 000000000..6fa546b9d --- /dev/null +++ b/test/e2e/manifestbuilders/configmap.go @@ -0,0 +1,52 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" +) + +const configMapFixtureBasePath = "test/e2e/fixtures/yaml/vmoperator/configmap" + +type ConfigMap struct { + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` +} + +func GetConfigMapYamlGOSC(configMap ConfigMap) []byte { + configMapYamlIn := fixtures.ReadFile(configMapFixtureBasePath, "configmapgosc.yaml.in") + configMapYaml, _ := ReadConfigMapTemplate(configMap, configMapYamlIn) + + return configMapYaml +} + +func GetConfigMapYamlOvfEnv(configMap ConfigMap) []byte { + configMapYamlIn := fixtures.ReadFile(configMapFixtureBasePath, "configmapOvfEnv.yaml.in") + configMapYaml, _ := ReadConfigMapTemplate(configMap, configMapYamlIn) + + return configMapYaml +} + +func GetConfigMapYamlVAppConfig(configMap ConfigMap) []byte { + configMapYamlIn := fixtures.ReadFile(configMapFixtureBasePath, "configmapvapp.yaml.in") + configMapYaml, _ := ReadConfigMapTemplate(configMap, configMapYamlIn) + // templating cannot be parsed here only to keep its text + dataYaml := fixtures.ReadFileBytes(configMapFixtureBasePath, "vappData.yaml") + + return append(configMapYaml, dataYaml...) +} + +func ReadConfigMapTemplate(configMap ConfigMap, input string) ([]byte, error) { + tmpl := template.Must(template.New("configmap").Parse(input)) + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, configMap) + if err != nil { + e2eframework.Failf("Failed executing configmap template: %v", err) + } + + return parsed.Bytes(), nil +} diff --git a/test/e2e/manifestbuilders/contentsourcebinding.go b/test/e2e/manifestbuilders/contentsourcebinding.go new file mode 100644 index 000000000..10ca01fcf --- /dev/null +++ b/test/e2e/manifestbuilders/contentsourcebinding.go @@ -0,0 +1,40 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" +) + +// Util function to return a ContentSourceBinding yaml from a templatized fixture. +func GetContentSourceBindingYaml(namespace, contentSourceName string) []byte { + test := "test/e2e/fixtures/yaml/vmoperator/contentsources" + classBindingYamlIn := fixtures.ReadFile(test, "contentsourcebindings.yaml.in") + contentSourceBindingYaml, _ := ReadContentSourceBinding(namespace, contentSourceName, classBindingYamlIn) + + return contentSourceBindingYaml +} + +func ReadContentSourceBinding(ns, contentSourceName, input string) ([]byte, error) { + tmpl := template.Must(template.New("contentsourcebinding").Parse(input)) + + config := struct { + Namespace string + Name string + }{ + ns, + contentSourceName, + } + + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, config) + if err != nil { + e2eframework.Failf("Failed executing template: %v", err) + } + + return parsed.Bytes(), nil +} diff --git a/test/e2e/manifestbuilders/encryptionclass.go b/test/e2e/manifestbuilders/encryptionclass.go new file mode 100644 index 000000000..9a040de16 --- /dev/null +++ b/test/e2e/manifestbuilders/encryptionclass.go @@ -0,0 +1,38 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" +) + +type EncryptionClass struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + KeyProvider string `json:"keyProvider"` + KeyID string `json:"keyID,omitempty"` +} + +func GetEncryptionClassYaml(class EncryptionClass) []byte { + input := ` +apiVersion: encryption.vmware.com/v1alpha1 +kind: EncryptionClass +metadata: + namespace: {{.Namespace}} + name: {{.Name}} +spec: + keyProvider: {{.KeyProvider}} + keyID: "{{.KeyID}}" +` + + tmpl := template.Must(template.New("EncryptionClass").Parse(input)) + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, class) + if err != nil { + e2eframework.Failf("Failed executing EncryptionClass template: %v", err) + } + + return parsed.Bytes() +} diff --git a/test/e2e/manifestbuilders/podVM.go b/test/e2e/manifestbuilders/podVM.go new file mode 100644 index 000000000..3f55702ae --- /dev/null +++ b/test/e2e/manifestbuilders/podVM.go @@ -0,0 +1,47 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" +) + +type PodVMConfig struct { + Metadata metadata `yaml:"metadata"` +} + +type metadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` +} + +type PodVMTemplateConfig struct { + Name string + Namespace string + PrivateKeySecretName string `json:"privateKeySecretName,omitempty"` + CustomUserPrivateKeySecretName string `json:"customUserPrivateKeySecretName,omitempty"` + MemoryRequest string `json:"memoryRequest,omitempty"` + MemoryLimit string `json:"memoryLimit,omitempty"` + CPURequest string `json:"cpuRequest,omitempty"` + CPULimit string `json:"cpuLimit,omitempty"` +} + +func BuildPodVMYamlTemplate(inputConfig PodVMTemplateConfig) ([]byte, error) { + podVMYaml := fixtures.ReadFile("test/e2e/fixtures/yaml/podvm", "podvm.yaml.in") + + inputTemplate := template.Must(template.New("podvm").Parse(podVMYaml)) + + parsed := new(bytes.Buffer) + + err := inputTemplate.Execute(parsed, inputConfig) + if err != nil { + return nil, err + } + + e2eframework.Logf("Generated PodVM yaml from template is:\n%s", parsed.String()) + + return parsed.Bytes(), nil +} diff --git a/test/e2e/manifestbuilders/secret.go b/test/e2e/manifestbuilders/secret.go new file mode 100644 index 000000000..16b2257ac --- /dev/null +++ b/test/e2e/manifestbuilders/secret.go @@ -0,0 +1,77 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + secretFilePath = "test/e2e/fixtures/yaml/vmoperator/secret" +) + +type Secret struct { + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` +} + +func GetSecretYamlCloudConfig(secret Secret) []byte { + secretYamlIn := fixtures.ReadFile(secretFilePath, "secretCloudConfig.yaml.in") + secretYaml := ReadSecretTemplate(secret, secretYamlIn) + + return secretYaml +} + +func GetSecretYamlInlineCloudInitData(secret Secret) []byte { + secretYamlIn := fixtures.ReadFile(secretFilePath, "secretInlineCloudInitData.yaml.in") + secretYaml := ReadSecretTemplate(secret, secretYamlIn) + + return secretYaml +} + +func GetSecretYamlInlineSysprepData(secret Secret) []byte { + secretYamlIn := fixtures.ReadFile(secretFilePath, "secretInlineSysprepData.yaml.in") + secretYaml := ReadSecretTemplate(secret, secretYamlIn) + + return secretYaml +} + +func GetSecretYamlOvfEnv(secret Secret) []byte { + secretYamlIn := fixtures.ReadFile(secretFilePath, "secretOvfEnv.yaml.in") + secretYaml := ReadSecretTemplate(secret, secretYamlIn) + + return secretYaml +} + +func GetSecretYamlVAppConfig(secret Secret) []byte { + secretYamlIn := fixtures.ReadFile(secretFilePath, "secretEmpty.yaml.in") + secretYaml := ReadSecretTemplate(secret, secretYamlIn) + // Add stringData in separate to keep its template text as is. + stringDataYaml := fixtures.ReadFileBytes(secretFilePath, "vAppStringData.yaml") + + return append(secretYaml, stringDataYaml...) +} + +func GetSecretYamlSysprepConfig(secret Secret) []byte { + secretYamlIn := fixtures.ReadFile(secretFilePath, "secretEmpty.yaml.in") + secretYaml := ReadSecretTemplate(secret, secretYamlIn) + // Add stringData in separate to keep its template text as is. + stringDataYaml := fixtures.ReadFileBytes(secretFilePath, "sysprepStringData.yaml") + + return append(secretYaml, stringDataYaml...) +} + +func ReadSecretTemplate(secret Secret, input string) []byte { + tmpl := template.Must(template.New("secret").Parse(input)) + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, secret) + if err != nil { + e2eframework.Failf("Failed executing secret template: %v", err) + } + + return parsed.Bytes() +} diff --git a/test/e2e/manifestbuilders/securitypolicy.go b/test/e2e/manifestbuilders/securitypolicy.go new file mode 100644 index 000000000..4fc720a66 --- /dev/null +++ b/test/e2e/manifestbuilders/securitypolicy.go @@ -0,0 +1,39 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" + e2eframework "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + securitypolicyFilePath = "test/e2e/fixtures/yaml/vmoperator/securitypolicy" +) + +type SecurityPolicy struct { + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` +} + +// Util function to return a Namespaced SecurityPolicy yaml from a templatized fixture. +func GetSecurityPolicyYaml(securitypolicy SecurityPolicy) []byte { + securitypolicyYamlIn := fixtures.ReadFile(securitypolicyFilePath, "securitypolicy.yaml.in") + securitypolicyYaml := ReadSecurityPolicy(securitypolicy, securitypolicyYamlIn) + + return securitypolicyYaml +} + +func ReadSecurityPolicy(securitypolicy SecurityPolicy, input string) []byte { + tmpl := template.Must(template.New("securitypolicy").Parse(input)) + + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, securitypolicy) + if err != nil { + e2eframework.Failf("Failed executing securitypolicy template: %v", err) + } + + return parsed.Bytes() +} diff --git a/test/e2e/manifestbuilders/storageclass.go b/test/e2e/manifestbuilders/storageclass.go new file mode 100644 index 000000000..26b4b5886 --- /dev/null +++ b/test/e2e/manifestbuilders/storageclass.go @@ -0,0 +1,19 @@ +package manifestbuilders + +import ( + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" +) + +func GetStorageQuotaYAML() ([]byte, error) { + dir := "test/e2e/fixtures/yaml/vmoperator/storageclass" + yaml := fixtures.ReadFileBytes(dir, "gc-storage-quota.yaml") + + return yaml, nil +} + +func GetStorageClassYAML() ([]byte, error) { + dir := "test/e2e/fixtures/yaml/vmoperator/storageclass" + yaml := fixtures.ReadFileBytes(dir, "gc-storage-profile.yaml") + + return yaml, nil +} diff --git a/test/e2e/manifestbuilders/subnet.go b/test/e2e/manifestbuilders/subnet.go new file mode 100644 index 000000000..534bf7405 --- /dev/null +++ b/test/e2e/manifestbuilders/subnet.go @@ -0,0 +1,71 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" +) + +const ( + subnetFilePath = "test/e2e/fixtures/yaml/vmoperator/subnet" +) + +type SubnetOrSubnetSet struct { + Kind string `json:"kind,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` +} + +// Util function to return a Namespaced Subnet or SubnetSet yaml from a templatized fixture. +func GetSubnetOrSubnetSetYaml(subnet SubnetOrSubnetSet) []byte { + subnetYamlIn := fixtures.ReadFile(subnetFilePath, "subnet.yaml.in") + subnetYaml := ReadSubnet(subnet, subnetYamlIn) + + return subnetYaml +} + +func GetDHCPSubnetOrSubnetSetYaml(subnet SubnetOrSubnetSet, private bool) []byte { + subnetYaml := GetSubnetOrSubnetSetYaml(subnet) + // Add DHCP spec in separate to keep its template text as is. + dhcpYaml := fixtures.ReadFileBytes(subnetFilePath, "subnetDHCP.yaml") + // Determine the access mode YAML configuration. + var accessModeYaml []byte + if private { + accessModeYaml = fixtures.ReadFileBytes(subnetFilePath, "subnetPrivateAccessMode.yaml") + } else { + accessModeYaml = fixtures.ReadFileBytes(subnetFilePath, "subnetPublicAccessMode.yaml") + } + + return append(append(subnetYaml, dhcpYaml...), accessModeYaml...) +} + +func GetCIDRSubnetOrSubnetSetYaml(subnet SubnetOrSubnetSet, private bool) []byte { + subnetYaml := GetSubnetOrSubnetSetYaml(subnet) + // Add CIDR spec in separate to keep its template text as is. + cidrYaml := fixtures.ReadFileBytes(subnetFilePath, "subnetCIDR.yaml") + // Determine the access mode YAML configuration. + var accessModeYaml []byte + if private { + accessModeYaml = fixtures.ReadFileBytes(subnetFilePath, "subnetPrivateAccessMode.yaml") + } else { + accessModeYaml = fixtures.ReadFileBytes(subnetFilePath, "subnetPublicAccessMode.yaml") + } + + return append(append(subnetYaml, cidrYaml...), accessModeYaml...) +} + +func ReadSubnet(subnet SubnetOrSubnetSet, input string) []byte { + tmpl := template.Must(template.New("subnet").Parse(input)) + + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, subnet) + if err != nil { + e2eframework.Failf("Failed executing subnet template: %v", err) + } + + return parsed.Bytes() +} diff --git a/test/e2e/manifestbuilders/template.go b/test/e2e/manifestbuilders/template.go new file mode 100644 index 000000000..bf935b905 --- /dev/null +++ b/test/e2e/manifestbuilders/template.go @@ -0,0 +1,27 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package manifestbuilders + +import ( + "bytes" + "text/template" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" +) + +func GetYaml[T any](yamlBuilder T, path, file, newTemplateName string) []byte { + yamlIn := fixtures.ReadFile(path, file) + tmpl := template.Must(template.New(newTemplateName).Parse(yamlIn)) + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, yamlBuilder) + if err != nil { + e2eframework.Failf("Failed executing %s : %v", newTemplateName, err) + } + + return parsed.Bytes() +} diff --git a/test/e2e/manifestbuilders/virtualmachine.go b/test/e2e/manifestbuilders/virtualmachine.go new file mode 100644 index 000000000..fcc6ea9e6 --- /dev/null +++ b/test/e2e/manifestbuilders/virtualmachine.go @@ -0,0 +1,204 @@ +// Copyright (c) 2020-2024 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package manifestbuilders + +import ( + "bytes" + "text/template" + + corev1 "k8s.io/api/core/v1" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" +) + +const ( + vmYamlDir = "test/e2e/fixtures/yaml/vmoperator/virtualmachines" +) + +type Network struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` +} +type NetworkA2 struct { + Interfaces []InterfaceSpec `json:"interfaces,omitempty"` +} +type InterfaceSpec struct { + Name string `json:"name,omitempty"` + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` +} +type Bootstrap struct { + CloudInit *CloudInit `json:"cloudInit,omitempty"` + Sysprep *Sysprep `json:"sysprep,omitempty"` + VAppConfig *VAppConfig `json:"vAppConfig,omitempty"` + LinuxPrep *LinuxPrep `json:"linuxPrep,omitempty"` +} + +type Cdrom struct { + Name string `json:"name,omitempty"` + ImageName string `json:"imageName,omitempty"` + ImageKind string `json:"imageKind,omitempty"` + Connected bool `json:"connected,omitempty"` + AllowGuestControl bool `json:"allowGuestControl,omitempty"` + ControllerBusNumber *int32 `json:"controllerBusNumber,omitempty"` + ControllerType *vmopv1a5.VirtualControllerType `json:"controllerType,omitempty"` + UnitNumber *int32 `json:"unitNumber,omitempty"` +} + +type CloudInit struct { + RawCloudConfig *KeySelector `json:"rawCloudConfig,omitempty"` + CloudConfig *string `json:"cloudConfig,omitempty"` +} + +type Sysprep struct { + RawSysprep *KeySelector `json:"rawSysprep,omitempty"` + Sysprep *string `json:"sysprep,omitempty"` +} + +type VAppConfig struct { + RawProperties *string `json:"rawProperties,omitempty"` + Properties *[]KeyValueOrSecretKeySelectorPair `json:"properties,omitempty"` +} + +type LinuxPrep struct { + HardwareClockIsUTC bool `json:"hardwareClockIsUTC,omitempty"` + TimeZone string `json:"timeZone,omitempty"` + CustomizeAtNextPowerOn *bool `json:"customizeAtNextPowerOn,omitempty"` +} + +type KeySelector struct { + Key string `json:"key,omitempty"` + Name string `json:"name,omitempty"` +} + +type KeyValueOrSecretKeySelectorPair struct { + Key string `json:"key"` + Value ValueOrSecretKeySelector `json:"value,omitempty"` +} + +// Only have Value for simplicity. +type ValueOrSecretKeySelector struct { + Value string `json:"value,omitempty"` +} + +type Crypto struct { + EncryptionClassName string `json:"encryptionClassName,omitempty"` + UseDefaultKeyProvider bool `json:"useDefaultKeyProvider,omitempty"` +} + +type PVC struct { + VolumeName string `json:"volume_name,omitempty"` + ClaimName string `json:"claim_name,omitempty"` + StorageClassName string `json:"storage_class_name,omitempty"` + RequestSize string `json:"request_size,omitempty"` + Namespace string `json:"namespace,omitempty"` + ControllerBusNumber *int32 `json:"controller_bus_number,omitempty"` + UnitNumber *int32 `json:"unit_number,omitempty"` + SharingMode *string `json:"sharing_mode,omitempty"` + DiskMode *string `json:"disk_mode,omitempty"` + + VolumeMode *corev1.PersistentVolumeMode `json:"volume_mode,omitempty"` + AccessModes []corev1.PersistentVolumeAccessMode `json:"access_modes,omitempty"` + ControllerType *vmopv1a5.VirtualControllerType `json:"controller_type,omitempty"` + ApplicationType vmopv1a5.VolumeApplicationType `json:"application_type,omitempty"` +} + +type VirtualMachineYaml struct { + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + ImageName string `json:"image_name,omitempty"` + VMClassName string `json:"vm_class_name,omitempty"` + StorageClassName string `json:"storage_class_name,omitempty"` + ResourcePolicy string `json:"resource_policy,omitempty"` + Network Network `json:"network,omitempty"` + NetworkA2 NetworkA2 `json:"network_a2,omitempty"` + Transport string `json:"transport,omitempty"` + ConfigMapName string `json:"config_map_name,omitempty"` + SecretName string `json:"secret_name,omitempty"` + PowerState string `json:"power_state,omitempty"` + Bootstrap Bootstrap `json:"bootstrap,omitempty"` + // Deprecated: For v1alpha5, use Hardware.Cdrom instead. + Cdrom []Cdrom `json:"cdrom,omitempty"` + GuestID string `json:"guest_id,omitempty"` + PVCNames []string `json:"pvc_names,omitempty"` + Crypto *Crypto `json:"crypto,omitempty"` + GroupName string `json:"groupName,omitempty"` + CurrentSnapshotName string `json:"currentSnapshotName,omitempty"` + PVCs []PVC `json:"pvcs,omitempty"` + Affinity *vmopv1a5.AffinitySpec `json:"affinity,omitempty"` + Hardware *vmopv1a5.VirtualMachineHardwareSpec `json:"hardware,omitempty"` + Policies []vmopv1a5.PolicySpec `json:"policies,omitempty"` +} + +// GetVirtualMachineYaml returns a v1alpha1 VirtualMachine yaml from a templatized fixture. +func GetVirtualMachineYaml(vmYaml VirtualMachineYaml) []byte { + vmYamlIn := fixtures.ReadFile(vmYamlDir, "singlevm.yaml.in") + vmYamlBytes, _ := ReadVirtualMachineTemplate(vmYaml, vmYamlIn) + + return vmYamlBytes +} + +// GetVirtualMachineYamlA2 returns a v1alpha2 VirtualMachine yaml from a templatized fixture. +func GetVirtualMachineYamlA2(vmYaml VirtualMachineYaml) []byte { + vmYamlIn := fixtures.ReadFile(vmYamlDir, "v1a2singlevm.yaml.in") + vmYamlBytes, _ := ReadVirtualMachineTemplate(vmYaml, vmYamlIn) + + return vmYamlBytes +} + +// GetVirtualMachineWithMultiNetworkYamlA2 returns a v1alpha2 VirtualMachine with multiple network yaml +// from a templatized fixture. +func GetVirtualMachineWithMultiNetworkYamlA2(vmYaml VirtualMachineYaml) []byte { + vmYamlIn := fixtures.ReadFile(vmYamlDir, "v1a2vm-multi-network.yaml.in") + vmYamlBytes, _ := ReadVirtualMachineTemplate(vmYaml, vmYamlIn) + + return vmYamlBytes +} + +// GetVirtualMachineYamlA3 returns a v1alpha3 VirtualMachine YAML from a templated fixture. +func GetVirtualMachineYamlA3(vmYaml VirtualMachineYaml) []byte { + vmYamlIn := fixtures.ReadFile(vmYamlDir, "v1a3singlevm.yaml.in") + vmYamlBytes, _ := ReadVirtualMachineTemplate(vmYaml, vmYamlIn) + + return vmYamlBytes +} + +// GetVirtualMachineYamlA5 returns a v1alpha5 VirtualMachine YAML from a templated fixture. +func GetVirtualMachineYamlA5(vmYaml VirtualMachineYaml) []byte { + vmYamlIn := fixtures.ReadFile(vmYamlDir, "v1a5singlevm.yaml.in") + vmYamlBytes, _ := ReadVirtualMachineTemplate(vmYaml, vmYamlIn) + + return vmYamlBytes +} + +// GetPersistentVolumeClaimYaml renders a single PersistentVolumeClaim manifest from +// the same PVC fields used by GetVirtualMachineYamlA5 (see createPvcsFromSpec). +func GetPersistentVolumeClaimYaml(pvc PVC) []byte { + pvcYamlIn := fixtures.ReadFile(vmYamlDir, "pvc.yaml.in") + tmpl := template.Must(template.New("pvc").Parse(pvcYamlIn)) + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, pvc) + if err != nil { + e2eframework.Failf("Failed executing pvc template: %v", err) + } + + return parsed.Bytes() +} + +func ReadVirtualMachineTemplate(vmYaml VirtualMachineYaml, input string) ([]byte, error) { + tmpl := template.Must(template.New("vm").Parse(input)) + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, vmYaml) + if err != nil { + e2eframework.Failf("Failed executing vm template: %v", err) + } + + return parsed.Bytes(), nil +} diff --git a/test/e2e/manifestbuilders/virtualmachineclass.go b/test/e2e/manifestbuilders/virtualmachineclass.go new file mode 100644 index 000000000..18a9f7cdf --- /dev/null +++ b/test/e2e/manifestbuilders/virtualmachineclass.go @@ -0,0 +1,40 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" +) + +// Util function to return a Namespaced VirtualMachineClass yaml from a templatized fixture. +func GetVirtualMachineClassYaml(namespace, vmClassName string) []byte { + test := "test/e2e/fixtures/yaml/vmoperator/virtualmachineclasses" + classYamlIn := fixtures.ReadFile(test, "vmclass.yaml.in") + vmClassYaml, _ := ReadVirtualMachineClassBinding(namespace, vmClassName, classYamlIn) + + return vmClassYaml +} + +func ReadVirtualMachineClass(ns, vmClassName, input string) ([]byte, error) { + tmpl := template.Must(template.New("vmclass").Parse(input)) + + config := struct { + Namespace string + Name string + }{ + ns, + vmClassName, + } + + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, config) + if err != nil { + e2eframework.Failf("Failed executing template: %v", err) + } + + return parsed.Bytes(), nil +} diff --git a/test/e2e/manifestbuilders/virtualmachineclassbinding.go b/test/e2e/manifestbuilders/virtualmachineclassbinding.go new file mode 100644 index 000000000..7fe4e916e --- /dev/null +++ b/test/e2e/manifestbuilders/virtualmachineclassbinding.go @@ -0,0 +1,40 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" +) + +// Util function to return a VirtualMachineClassBinding yaml from a templatized fixture. +func GetVirtualMachineClassBindingYaml(namespace, vmClassName string) []byte { + test := "test/e2e/fixtures/yaml/vmoperator/virtualmachineclasses" + classBindingYamlIn := fixtures.ReadFile(test, "vmclassbindings.yaml.in") + vmClassBindingYaml, _ := ReadVirtualMachineClassBinding(namespace, vmClassName, classBindingYamlIn) + + return vmClassBindingYaml +} + +func ReadVirtualMachineClassBinding(ns, vmClassName, input string) ([]byte, error) { + tmpl := template.Must(template.New("vmclassbinding").Parse(input)) + + config := struct { + Namespace string + Name string + }{ + ns, + vmClassName, + } + + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, config) + if err != nil { + e2eframework.Failf("Failed executing template: %v", err) + } + + return parsed.Bytes(), nil +} diff --git a/test/e2e/manifestbuilders/virtualmachinesnapshot.go b/test/e2e/manifestbuilders/virtualmachinesnapshot.go new file mode 100644 index 000000000..fd3007769 --- /dev/null +++ b/test/e2e/manifestbuilders/virtualmachinesnapshot.go @@ -0,0 +1,27 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package manifestbuilders + +const ( + virtualMachineSnapshotDir = "test/e2e/fixtures/yaml/vmoperator/virtualmachinesnapshot" +) + +type VirtualMachineSnapshotYaml struct { + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + VMName string `json:"vmName,omitempty"` + Memory bool `json:"memory,omitempty"` + Quiesce string `json:"quiesce,omitempty"` + Description string `json:"description,omitempty"` + ImportedSnapshot bool `json:"importedSnapshot,omitempty"` +} + +func GetVirtualMachineSnapshotYaml(vmSnapshotYaml VirtualMachineSnapshotYaml) []byte { + return GetYaml( + vmSnapshotYaml, + virtualMachineSnapshotDir, + "v1alpha5-vmsnapshot.yaml.in", + "VirtualMachineSnapshot") +} diff --git a/test/e2e/manifestbuilders/vmgroup.go b/test/e2e/manifestbuilders/vmgroup.go new file mode 100644 index 000000000..43778d016 --- /dev/null +++ b/test/e2e/manifestbuilders/vmgroup.go @@ -0,0 +1,42 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package manifestbuilders + +import ( + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" +) + +type VirtualMachineGroupYaml struct { + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + GroupName string `json:"groupName,omitempty"` + PowerState string `json:"powerState,omitempty"` + PowerOffMode string `json:"powerOffMode,omitempty"` + NextForcePowerStateSyncTime string `json:"nextForcePowerStateSyncTime,omitempty"` + Members []vmopv1a5.GroupMember `json:"members,omitempty"` + BootOrder []BootOrder `json:"bootOrder,omitempty"` +} + +// This struct is needed to serialize the bootOrder.PowerOnDelay field correctly as it's a pointer type in VMOP API. +type BootOrder struct { + Members []vmopv1a5.GroupMember `json:"members,omitempty"` + PowerOnDelay string `json:"powerOnDelay,omitempty"` +} + +func GetVirtualMachineGroupYaml(vmGroupYaml VirtualMachineGroupYaml) []byte { + return GetYaml( + vmGroupYaml, + "test/e2e/fixtures/yaml/vmoperator/virtualmachinegroups", + "vm-group.yaml.in", + "VirtualMachineGroup") +} + +func GetVirtualMachineGroupWithBootOrderYaml(vmGroupYaml VirtualMachineGroupYaml) []byte { + return GetYaml( + vmGroupYaml, + "test/e2e/fixtures/yaml/vmoperator/virtualmachinegroups", + "vm-group-with-boot-order.yaml.in", + "VirtualMachineGroup") +} diff --git a/test/e2e/manifestbuilders/vmgrouppublishrequest.go b/test/e2e/manifestbuilders/vmgrouppublishrequest.go new file mode 100644 index 000000000..527d1c60c --- /dev/null +++ b/test/e2e/manifestbuilders/vmgrouppublishrequest.go @@ -0,0 +1,22 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package manifestbuilders + +type VirtualMachineGroupPublishRequestYaml struct { + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Source string `json:"source,omitempty"` + Target string `json:"target,omitempty"` + VirtualMachines []string `json:"virtualMachines,omitempty"` + TTLSecondsAfterFinished int64 `json:"ttlSecondsAfterFinished,omitempty"` +} + +func GetVirtualMachineGroupPublishRequestYaml(vmGroupPubYaml VirtualMachineGroupPublishRequestYaml) []byte { + return GetYaml( + vmGroupPubYaml, + "test/e2e/fixtures/yaml/vmoperator/virtualmachinegrouppublishrequests", + "vm-group-publish.yaml.in", + "VirtualMachineGroupPublishRequest") +} diff --git a/test/e2e/manifestbuilders/vmpublishrequest.go b/test/e2e/manifestbuilders/vmpublishrequest.go new file mode 100644 index 000000000..e12aa753a --- /dev/null +++ b/test/e2e/manifestbuilders/vmpublishrequest.go @@ -0,0 +1,57 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" +) + +type VirtualMachinePublishRequestSource struct { + Name string `json:"name,omitempty"` +} + +type VirtualMachinePublishRequestTarget struct { + Item VirtualMachinePublishRequestTargetItem `json:"item,omitempty"` + Location VirtualMachinePublishRequestTargetLocation `json:"location,omitempty"` +} + +type VirtualMachinePublishRequestTargetItem struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` +} + +type VirtualMachinePublishRequestTargetLocation struct { + Name string `json:"name,omitempty"` +} + +type VirtualMachinePublishRequestYaml struct { + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Source VirtualMachinePublishRequestSource `json:"source,omitempty"` + Target VirtualMachinePublishRequestTarget `json:"target,omitempty"` +} + +func GetVirtualMachinePublishRequestYaml(vmPublishRequestYaml VirtualMachinePublishRequestYaml) []byte { + test := "test/e2e/fixtures/yaml/vmoperator/virtualmachinepublishrequests" + vmPublishRequestYamlIn := fixtures.ReadFile(test, "singlevirtualmachinepublishrequest.yaml.in") + vmPublishRequestYamlBytes, _ := ReadVirtualMachinePublishRequestTemplate(vmPublishRequestYaml, vmPublishRequestYamlIn) + + return vmPublishRequestYamlBytes +} + +func ReadVirtualMachinePublishRequestTemplate(virtualMachinePublishRequestYaml VirtualMachinePublishRequestYaml, input string) ([]byte, error) { + tmpl := template.Must(template.New("vmPublishRequest").Parse(input)) + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, virtualMachinePublishRequestYaml) + if err != nil { + e2eframework.Failf("Failed executing virtualMachinePublishRequestYaml template: %v", err) + } + + return parsed.Bytes(), nil +} diff --git a/test/e2e/manifestbuilders/vmwebconsolerequest.go b/test/e2e/manifestbuilders/vmwebconsolerequest.go new file mode 100644 index 000000000..839c9981e --- /dev/null +++ b/test/e2e/manifestbuilders/vmwebconsolerequest.go @@ -0,0 +1,44 @@ +package manifestbuilders + +import ( + "bytes" + "text/template" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/fixtures" +) + +type VirtualMachineWebConsoleRequestYaml struct { + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + VMName string `json:"virtualMachineName"` +} + +func GetV1A1WebConsoleRequestYaml(vmWebConsoleRequestYaml VirtualMachineWebConsoleRequestYaml) []byte { + test := "test/e2e/fixtures/yaml/vmoperator/virtualmachinewebconsolerequests" + webConsoleRequestYamlIn := fixtures.ReadFile(test, "webconsolerequests.yaml.in") + webConsoleRequestYamlBytes, _ := ReadVirtualMachineWebConsoleRequestTemplate(vmWebConsoleRequestYaml, webConsoleRequestYamlIn) + + return webConsoleRequestYamlBytes +} + +func GetVirtualMachineWebConsoleRequestYaml(vmWebConsoleRequestYaml VirtualMachineWebConsoleRequestYaml) []byte { + test := "test/e2e/fixtures/yaml/vmoperator/virtualmachinewebconsolerequests" + vmWebConsoleRequestYamlIn := fixtures.ReadFile(test, "virtualmachinewebconsolerequests.yaml.in") + vmWebConsoleRequestYamlBytes, _ := ReadVirtualMachineWebConsoleRequestTemplate(vmWebConsoleRequestYaml, vmWebConsoleRequestYamlIn) + + return vmWebConsoleRequestYamlBytes +} + +func ReadVirtualMachineWebConsoleRequestTemplate(virtualMachineWebConsoleRequestYaml VirtualMachineWebConsoleRequestYaml, input string) ([]byte, error) { + tmpl := template.Must(template.New("vmWebConsoleRequest").Parse(input)) + parsed := new(bytes.Buffer) + + err := tmpl.Execute(parsed, virtualMachineWebConsoleRequestYaml) + if err != nil { + e2eframework.Failf("Failed executing virtualMachineWebConsoleRequestYaml template: %v", err) + } + + return parsed.Bytes(), nil +} diff --git a/test/e2e/testutils/auth.go b/test/e2e/testutils/auth.go new file mode 100644 index 000000000..a5486b88d --- /dev/null +++ b/test/e2e/testutils/auth.go @@ -0,0 +1,84 @@ +package testutils + +import ( + "context" + "fmt" + "net/url" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/crypto/ssh" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/kubectl" + libssh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" +) + +func GetHelpersFromKubeconfig(ctx context.Context, kubeconfigPath string) (libssh.SSHCommandRunner, wcp.WorkloadManagementAPI, string) { + vCenterHostname := vcenter.GetVCPNIDFromKubeconfig(ctx, kubeconfigPath) + Expect(vCenterHostname).NotTo(Equal(""), "Unable to determine VC PNID") + + var err error + // This is the port to SSH into the VC with. + vCenterPort := 22 + wcpClient := wcp.NewClientUsingKubeconfigFile(ctx, kubeconfigPath) + + supervisorClusterIPRaw := kubectl.GetKubectlClusterForCurrentContext(ctx, kubeconfigPath) + parsedSupervisorURL, err := url.Parse(supervisorClusterIPRaw) + Expect(err).NotTo(HaveOccurred(), "failed to parse supervisor cluster ip due to %v", err) + + supervisorClusterIP := parsedSupervisorURL.Hostname() + Expect(supervisorClusterIP).NotTo(Equal(""), "Unable to get supervisor cluster host") + + sshCommandRunner, err := libssh.NewSSHCommandRunner(vCenterHostname, vCenterPort, testbed.RootUsername, []ssh.AuthMethod{ssh.Password(testbed.RootPassword)}) + Expect(err).NotTo(HaveOccurred()) + + return sshCommandRunner, wcpClient, supervisorClusterIP +} + +// Helper method to create a user on the VC and login, returning a kubectl plugin handle with a reference the their kubeconfig. +func CreateUserAndLogin(user *vcenter.User, supervisorClusterIP, tanzuKubernetesClusterForTest string, tanzuKubernetesClusterNamespaceForTest string) *kubectl.KubectlPlugin { + By("creating a new user") + vcenter.CreateUserOrFail(user) + + return LoginWithUserWithRetry(user, supervisorClusterIP, tanzuKubernetesClusterForTest, tanzuKubernetesClusterNamespaceForTest) +} + +// Helper method to login with the given user on the VC and login, returning a kubectl plugin handle with a reference the their kubeconfig. +func LoginWithUserWithRetry(user *vcenter.User, supervisorClusterIP, tanzuKubernetesClusterForTest string, tanzuKubernetesClusterNamespaceForTest string) *kubectl.KubectlPlugin { + By("logging in as the user") + + kubectlPlugin := kubectl.NewKubectlPlugin(fmt.Sprintf("%s-kubeconfig", user.Credentials.Username)). + WithUsername(user.Credentials.Username). + WithPassword(user.Credentials.Password). + WithServer(supervisorClusterIP). + WithInsecureFlag(true) + + if tanzuKubernetesClusterForTest != "" { + kubectlPlugin = kubectlPlugin.WithTanzuKubernetesClusterName(tanzuKubernetesClusterForTest) + } + + if tanzuKubernetesClusterNamespaceForTest != "" { + kubectlPlugin = kubectlPlugin.WithTanzuKubernetesClusterNamespace(tanzuKubernetesClusterNamespaceForTest) + } + + Eventually(func() error { + return kubectlPlugin.Login() + }, 5*time.Minute, 10*time.Second).Should(Succeed(), "time out while trying to vsphere login as %s", user.Credentials.Username) + + return kubectlPlugin +} + +func SetUserPermissionsOnNamespace(wcpClient wcp.WorkloadManagementAPI, user *vcenter.User, accessType wcp.AccessType, namespaceForTest string) { + By("granting the new user permissions in the supervisor namespace") + Expect(wcpClient.CreateNamespacePermissions( + wcp.Principal{Type: wcp.UserSubjectType, + Name: user.Credentials.Username, + Domain: "vsphere.local"}, + namespaceForTest, + accessType, + )).To(Succeed()) +} diff --git a/test/e2e/testutils/csi.go b/test/e2e/testutils/csi.go new file mode 100644 index 000000000..86974264d --- /dev/null +++ b/test/e2e/testutils/csi.go @@ -0,0 +1,70 @@ +package testutils + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + e2eframework "k8s.io/kubernetes/test/e2e/framework" + e2epv "k8s.io/kubernetes/test/e2e/framework/pv" +) + +const ( + ExecutedDuration = 5 * time.Minute + PollDuration = 5 * time.Second + StorageAppSelector = "gc-e2e-storage" +) + +// AssertCreatePVC creates a PVC under a namespace with provided storageclass. +func AssertCreatePVC(client kubernetes.Interface, name, namespace, storageClassName string) { + ctx := context.TODO() + // Create the PVC + pvc := ConstructPVC(name, storageClassName) + + Eventually(func() error { + _, err := client.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) + if err != nil { + return err + } + + return nil + }, ExecutedDuration, PollDuration).Should(Succeed()) + + // Validate the PVC is bound + By("Validating the creation of pvc") + + timeouts := e2eframework.NewTimeoutContext() + err := e2epv.WaitForPersistentVolumeClaimPhase(context.Background(), corev1.ClaimBound, client, namespace, name, 2*time.Second, timeouts.ClaimBound) + e2eframework.ExpectNoError(err) +} + +// ConstructPVC returns a PVC object with user-specified pvcname and storage class. +func ConstructPVC(pvcName, storageClassName string) *corev1.PersistentVolumeClaim { + pvc := &corev1.PersistentVolumeClaim{} + pvc.ObjectMeta.Name = pvcName + pvc.ObjectMeta.Annotations = map[string]string{ + "volume.beta.kubernetes.io/storage-class": storageClassName, + } + pvc.ObjectMeta.Labels = map[string]string{ + "app": StorageAppSelector, + "type": "pvc", + } + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + } + pvc.Spec.Resources = corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + "storage": resource.MustParse("100Mi"), + }, + } + pvc.Spec.StorageClassName = &storageClassName + + return pvc +} diff --git a/test/e2e/utils/capabilities.go b/test/e2e/utils/capabilities.go new file mode 100644 index 000000000..67398dbb8 --- /dev/null +++ b/test/e2e/utils/capabilities.go @@ -0,0 +1,216 @@ +// Copyright (c) 2024-2025 Broadcom. All Rights Reserved. + +package utils + +import ( + "context" + "fmt" + "strconv" + "sync" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + clientset "k8s.io/client-go/kubernetes" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + e2essh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" +) + +const ( + UserPodsOnVDS = "User_Pods_On_VDS_Supported" + + SupervisorAsyncUpgradeFSS = "WCP_Supervisor_Async_Upgrade" + SupervisorVMSnapshotFSS = "WCP_VMService_VM_Snapshots" + + APIServerToWebhookAuth = "supports_apiserver_to_webhook_authentication" + + capabilityConfigMapName = "wcp-cluster-capabilities" + capabilityConfigMapNs = "kube-system" + + capabilitiesCRName = "supervisor-capabilities" + capabilitiesCRDName = "capabilities.iaas.vmware.com" +) + +var ( + CapabilityGRV = schema.GroupVersionResource{ + Group: "iaas.vmware.com", + Version: "v1alpha1", + Resource: "capabilities", + } + + clientAuthToWebhookEnabled bool + clientAuthToWebhookEnabledOnce sync.Once +) + +func DoesSupervisorCapabilityExist(ctx context.Context, client clientset.Interface, dynamicClient dynamic.Interface, capabilityKey string, asyncSupervisorFSSEnabled bool) bool { + if !asyncSupervisorFSSEnabled { + _, exists := getWcpClusterCapabilityFromCM(ctx, client, capabilityKey) + return exists + } + + _, exists := getSupervisorCapabilityFromCR(ctx, dynamicClient, capabilityKey) + + return exists +} + +// IsSupervisorCapabilityEnabled returns whether the given capability is enabled on the Supervisor. +// A false will be returned if the given capability doesn't exist. +func IsSupervisorCapabilityEnabled(ctx context.Context, client clientset.Interface, dynamicClient dynamic.Interface, capabilityKey string, asyncSupervisorFSSEnabled bool) bool { + if !asyncSupervisorFSSEnabled { + enabled, _ := getWcpClusterCapabilityFromCM(ctx, client, capabilityKey) + return enabled + } + + enabled, _ := getSupervisorCapabilityFromCR(ctx, dynamicClient, capabilityKey) + + return enabled +} + +// EnableSupervisorCapability enables the capability on the given Supervisor *if the capability exists*. +func EnableSupervisorCapability(ctx context.Context, client clientset.Interface, dynamicClient dynamic.Interface, svSSHCommandRunner e2essh.SSHCommandRunner, capability string, asyncSupervisorFSSEnabled bool) { + setSupervisorCapability(ctx, client, dynamicClient, svSSHCommandRunner, asyncSupervisorFSSEnabled, capability, true) +} + +// DisableSupervisorCapability disables the capability on the given Supervisor *if the capability exists*. +func DisableSupervisorCapability(ctx context.Context, client clientset.Interface, dynamicClient dynamic.Interface, svSSHCommandRunner e2essh.SSHCommandRunner, capability string, asyncSupervisorFSSEnabled bool) { + setSupervisorCapability(ctx, client, dynamicClient, svSSHCommandRunner, asyncSupervisorFSSEnabled, capability, false) +} + +func IsWcpClusterCapabilityEnabled(ctx context.Context, client clientset.Interface, capability string) bool { + enabled, _ := getWcpClusterCapabilityFromCM(ctx, client, capability) + return enabled +} + +func IsClientAuthToWebhookEnabled(ctx context.Context, client clientset.Interface) bool { + clientAuthToWebhookEnabledOnce.Do(func() { + clientAuthToWebhookEnabled = IsWcpClusterCapabilityEnabled(ctx, client, APIServerToWebhookAuth) + }) + + return clientAuthToWebhookEnabled +} + +// CheckSupervisorCapabilitiesCRDSupport is used to check if the supervisor capability CRD exists. +func CheckSupervisorCapabilitiesCRDSupport(ctx context.Context, client ctrlclient.Client) (bool, error) { + crd := &apiextensionsv1.CustomResourceDefinition{} + crd.SetName(capabilitiesCRDName) + + err := client.Get(ctx, ctrlclient.ObjectKey{Name: capabilitiesCRDName}, crd) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + + return false, err + } + + return true, nil +} + +// return whether the capability enabled, whether the capability exists. +func getWcpClusterCapabilityFromCM(ctx context.Context, client clientset.Interface, capabilityKey string) (bool, bool) { + cm := getWcpClusterCapabilitiesCM(ctx, client) + + val, ok := cm.Data[capabilityKey] + if !ok { + return false, false + } + + return val == "true", true +} + +// return whether the capability enabled, whether the capability exists. +func getSupervisorCapabilityFromCR(ctx context.Context, dynamicClient dynamic.Interface, capabilityKey string) (bool, bool) { + cr := getSupervisorCapabilitiesCR(ctx, dynamicClient) + + status, found, err := unstructured.NestedMap(cr.Object, "status", "supervisor", capabilityKey) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("error extracting nested field: %v", err)) + + if !found { + return false, false + } + + return status["activated"].(bool), true +} + +func setSupervisorCapability(ctx context.Context, client clientset.Interface, dynamicClient dynamic.Interface, svSSHCommandRunner e2essh.SSHCommandRunner, asyncSupervisorFSSEnabled bool, capability string, enable bool) { + if !asyncSupervisorFSSEnabled { + setSupervisorCapabilityFromCM(ctx, client, capability, enable) + } + + setSupervisorCapabilityFromCR(ctx, dynamicClient, svSSHCommandRunner, capability, enable) +} + +func setSupervisorCapabilityFromCM(ctx context.Context, client clientset.Interface, capability string, enable bool) { + cm := getWcpClusterCapabilitiesCM(ctx, client) + if _, ok := cm.Data[capability]; !ok { + e2eframework.Logf("Capability %s doesn't exist, skip setting it to %t", capability, enable) + return + } + + cm.Data[capability] = strconv.FormatBool(enable) + updateWcpClusterCapabilitiesCM(ctx, client, cm) +} + +func setSupervisorCapabilityFromCR(ctx context.Context, client dynamic.Interface, svSSHCommandRunner e2essh.SSHCommandRunner, capabilityKey string, enable bool) { + cr := getSupervisorCapabilitiesCR(ctx, client) + crMap := cr.UnstructuredContent() + + supervisorCapList := crMap["spec"].(map[string]any)["supervisor"].([]any) + capabilityExists := false + + for _, item := range supervisorCapList { + itemMap := item.(map[string]any) + if itemMap["name"] == capabilityKey { + itemMap["enabled"] = enable + capabilityExists = true + + break + } + } + + if !capabilityExists { + e2eframework.Logf("Capability %s doesn't exist, skip setting it to %t", capabilityKey, enable) + return + } + + cr.SetUnstructuredContent(crMap) + updateSupervisorCapabilitiesCR(svSSHCommandRunner, cr) +} + +func getWcpClusterCapabilitiesCM(ctx context.Context, client clientset.Interface) *corev1.ConfigMap { + cm, err := client.CoreV1().ConfigMaps(capabilityConfigMapNs).Get(ctx, capabilityConfigMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("error getting ConfigMap(%s/%s): %v", capabilityConfigMapNs, capabilitiesCRName, err)) + + return cm +} + +func updateWcpClusterCapabilitiesCM(ctx context.Context, client clientset.Interface, cm *corev1.ConfigMap) { + _, err := client.CoreV1().ConfigMaps(capabilityConfigMapNs).Update(ctx, cm, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("error patching ConfigMap(%s/%s): %v", capabilityConfigMapNs, capabilitiesCRName, err)) +} + +func getSupervisorCapabilitiesCR(ctx context.Context, client dynamic.Interface) *unstructured.Unstructured { + cr, err := client.Resource(CapabilityGRV).Get(ctx, capabilitiesCRName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("error getting Capabilities %s: %v", capabilitiesCRName, err)) + + return cr +} + +// User "sso:Administrator@vsphere.local" cannot update resource "capabilities" in API group "iaas.vmware.com" at the cluster scope +// Patching the Capability as root user. +func updateSupervisorCapabilitiesCR(svSSHCommandRunner e2essh.SSHCommandRunner, cr *unstructured.Unstructured) { + yamlData, err := yaml.Marshal(cr) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("error marshaling YAML: %v", err)) + + kubeCtlExecCmd := fmt.Sprintf("cat < Content Libraries +- Create + - Name `vmservice` + - Subscription URL: `https://wp-content-pstg.broadcom.com/vmsvc/lib.json` + - Wait for library sync to complete + +### 2. Storage Classes + +The e2e tests require two storage policies to be created beforehand: +- `wcpglobal_storage_profile` exists or create one +- `worker-storagepolicy` + +Testbeds created using the `dev-integ-vds` already have the `wcpglobal_storage_profile`, +so you would need to only create the `worker-storagepolicy`. + +#### Option 1: Create missing policy + +Run the following script which creates the `worker-storagepolicy` for you. +```bash +GOVC_PASSWORD="${SSH_PASSWORD}" ./hack/managestoragepolicy.sh create "${VCSA_IP}" +``` + +#### Option 2: From VC UI + +- Left pane > Policies and Profiles +- VM Storage Policies from the left lane +- Confirm `wcpglobal_storage_profile` exists or create one + - K8s compliant name must be `wcpglobal-storage-profile` +- Create `worker-storagepolicy` + - K8s compliant name must be `worker-storagepolicy` +- Add both policies to any namespace in the Supervisor cluster. +This will add the storage policy CRD to the cluster which is a cluster resource. + - Supervisor > Select any namespace + - Storage > Edit button + - Select both storage policies + +## Running tests + +### Option 1 (VDS Testbeds) + +You can use the `configure-local.sh` script to configure your local shell to run the +E2E tests if you are using a VDS testbed. The script downloads the kubeconfig +from the WCP Control Plane node, places it in `~/.kube/wcp-config`, and exports +the required environment variables. + +You can run the tests by following, + +```bash +export TEST_TAGS="vmservice" # or a different tag you want to run. +export TEST_FOCUS="" # the tests you want to execute. +export TEST_SKIP="" # optional: if you want to skip specific tests. + +source ./hack/configure-local.sh +make wcp-vmservice-e2e +``` + +### Option 2 (Non VDS) + +Do the following to prepare your local shell session + +```bash +cp ~/.kube/wcp-config +export KUBECONFIG=~/.kube/wcp-config + +export VCSA_IP="" +export SSH_PASSWORD="" +export VCSA_PASSWORD="${SSH_PASSWORD}" +export NETWORK="vds" # vds or nsx. +``` + +Run the tests + +```bash +export TEST_TAGS="vmservice" # or a different tag you want to run. +export TEST_FOCUS="" # the tests you want to execute. +export TEST_SKIP="" # optional: if you want to skip specific tests. + +make wcp-vmservice-e2e +``` \ No newline at end of file diff --git a/test/e2e/vmservice/common/scheme.go b/test/e2e/vmservice/common/scheme.go new file mode 100644 index 000000000..b053baa26 --- /dev/null +++ b/test/e2e/vmservice/common/scheme.go @@ -0,0 +1,128 @@ +// Copyright (c) 2019-2024 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package common + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + storageV1 "k8s.io/api/storage/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + vmopv1a1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + vmopv1a2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + vmopv1a3 "github.com/vmware-tanzu/vm-operator/api/v1alpha3" + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + imageregistryv1alpha1 "github.com/vmware-tanzu/vm-operator/external/image-registry-operator/api/v1alpha1" + imageregistryv1alpha2 "github.com/vmware-tanzu/vm-operator/external/image-registry-operator/api/v1alpha2" + mopv1alpha2 "github.com/vmware-tanzu/vm-operator/external/mobility-operator/api/v1alpha2" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + vpcv1alpha1 "github.com/vmware-tanzu/vm-operator/external/nsx-operator/api/vpc/v1alpha1" + spqv1 "github.com/vmware-tanzu/vm-operator/external/storage-policy-quota/api/v1alpha1" + topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + cnsunregistervolumev1alpha1 "github.com/vmware-tanzu/vm-operator/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1" + cnsv1alpha1 "github.com/vmware-tanzu/vm-operator/external/vsphere-csi-driver/api/v1alpha1" +) + +// InitScheme adds scheme to each API typed resources. +func InitScheme() *runtime.Scheme { + sc := runtime.NewScheme() + addSchemes(sc) + + return sc +} + +func addSchemes(sc *runtime.Scheme) { + err := corev1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add core v1 to scheme: %v", err) + } + + err = appsv1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add apps v1 to scheme: %v", err) + } + + err = storageV1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add storage v1 to scheme: %v", err) + } + + err = cnsv1alpha1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable to add cns v1alpha1 to scheme: %v", err) + } + + err = vmopv1a1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add v1alpha1 VMOP APIs to scheme: %v", err) + } + + err = vmopv1a2.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add v1alpha2 VMOP APIs to scheme: %v", err) + } + + err = vmopv1a3.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add v1alpha3 VMOP APIs to scheme: %v", err) + } + + err = vmopv1a5.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add v1alpha5 VMOP APIs to scheme: %v", err) + } + + err = mopv1alpha2.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add v1alpha2 Mobility Operator APIs to scheme: %v", err) + } + + err = imageregistryv1alpha1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable to add Image Registry APIs to scheme: %v", err) + } + + err = imageregistryv1alpha2.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable to add Image Registry APIs to scheme: %v", err) + } + + err = topologyv1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add tanzu topology APIs to scheme: %v", err) + } + + err = ncpv1alpha1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add ncp v1alpha1 to scheme: %v", err) + } + + err = netopv1alpha1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add net-operator v1alpha1 to scheme: %v", err) + } + + err = vpcv1alpha1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable to add NSX Operator APIs to scheme: %v", err) + } + + err = spqv1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable to add SPQ Operator APIs to scheme: %v", err) + } + + err = cnsunregistervolumev1alpha1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable to add CnsUnregisterVolume APIs to scheme: %v", err) + } + + err = apiextensionsv1.AddToScheme(sc) + if err != nil { + e2eframework.Failf("unable add apiextensions v1 to scheme: %v", err) + } +} diff --git a/test/e2e/vmservice/common/vmservice_clusterproxy.go b/test/e2e/vmservice/common/vmservice_clusterproxy.go new file mode 100644 index 000000000..4e83fb298 --- /dev/null +++ b/test/e2e/vmservice/common/vmservice_clusterproxy.go @@ -0,0 +1,165 @@ +package common + +import ( + "bytes" + "context" + "io" + "net/url" + "os" + + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/kubectl" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/supervisor" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type VMServiceClusterProxy struct { + *wcpframework.WCPClusterProxy + + adminConfPath string +} + +func NewVMServiceClusterProxy(name string, kubeconfigPath string, scheme *runtime.Scheme) *VMServiceClusterProxy { + baseClusterProxy := wcpframework.NewWCPClusterProxy(name, kubeconfigPath, scheme) + + proxy := &VMServiceClusterProxy{ + baseClusterProxy, + "", + } + + return proxy +} + +// Create wraps `kubectl create ...` and prints the output so we can see what gets created to the cluster. +func (p *VMServiceClusterProxy) CreateWithArgs(ctx context.Context, resources []byte, args ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Create") + Expect(resources).NotTo(BeEmpty(), "resources is required for Create") + + return framework.KubectlCreateWithArgs(ctx, p.GetKubeconfigPath(), resources, p.args(args)...) +} + +func (p *VMServiceClusterProxy) CreateRawWithArgs(ctx context.Context, resources []byte, args ...string) ([]byte, []byte, error) { + Expect(ctx).NotTo(BeNil(), "ctx is required for Create") + Expect(resources).NotTo(BeEmpty(), "resources is required for Create") + + return framework.KubectlCreateRawWithArgs(ctx, p.GetKubeconfigPath(), resources, p.args(args)...) +} + +// Delete wraps `kubectl delete ...` and prints the output so we can see what gets deleted from the cluster. +func (p *VMServiceClusterProxy) DeleteWithArgs(ctx context.Context, resources []byte, args ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Delete") + Expect(resources).NotTo(BeEmpty(), "resources is required for Delete") + + return framework.KubectlDeleteWithArgs(ctx, p.GetKubeconfigPath(), resources, p.args(args)...) +} + +// Apply wraps `kubectl apply ...` and prints the output so we can see what gets applied to the cluster. +func (p *VMServiceClusterProxy) ApplyWithArgs(ctx context.Context, resources []byte, args ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Apply") + Expect(resources).NotTo(BeEmpty(), "resources is required for Apply") + + return framework.KubectlApplyWithArgs(ctx, p.GetKubeconfigPath(), resources, p.args(args)...) +} + +// Exec performs kubectl exec with following flags. +func (p *VMServiceClusterProxy) Exec(ctx context.Context, args ...string) ([]byte, error) { + Expect(ctx).NotTo(BeNil(), "ctx is required for Exec") + + return framework.KubectlExec(ctx, p.GetKubeconfigPath(), args...) +} + +// Label wraps `kubectl label ...` and prints the output so we can see what gets applied to the cluster. +func (p *VMServiceClusterProxy) Label(ctx context.Context, args ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Label") + + return framework.KubectlLabel(ctx, p.GetKubeconfigPath(), args...) +} + +func getAPIServerAdminConf(ctx context.Context, svKubeConfig string) (string, error) { + apiServerSSHCommandRunner, err := supervisor.GetAPIServerCommandRunner(ctx, svKubeConfig) + if err != nil { + return "", err + } + + conf, err := apiServerSSHCommandRunner.RunCommand("cat /etc/kubernetes/admin.conf") + if err != nil { + return "", err + } + + apiServerURL := kubectl.GetKubectlClusterForCurrentContext(ctx, svKubeConfig) + + parsedAPIServerURL, err := url.Parse(apiServerURL) + if err != nil { + return "", err + } + + apiServerIP := parsedAPIServerURL.Hostname() + + e2eframework.Logf("Using %s for API server IP with admin.conf", apiServerIP) + + conf = bytes.Replace(conf, []byte("127.0.0.1"), []byte(apiServerIP), 1) + + f, err := os.CreateTemp("", "gce2e-admin.conf") + if err != nil { + return "", err + } + + _, cerr := io.Copy(f, bytes.NewReader(conf)) + + if err = f.Close(); err != nil { + _ = os.Remove(f.Name()) + return "", err + } + + if cerr != nil { + _ = os.Remove(f.Name()) + return "", cerr + } + + return f.Name(), nil +} + +func (p *VMServiceClusterProxy) NewAdminClusterProxy(ctx context.Context) (*VMServiceClusterProxy, error) { + adminConfPath, err := getAPIServerAdminConf(ctx, p.GetKubeconfigPath()) + if err != nil { + return nil, err + } + + proxy := NewVMServiceClusterProxy("admin", adminConfPath, p.GetScheme()) + proxy.adminConfPath = adminConfPath + + return proxy, nil +} + +func (p *VMServiceClusterProxy) GetAdminClient() (client.Client, error) { + config := p.GetRESTConfig() + + // We replace 127.0.0.1 in admin.conf, but API server IP used may not be + // one of the certificate IP SANs, causing TLS verify to fail. + // Same as `kubectl --insecure-skip-tls-verify` + config.Insecure = true + config.CAData = nil + + return client.New(config, client.Options{Scheme: p.GetScheme()}) +} + +func (p *VMServiceClusterProxy) Dispose(ctx context.Context) { + if p.adminConfPath != "" { + _ = os.Remove(p.adminConfPath) + } + + p.WCPClusterProxy.Dispose(ctx) +} + +func (p *VMServiceClusterProxy) args(args []string) []string { + if p.adminConfPath != "" { + args = append(args, "--insecure-skip-tls-verify") + } + + return args +} diff --git a/test/e2e/vmservice/config/config.go b/test/e2e/vmservice/config/config.go new file mode 100644 index 000000000..a52b6d606 --- /dev/null +++ b/test/e2e/vmservice/config/config.go @@ -0,0 +1,133 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "fmt" + "os" + "strings" + + "time" + + . "github.com/onsi/gomega" + "sigs.k8s.io/yaml" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" +) + +// E2EConfig extends the shared framework config with infrastructure settings. +// For configuration, it should be a "bottom up" manner in terms of the layering architecture. +type E2EConfig struct { + framework.Config + + InfraConfig *InfraConfig `json:"infraConfig,omitempty"` +} + +type InfraConfig struct { + // kind, vcenter + InfraName string `json:"infraName,omitempty"` + KubeconfigPath string `json:"kubeconfigPath,omitempty"` + NetworkingTopology string `json:"networkingTopology,omitempty"` + KeepVCSIM bool `json:"keepVCSIM,omitempty"` + // Assume only one namespace for mgmt cluster on the infra provider + ManagementClusterConfig *ManagementClusterConfig `json:"managementClusterConfig,omitempty"` +} + +type ManagementClusterConfig struct { + ManagementClusterName string `json:"managementClusterName,omitempty"` + Resources *Resources `json:"resources,omitempty"` +} + +type Resources struct { + PhotonImageDisplayName string `json:"photonImageDisplayName,omitempty"` + UbuntuImageDisplayName string `json:"ubuntuImageDisplayName,omitempty"` + WindowsImageDisplayName string `json:"windowsImageDisplayName,omitempty"` + StorageClassName string `json:"storageClassName,omitempty"` + WorkerStorageClassName string `json:"workerStorageClassName,omitempty"` + VMClassName string `json:"vmClassName,omitempty"` + VMResourcePolicyName string `json:"vmResourcePolicyName,omitempty"` + ContentLibrarySubscriptionURL string `json:"contentLibrarySubscriptionURL,omitempty"` +} + +func (c *E2EConfig) GetInfraConfig() *InfraConfig { + return c.InfraConfig +} + +// GetIntervals returns the value in the format: "default/key: ["10m", "5s"]". +func (c *E2EConfig) GetIntervals(spec, key string) []any { + intervals, ok := c.Intervals[fmt.Sprintf("%s/%s", spec, key)] + if !ok { + if intervals, ok = c.Intervals[fmt.Sprintf("default/%s", key)]; !ok { + return nil + } + } + + intervalsInterfaces := make([]any, len(intervals)) + for i := range intervals { + intervalsInterfaces[i] = intervals[i] + } + + return intervalsInterfaces +} + +// GetVariable returns a variable from the e2e config file. +func (c *E2EConfig) GetVariable(varName string) string { + version, ok := c.Variables[varName] + Expect(ok).NotTo(BeFalse(), "failed to get variable %q", varName) + + return version +} + +// LoadE2EConfig loads the configuration for the e2e test environment. +func LoadE2EConfig(configPath string) *E2EConfig { + configData, err := os.ReadFile(configPath) + + Expect(err).ToNot(HaveOccurred(), "failed to read the e2e test config file: %s", configPath) + Expect(configData).ToNot(BeEmpty(), "the e2e test config file should not be empty: %s", configPath) + + configData = []byte(os.Expand(string(configData), func(v string) string { + parts := strings.SplitN(v, ":-", 2) + if val, ok := os.LookupEnv(parts[0]); ok && val != "" { + return val + } + + if len(parts) == 2 { + return parts[1] + } + + return "" + })) + config := &E2EConfig{} + Expect(yaml.Unmarshal(configData, config)).To(Succeed(), "failed to convert the e2e test config file to yaml") + + Expect(config.Validate()).To(Succeed(), "The e2e test config file is not valid") + + return config +} + +func (c *E2EConfig) Validate() error { + // kind or wcp must be provided + if c.InfraConfig.InfraName == "" { + return framework.ErrEmptyArg("InfraConfig.InfraName") + } + + // Intervals should be valid ginkgo intervals. + for k, intervals := range c.Intervals { + switch len(intervals) { + case 0: + return framework.ErrInvalidArg("Intervals[%s]=%q", k, intervals) + case 1, 2: + default: + return framework.ErrInvalidArg("Intervals[%s]=%q", k, intervals) + } + + for _, i := range intervals { + if _, err := time.ParseDuration(i); err != nil { + return framework.ErrInvalidArg("Intervals[%s]=%q", k, intervals) + } + } + } + + return nil +} diff --git a/test/e2e/vmservice/config/kind.yaml b/test/e2e/vmservice/config/kind.yaml new file mode 100644 index 000000000..49918fc63 --- /dev/null +++ b/test/e2e/vmservice/config/kind.yaml @@ -0,0 +1,59 @@ +--- +infraConfig: + infraName: "kind" + kubeconfigPath: "$HOME/.kube/config" + keepVCSIM: false + + managementClusterConfig: + managementClusterName: "kind-e2e" + resources: + photonImageDisplayName: "photon-k8sd-v1.15.5-vmware.1-guest.1" + storageClassName: "gc-storage-profile" + vmClassName: "best-effort-small" + vmResourcePolicyName: "" + +variables: + VMOPNamespace: "vmware-system-vmop" + VMOPDeploymentName: "vmware-system-vmop-controller-manager" + VMOPLeaderElectionResourceName: "vmware-system-vmop-controller-manager-runtime" + VMOPImagePrefix: "vmoperator-controller" + VMOPManagerCommand: "/manager" + EnvFSSVMSVC: "FSS_WCP_VMSERVICE" + EnvFSSHA: "FSS_WCP_FAULTDOMAINS" + EnvFSSInstanceStorage: "FSS_WCP_INSTANCE_STORAGE" + EnvFSSVMClassAsConfig: "FSS_WCP_VM_CLASS_AS_CONFIG" + EnvFSSVMClassAsConfigDaynDate: "FSS_WCP_VM_CLASS_AS_CONFIG_DAYNDATE" + EnvFSSVMImageRegistry: "FSS_WCP_VM_IMAGE_REGISTRY" + EnvFSSNamespacedVMClass: "FSS_WCP_NAMESPACED_VM_CLASS" + EnvFSSWindowsSysprep: "FSS_WCP_WINDOWS_SYSPREP" + EnvFSSVMServiceBackupRestore: "FSS_WCP_VMSERVICE_BACKUPRESTORE" + EnvFSSIncrementalRestore: "FSS_WCP_VMSERVICE_INCREMENTAL_RESTORE" + EnvFSSVMResizeCPUMemory: "FSS_WCP_VMSERVICE_RESIZE_CPU_MEMORY" + EnvFSSIsoSupport: "FSS_WCP_VMSERVICE_ISO_SUPPORT" + EnvFSSPodVMOnStretchedSupervisor: "FSS_PODVMONSTRETCHEDSUPERVISOR" + EnvFSSBYOK: "FSS_WCP_VMSERVICE_BYOK" + EnvWorkloadIsolation: "FSS_WCP_WORKLOAD_DOMAIN_ISOLATION" + +intervals: + # Please keep the wait-virtual-machine values in sync between the different + # configuration files. + default/wait-virtual-machine-creation: ["5m", "10s"] + default/wait-virtual-machine-deletion: ["5m", "10s"] + default/wait-virtual-machine-condition-update: ["5m", "10s"] + default/wait-virtual-machine-powerstate: ["5m", "10s"] + default/wait-virtual-machine-resize: ["5m", "10s"] + default/wait-virtual-machine-moid: ["5m", "10s"] + default/wait-virtual-machine-vmip: ["5m", "10s"] + default/wait-virtual-machine-image-creation: ["20s", "2s"] + default/consistent-virtual-machine-condition: ["30s", "5s"] + + default/wait-config-map-creation: ["2m", "5s"] + default/wait-secret-creation: ["2m", "5s"] + default/wait-pod-ready: ["2m", "10s"] + default/login-retry-timeout: ["5m", "10s"] + default/wait-subnet-creation: ["2m", "10s"] + default/wait-subnet-deletion: ["6m", "10s"] + default/wait-security-policy-creation: ["2m", "10s"] + default/wait-pvc-attachment: ["5m", "10s"] + default/wait-worker-storage-class: ["15m", "15s"] + default/wait-jumpbox-sshpass-ready: ["30m", "10s"] diff --git a/test/e2e/vmservice/config/wcp.yaml b/test/e2e/vmservice/config/wcp.yaml new file mode 100644 index 000000000..3836f8cbb --- /dev/null +++ b/test/e2e/vmservice/config/wcp.yaml @@ -0,0 +1,114 @@ +--- +infraConfig: + infraName: "wcp" + kubeconfigPath: "$HOME/.kube/wcp-config" + networkingTopology: "${NETWORK:-vds}" + managementClusterConfig: + managementClusterName: "wcp" + resources: + photonImageDisplayName: "photon-5.0" + windowsImageDisplayName: "windows-server2022-efi-tools" + storageClassName: "${STORAGE_CLASS:-wcpglobal-storage-profile}" + workerStorageClassName: "${WORKER_STORAGE_CLASS:-worker-storagepolicy}" + vmClassName: "best-effort-small" + vmResourcePolicyName: "" + contentLibrarySubscriptionURL: "${CONTENT_LIBRARY_SUBSCRIPTION_URL:-https://wp-content-pstg.broadcom.com/vmsvc/lib.json}" + +versionDiscovery: + enabled: true + systemSpecs: + guest-cluster-controller: docker-registry.kube-system.svc:5000/tkg-svs/package/tkg-service@sha256:b190d50fc2d35329d1be5299c24e9bf45bf7f8e40f36323c6648c9f9ed43902d + vm-operator: vmoperator-controller:1.9.0-e83c447f + +variables: + VMOPNamespace: "vmware-system-vmop" + VMOPDeploymentName: "vmware-system-vmop-controller-manager" + VMOPLeaderElectionResourceName: "vmware-system-vmop-controller-manager-runtime" + VMOPImagePrefix: "localhost:5000/vmware/vmop" + VMOPManagerCommand: "/manager" + EnvFSSVMSVC: "FSS_WCP_VMSERVICE" + EnvFSSInstanceStorage: "FSS_WCP_INSTANCE_STORAGE" + EnvFSSVMClassAsConfig: "FSS_WCP_VM_CLASS_AS_CONFIG" + EnvFSSVMClassAsConfigDaynDate: "FSS_WCP_VM_CLASS_AS_CONFIG_DAYNDATE" + EnvFSSVMImageRegistry: "FSS_WCP_VM_IMAGE_REGISTRY" + EnvFSSNamespacedVMClass: "FSS_WCP_NAMESPACED_VM_CLASS" + EnvFSSWindowsSysprep: "FSS_WCP_WINDOWS_SYSPREP" + EnvFSSV1alpha2: "FSS_WCP_VMSERVICE_V1ALPHA2" + EnvFSSVMServiceBackupRestore: "FSS_WCP_VMSERVICE_BACKUPRESTORE" + EnvFSSIncrementalRestore: "FSS_WCP_VMSERVICE_INCREMENTAL_RESTORE" + EnvFSSVMResizeCPUMemory: "FSS_WCP_VMSERVICE_RESIZE_CPU_MEMORY" + EnvFSSIsoSupport: "FSS_WCP_VMSERVICE_ISO_SUPPORT" + EnvNetworkProvider: "NETWORK_PROVIDER" + EnvFSSPodVMOnStretchedSupervisor: "FSS_PODVMONSTRETCHEDSUPERVISOR" + EnvFSSBYOK: "FSS_WCP_VMSERVICE_BYOK" + EnvWorkloadIsolation: "FSS_WCP_WORKLOAD_DOMAIN_ISOLATION" + +variables: + VMOPNamespace: "vmware-system-vmop" + VMOPDeploymentName: "vmware-system-vmop-controller-manager" + VMOPLeaderElectionResourceName: "vmware-system-vmop-controller-manager-runtime" + VMOPImagePrefix: "localhost:5000/vmware/vmop" + VMOPManagerCommand: "/manager" + EnvFSSVMSVC: "FSS_WCP_VMSERVICE" + EnvFSSInstanceStorage: "FSS_WCP_INSTANCE_STORAGE" + EnvFSSVMClassAsConfig: "FSS_WCP_VM_CLASS_AS_CONFIG" + EnvFSSVMClassAsConfigDaynDate: "FSS_WCP_VM_CLASS_AS_CONFIG_DAYNDATE" + EnvFSSVMImageRegistry: "FSS_WCP_VM_IMAGE_REGISTRY" + EnvFSSNamespacedVMClass: "FSS_WCP_NAMESPACED_VM_CLASS" + EnvFSSWindowsSysprep: "FSS_WCP_WINDOWS_SYSPREP" + EnvFSSV1alpha2: "FSS_WCP_VMSERVICE_V1ALPHA2" + EnvFSSVMServiceBackupRestore: "FSS_WCP_VMSERVICE_BACKUPRESTORE" + EnvFSSIncrementalRestore: "FSS_WCP_VMSERVICE_INCREMENTAL_RESTORE" + EnvFSSVMResizeCPUMemory: "FSS_WCP_VMSERVICE_RESIZE_CPU_MEMORY" + EnvFSSIsoSupport: "FSS_WCP_VMSERVICE_ISO_SUPPORT" + EnvNetworkProvider: "NETWORK_PROVIDER" + EnvFSSPodVMOnStretchedSupervisor: "FSS_PODVMONSTRETCHEDSUPERVISOR" + EnvFSSBYOK: "FSS_WCP_VMSERVICE_BYOK" + EnvWorkloadIsolation: "FSS_WCP_WORKLOAD_DOMAIN_ISOLATION" + # E2E Namespace used for the VM service tests. If not set, a random namespace will be created. + E2ENamespace: "${E2E_NAMESPACE:-}" + +intervals: + # Please keep the wait-virtual-machine values in sync between the different + # configuration files. + default/wait-virtual-machine-creation: ["5m", "10s"] + default/wait-virtual-machine-deletion: ["5m", "10s"] + default/wait-virtual-machine-condition-update: ["5m", "10s"] + default/wait-virtual-machine-powerstate: ["5m", "10s"] + default/wait-virtual-machine-resize: ["5m", "10s"] + default/wait-virtual-machine-moid: ["5m", "10s"] + default/wait-virtual-machine-restart-mode-update: ["2m", "10s"] + default/wait-virtual-machine-vmip: ["5m", "10s"] + default/wait-virtual-machine-image-creation: ["3m", "5s"] + default/consistent-virtual-machine-condition: ["30s", "5s"] + default/wait-virtual-machine-group-deletion: [ "60s", "5s" ] + default/wait-virtual-machine-group-condition-update: [ "6m", "5s" ] + default/wait-virtual-machine-publish-request-creation: ["60s", "5s"] + default/wait-virtual-machine-publish-request-condition: ["6m", "5s"] + default/wait-virtual-machine-publish-request-deletion: ["60s", "2s"] + default/wait-virtual-machine-group-publish-request-condition: [ "6m", "5s" ] + default/wait-virtual-machine-group-publish-request-deletion: [ "60s", "2s" ] + default/wait-virtual-machine-snapshot-condition: ["10m", "5s"] + default/wait-virtual-machine-snapshot-deletion: ["60s", "2s"] + default/wait-virtual-machine-snapshot-quota-usage: ["60s", "5s"] + default/wait-virtual-machine-snapshot-related-resource: ["60s", "5s"] + default/wait-virtual-machine-snapshot-revert: ["5m", "5s"] + default/wait-virtual-machine-snapshot-update: ["5m", "5s"] + default/wait-virtual-machine-web-console-request-creation: ["30s", "5s"] + default/wait-virtual-machine-cns-node-batch-attachment-creation: ["10m", "5s"] + default/wait-virtual-machine-cns-node-batch-attachment-deletion: ["5m", "5s"] + default/wait-content-library-name: ["2m", "5s"] + default/wait-storage-class-ready: ["5m", "15s"] + default/wait-namespace-ready: ["5m", "10s"] + default/wait-backup-to-complete: ["10m", "5s"] + + default/wait-config-map-creation: ["3m", "10s"] + default/wait-secret-creation: ["3m", "10s"] + default/wait-pod-ready: ["2m", "10s"] + default/login-retry-timeout: ["5m", "10s"] + default/wait-subnet-creation: ["2m", "10s"] + default/wait-subnet-deletion: ["6m", "10s"] + default/wait-security-policy-creation: ["2m", "10s"] + default/wait-pvc-attachment: ["5m", "10s"] + default/wait-pvc-deletion: ["2m", "5s"] + default/wait-jumpbox-sshpass-ready: ["30m", "10s"] \ No newline at end of file diff --git a/test/e2e/vmservice/consts/consts.go b/test/e2e/vmservice/consts/consts.go new file mode 100644 index 000000000..54014dd73 --- /dev/null +++ b/test/e2e/vmservice/consts/consts.go @@ -0,0 +1,38 @@ +// Copyright (c) 2023-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package consts + +const ( + NSX = "nsx" + VDS = "vds" + + NSXNetworkType = "nsx-t" + VDSNetworkType = "vsphere-distributed" + VPCNetworkType = "nsx-t-vpc" + + DefaultVDSNetworkName = "primary" + + WCP = "wcp" + KIND = "kind" + + VMServiceCLName = "vmservice" + + HTTPProxyEnv = "HTTP_PROXY" + DefaultVMUserName = "vmware" + DefaultVMPassword = "Admin!23" + AllowTCPForwardingKey = "AllowTcpForwarding" + SshdConfig = "/etc/ssh/sshd_config" + CmdRestartSSHD = "systemctl restart sshd" + SshPort = 22 + + JumpboxPodVMName = "jumpbox" + + VMGroupsCapabilityName = "supports_VM_service_VM_groups" + VirtualMachineSnapshotCapabilityName = "supports_VM_service_VM_snapshots" + VMPlacementPoliciesCapabilityName = "supports_VM_service_VM_placement_policies" + InventoryContentLibraryCapabilityName = "supports_inventory_content_library" + SharedDisksCapabilityName = "supports_shared_disks_with_VM_service_VMs" + AllDisksArePVCapabilityName = "supports_vm_service_all_disks_are_pvcs" + IaaSComputePoliciesCapabilityName = "supports_iaas_compute_policies" +) diff --git a/test/e2e/vmservice/lib/csi/batchattachment.go b/test/e2e/vmservice/lib/csi/batchattachment.go new file mode 100644 index 000000000..0dfbcbdf4 --- /dev/null +++ b/test/e2e/vmservice/lib/csi/batchattachment.go @@ -0,0 +1,109 @@ +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package csi + +import ( + "context" + + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + cnsoperatorv1alpha1 "github.com/vmware-tanzu/vm-operator/external/vsphere-csi-driver/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +// WaitForBatchAttachVolumesToBeAttached waits for the CnsNodeVMBatchAttachment +// resource to exist and for the specified volumes to be marked as attached. +func WaitForBatchAttachVolumesToBeAttached( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, name string, + volumeNames []string, +) bool { + return Eventually(func() bool { + batchAttachment, err := utils.GetCnsNodeVMBatchAttachment(ctx, client, ns, name) + if err != nil { + e2eframework.Logf("error getting CnsNodeVMBatchAttachment. retry due to: %v", err) + return false + } + + batchAttachmentVolAttached := make(map[string]bool) + + for _, volStatus := range batchAttachment.Status.VolumeStatus { + volName := volStatus.Name + if volName == "" { + e2eframework.Logf("volume name is empty. retrying") + return false + } + + // Check if volume is attached by looking at conditions + for _, condition := range volStatus.PersistentVolumeClaim.Conditions { + if condition.Type == cnsoperatorv1alpha1.ConditionAttached { + if condition.Status == metav1.ConditionTrue { + batchAttachmentVolAttached[volName] = true + } else { + batchAttachmentVolAttached[volName] = false + } + + break + } + } + } + + volumeNamesSet := make(map[string]bool) + for _, volumeName := range volumeNames { + volumeNamesSet[volumeName] = true + + attached, ok := batchAttachmentVolAttached[volumeName] + if !ok { + e2eframework.Logf("volume %s is not found in the batch attachment. retrying", volumeName) + return false + } + + if !attached { + e2eframework.Logf("volume %s is not attached. retrying", volumeName) + return false + } + } + + // Verify that the volume name lists are exactly equal. + if len(volumeNamesSet) != len(batchAttachmentVolAttached) { + e2eframework.Logf("volume names count not equal. retrying: %d != %d", + len(volumeNamesSet), len(batchAttachmentVolAttached)) + + return false + } + + for volumeName := range batchAttachmentVolAttached { + if !volumeNamesSet[volumeName] { + e2eframework.Logf("volume names are not equal. extra volume in batch attachment: %s", volumeName) + return false + } + } + + return true + }, + config.GetIntervals("default", "wait-virtual-machine-cns-node-batch-attachment-creation")...). + Should(BeTrue(), "Timed out waiting for CnsNodeVMBatchAttachment %s/%s to exist", ns, name) +} + +func WaitForCnsNodeVMBatchAttachmentToBeDeleted( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, name string, +) { + Eventually(func(g Gomega) { + _, err := utils.GetCnsNodeVMBatchAttachment(ctx, client, ns, name) + g.Expect(err).To(HaveOccurred()) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }, config.GetIntervals("default", "wait-virtual-machine-cns-node-batch-attachment-deletion")...). + Should(Succeed(), "Timed out waiting for CnsNodeVMBatchAttachment %s/%s to be deleted", ns, name) +} diff --git a/test/e2e/vmservice/lib/vmoperator/vmoperator.go b/test/e2e/vmservice/lib/vmoperator/vmoperator.go new file mode 100644 index 000000000..8d369f40e --- /dev/null +++ b/test/e2e/vmservice/lib/vmoperator/vmoperator.go @@ -0,0 +1,1318 @@ +// Copyright (c) 2019-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmoperator + +import ( + "bytes" + "context" + "fmt" + "math" + "net" + "reflect" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/sets" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1a1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + vmopv1a2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + vmopv1a3 "github.com/vmware-tanzu/vm-operator/api/v1alpha3" + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + vpcv1alpha1 "github.com/vmware-tanzu/vm-operator/external/nsx-operator/api/vpc/v1alpha1" + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" +) + +const virtualMachineKind = "VirtualMachine" + +const ( + vmopConfigurationConfigMap = "vsphere.provider.config.vmoperator.vmware.com" + vmClassBestEffortSmall = "best-effort-small" + vmClassBestEffortExtraSmall = "best-effort-xsmall" + NetworkProviderTypeVPC = "NSXT_VPC" +) + +// NetworkProviderInfo contains the network provider information for a VM. +type NetworkProviderInfo struct { + NetworkType string + IPv4 string + SubnetMask string + Gateway string +} + +func IsNetworkNsxtVPC(ctx context.Context, client ctrlclient.Client, config *config.E2EConfig) bool { + envs, err := utils.GetCommandEnvVars(ctx, client, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand")) + Expect(err).ToNot(HaveOccurred(), "%q cannot not be fetched from %q", config.GetVariable("EnvNetworkProvider"), config.GetVariable("VMOPManagerCommand")) + + return envs[config.GetVariable("EnvNetworkProvider")] == NetworkProviderTypeVPC +} + +// Utility function to ensure that a VirtualMachine with given name either exists or not, returns as soon as the CR exists in etcd. +// To check if the vSphere VM has been created, see WaitForVirtualMachineConditionCreated. +func WaitForVirtualMachineToExist(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName string) { + By("Verifying the existence of VM CR in etcd") + Eventually(func() bool { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + return vm != nil + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(BeTrue(), "Timed out waiting for k8s VirtualMachine %s to exist", vmName) +} + +// Utility function to wait for a VM to exist, VirtualMachineCreated condition to exist and expect to be True. +// This function fails when VirtualMachineCreated.Status == ConditionFalse, meaning the vSphere VM failed to be created. +// Use this function before helpers that wait on vSphere VM properties, +// such as WaitForVirtualMachinePowerState and WaitForVirtualMachineIP. +func WaitForVirtualMachineConditionCreated(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName string) { + kind := vmopv1a3.VirtualMachineConditionCreated + + By("Waiting for vSphere VM to be created") + Eventually(func(g Gomega) bool { + vm, err := utils.GetVirtualMachineA3(ctx, client, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + actualCondition := GetVirtualMachineConditionA3(vm, kind) + g.Expect(actualCondition).ToNot(BeNil()) + g.Expect(actualCondition.Status).To(Equal(metav1.ConditionTrue)) + + return true + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(BeTrue(), "Timed out waiting for VirtualMachine %s to be created", vmName) +} + +// Utility function to wait for the VM's Status.Class.Name to be updated. +func WaitForVirtualMachineStatusClassUpdated(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName, className string) { + Eventually(func(g Gomega) bool { + vm, err := utils.GetVirtualMachineA3(ctx, client, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + g.Expect(vm.Status.Class).NotTo(BeNil()) + g.Expect(vm.Status.Class.Name).To(Equal(className)) + + return true + }, config.GetIntervals("default", "wait-virtual-machine-resize")...).Should(BeTrue(), "Timed out waiting for VirtualMachines %s Status.Class to be updated to %s", vmName, className) +} + +func UpdateVirtualMachineClassName(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName, className string) { + Eventually(func() bool { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + vm.Spec.ClassName = className + if err := client.Update(ctx, vm); err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + return true + }, config.GetIntervals("default", "wait-virtual-machine-resize")...).Should(BeTrue(), "Timed out updating VirtualMachines %s ClassName to %s", vmName, className) +} + +func GetVirtualMachineCondition(vm *vmopv1a2.VirtualMachine, conditionType string) *metav1.Condition { + for _, condition := range vm.Status.Conditions { + if condition.Type == conditionType { + return &condition + } + } + + return nil +} + +// Utility function to check a particular condition consistency on a given list of Virtual Machine. +func CheckVirtualMachinesConditionConsistent(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, + ns string, vmName string, expectedCondition metav1.Condition) { + Consistently(func(g Gomega) { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + g.Expect(err).ToNot(HaveOccurred()) + + actualCondition := GetVirtualMachineCondition(vm, expectedCondition.Type) + g.Expect(actualCondition).ToNot(BeNil()) + g.Expect(actualCondition.Status).Should(Equal(expectedCondition.Status)) + + if actualCondition.Status == metav1.ConditionFalse { + g.Expect(actualCondition.Reason).Should(Equal(expectedCondition.Reason)) + } + }, config.GetIntervals("default", "consistent-virtual-machine-condition")...).Should(Succeed(), "VirtualMachine conditions changed") +} + +func GetVirtualMachineConditionA3(vm *vmopv1a3.VirtualMachine, conditionType string) *metav1.Condition { + var condition *metav1.Condition + + for _, c := range vm.Status.Conditions { + if c.Type == conditionType { + if condition != nil { + if condition.LastTransitionTime.After(c.LastTransitionTime.Time) { + continue + } + } + + condition = &c + } + } + + return condition +} + +// Utility function to check Virtual Machine creation. +func WaitForVirtualMachineCreation(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, vmName string) { + By(fmt.Sprintf("Verify that a single VirtualMachine '%s/%s' is created", ns, vmName)) + WaitForVirtualMachineToExist(ctx, config, svClusterClient, ns, vmName) + WaitForVirtualMachineConditionCreated(ctx, config, svClusterClient, ns, vmName) + WaitForVirtualMachinePowerState(ctx, config, svClusterClient, ns, vmName, string(vmopv1a5.VirtualMachinePowerStateOn)) + WaitForVirtualMachineIP(ctx, config, svClusterClient, ns, vmName) +} + +// Utility function to check Virtual Machine Status IP. +func WaitForVirtualMachineIP(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, vmName string) { + By(fmt.Sprintf("Verify that an IP (ipv4) is allocated to the VirtualMachine '%s/%s'", ns, vmName)) + Eventually(func() bool { + vm, err := utils.GetVirtualMachine(ctx, svClusterClient, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + return vm.Status.Network != nil && + vm.Status.Network.PrimaryIP4 != "" && + net.ParseIP(vm.Status.Network.PrimaryIP4).To4() != nil + }, config.GetIntervals("default", "wait-virtual-machine-vmip")...).Should(BeTrue()) +} + +// Utility function to check Virtual Machine Status MoID. +func WaitForVirtualMachineMOID(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, vmName string) { + By("Verify that the VirtualMachine has a Unique ID/MOID") + Eventually(func() bool { + vm, err := utils.GetVirtualMachine(ctx, svClusterClient, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + return vm.Status.UniqueID != "" + }, config.GetIntervals("default", "wait-virtual-machine-moid")...).Should(BeTrue()) +} + +// Utility function to check PVC Attachment with Virtual Machine Status. +func WaitForPVCAttachment(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, vmName, pvcName string) { + By("Verify that the VirtualMachine has a PVC attachment: " + pvcName) + // Note that we rely on the assumption that the volume name is the + // same as the PVC name. This is enforced by the VM manifest + // builder that uses the PVC name as the volume name. + Eventually(func() bool { + vm, err := utils.GetVirtualMachine(ctx, svClusterClient, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + for _, vol := range vm.Status.Volumes { + if strings.HasPrefix(vol.Name, pvcName) { + return vol.Attached + } + } + + return false + }, config.GetIntervals("default", "wait-pvc-attachment")...).Should(BeTrue()) +} + +// Utility function to check Virtual Machine Instance Storage Annotations. +func WaitForVirtualMachineInstanceStorageAnnotations(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, vmName string) { + By("Verify that we have a single VirtualMachine") + WaitForVirtualMachineToExist(ctx, config, svClusterClient, ns, vmName) + + By("Verify that VirtualMachine is up and running in vSphere and Instance Storage Annotations are added") + Eventually(func(g Gomega) bool { + vm, err := utils.GetVirtualMachine(ctx, svClusterClient, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + g.Expect(vm.GetAnnotations()).To(HaveKey("vmoperator.vmware.com/instance-storage-selected-node")) + + return true + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(BeTrue()) +} + +// Utility function to ensure that a condition on a VirtualMachine resource eventually reaches the expected status. +func WaitOnVirtualMachineConditionUpdate(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName string, + expectedCondition metav1.Condition) { + Eventually(func() bool { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + actualCondition := GetVirtualMachineCondition(vm, expectedCondition.Type) + if actualCondition == nil { + return false + } + + if actualCondition.Status != expectedCondition.Status { + // wait and retry condition fetch in a while + return false + } + + if actualCondition.Status == metav1.ConditionFalse { + // Wait for reason eventually to become the expected one. + // When we delete a VMClass, if WCP_Namespaced_VM_Class is not enabled, + // the reason may be VirtualMachineClassBindingNotFound at first, + // but it would eventually be VirtualMachineClassNotFound + if actualCondition.Reason != expectedCondition.Reason { + return false + } + } + + return true + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...).Should(BeTrue(), "Timed out waiting for VirtualMachines %s condition to be updated", vmName) +} + +// Utility function to check a particular condition on Virtual Machine creation. +func WaitOnVirtualMachineCondition( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, vmName string, + expectedCondition metav1.Condition) { + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + g.Expect(err).ToNot(HaveOccurred()) + + actualCondition := GetVirtualMachineCondition(vm, expectedCondition.Type) + g.Expect(actualCondition).ToNot(BeNil()) + + g.Expect(actualCondition.Status).Should(Equal(expectedCondition.Status)) + + if actualCondition.Status == metav1.ConditionFalse { + g.Expect(actualCondition.Reason).Should(Equal(expectedCondition.Reason)) + } + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out waiting for Condition: %+v on VirtualMachine: %s", expectedCondition, vmName) +} + +// Utility function to ensure that a VirtualMachine is deleted. +func WaitForVirtualMachineToBeDeleted(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName string) { + Eventually(func(g Gomega) { + _, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + g.Expect(err).To(HaveOccurred()) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }, config.GetIntervals("default", "wait-virtual-machine-deletion")...).Should(Succeed(), "Timed out waiting for VirtualMachine %s to be deleted", vmName) +} + +// Utility function to ensure that a Subnet/SubnetSet is deleted. +func WaitForSubnetOrSubnetSetToBeDeleted(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, subnetName, kind string) { + switch kind { + case "Subnet": + Eventually(func(g Gomega) { + _, err := utils.GetSubnet(ctx, client, ns, subnetName) + g.Expect(err).To(HaveOccurred()) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }, config.GetIntervals("default", "wait-subnet-deletion")...).Should(Succeed(), "Timed out waiting for Subnet %s to be deleted", subnetName) + case "SubnetSet": + Eventually(func(g Gomega) { + _, err := utils.GetSubnetSet(ctx, client, ns, subnetName) + g.Expect(err).To(HaveOccurred()) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }, config.GetIntervals("default", "wait-subnet-deletion")...).Should(Succeed(), "Timed out waiting for SubnetSet %s to be deleted", subnetName) + default: + Expect(false).To(BeTrue(), "unknown kind: %s", kind) + } +} + +func WaitForVirtualMachineZone(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, zone, vmName string) { + Eventually(func(g Gomega) bool { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + if zone == "" { + g.Expect(vm.Status.Zone).ToNot(BeEmpty()) + } else { + g.Expect(vm.Status.Zone).To(Equal(zone)) + } + + return true + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(BeTrue(), "Timed out waiting for VirtualMachines %s to be created in zone %s", vmName, zone) +} + +func UpdateVirtualMachinePowerState(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName, powerState string) { + Eventually(func() bool { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + vm.Spec.PowerState = vmopv1a2.VirtualMachinePowerState(powerState) + if err := client.Update(ctx, vm); err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + return true + }, config.GetIntervals("default", "wait-virtual-machine-powerstate")...).Should(BeTrue(), "Timed out updating VirtualMachines %s PowerState to %s", vmName, powerState) +} + +func WaitForVirtualMachinePowerState( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, vmName, expectedPowerState string) { + By(fmt.Sprintf("Waiting for VM power state to reach %s", expectedPowerState)) + Eventually(func() bool { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + return vm.Status.PowerState == vmopv1a2.VirtualMachinePowerState(expectedPowerState) + }, config.GetIntervals("default", "wait-virtual-machine-powerstate")...).Should(BeTrue(), "Timed out waiting for VirtualMachines %s PowerState to be updated to %s", vmName, expectedPowerState) +} + +func WaitForLinuxPrepCustomizeNextPowerOnFalse( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, vmName string) { + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, client, ns, vmName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(vm.Spec.Bootstrap).ToNot(BeNil()) + g.Expect(vm.Spec.Bootstrap.LinuxPrep).ToNot(BeNil()) + g.Expect(vm.Spec.Bootstrap.LinuxPrep.CustomizeAtNextPowerOn).To(HaveValue(BeFalse())) + }, config.GetIntervals("default", "wait-virtual-machine-vmip")...).Should(Succeed(), "Timed out waiting for VirtualMachines %s LinuxPrep CustomizeNextPowerOn to be updated to be false", vmName) +} + +func GetVirtualMachineMOID(ctx context.Context, client ctrlclient.Client, ns, vmName string) string { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + Expect(err).ShouldNot(HaveOccurred()) + Expect(vm.Status.UniqueID).ShouldNot(BeEmpty()) + + return vm.Status.UniqueID +} + +func GetVirtualMachineIP(ctx context.Context, client ctrlclient.Client, ns, vmName string) string { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + Expect(err).ShouldNot(HaveOccurred()) + + if vm.Status.Network == nil { + return "" + } + + return vm.Status.Network.PrimaryIP4 +} + +func DeleteVirtualMachine(ctx context.Context, client ctrlclient.Client, ns, vmName string) { + vm, err := utils.GetVirtualMachine(ctx, client, ns, vmName) + Expect(err).ShouldNot(HaveOccurred()) + Expect(client.Delete(ctx, vm)).Should(Succeed()) +} + +func DeleteSubnetOrSubnetSet(ctx context.Context, client ctrlclient.Client, ns, subnetName, kind string) { + var ( + obj ctrlclient.Object + err error + ) + + switch kind { + case "Subnet": + obj, err = utils.GetSubnet(ctx, client, ns, subnetName) + case "SubnetSet": + obj, err = utils.GetSubnetSet(ctx, client, ns, subnetName) + default: + err = fmt.Errorf("unsupported kind: %s", kind) + } + + Expect(err).ToNot(HaveOccurred()) + Expect(client.Delete(ctx, obj)).To(Succeed()) +} + +// DescribeAllVirtualMachinesInNamespace logs the output of `kubectl describe vm` for all VMs in the given namespace. +func DescribeAllVirtualMachinesInNamespace(ctx context.Context, client ctrlclient.Client, kubeconfigPath, ns string) { + e2eframework.Logf("Describing all VMs in namespace %s", ns) + + vmList := &vmopv1a2.VirtualMachineList{} + Expect(client.List(ctx, vmList, ctrlclient.InNamespace(ns))).Should(Succeed()) + + for _, vm := range vmList.Items { + stdout, stderr, err := framework.KubectlDescribeWithNamespacedName(ctx, kubeconfigPath, "vm", ns, vm.Name) + if err != nil { + e2eframework.Logf("Failed to run kubectl describe for VM '%s/%s': %s", ns, vm.Name, stderr) + continue + } + + e2eframework.Logf("kubectl describe vm -n %s %s:\n%s", ns, vm.Name, stdout) + } +} + +// DescribeResourceIfExists logs the output of `kubectl describe ` if the given resource exists. +func DescribeResourceIfExists(ctx context.Context, client ctrlclient.Client, kubeconfigPath, ns, resourceName, resource string) { + stdout, stderr, err := framework.KubectlDescribeWithNamespacedName(ctx, kubeconfigPath, resource, ns, resourceName) + if bytes.Contains(stderr, []byte("NotFound")) { + e2eframework.Logf("Skip kubectl describe output as the resource %s '%s/%s' doesn't exist", resource, ns, resourceName) + return + } + + Expect(err).ToNot(HaveOccurred(), "Failed to run kubectl describe for resource %s '%s/%s': %s", resource, ns, resourceName, stderr) + e2eframework.Logf("kubectl describe %s -n %s %s:\n%s", resource, ns, resourceName, stdout) +} + +// Utility function to get a image k8s name given its display name. +func WaitForVirtualMachineImageName(ctx context.Context, config *framework.Config, + client ctrlclient.Client, namespace, imageDisplayName string) (string, error) { + vmImageRegistryFss := utils.IsFssEnabled(ctx, client, + config.GetVariable("VMOPNamespace"), + config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), + config.GetVariable("EnvFSSVMImageRegistry")) + + var options []ctrlclient.ListOption + if vmImageRegistryFss { + options = append(options, ctrlclient.InNamespace(namespace)) + } + + var imageName string + + Eventually(func(g Gomega) bool { + imgList, err := utils.ListVirtualMachineImagesWithOptions(ctx, client, options) + g.Expect(err).ToNot(HaveOccurred(), "Failed to list VirtualMachineImages") + + for _, img := range imgList.Items { + if img.Status.Name == imageDisplayName { + imageName = img.Name + return true + } + } + + return false + }, config.GetIntervals("default", "wait-virtual-machine-image-creation")...).Should(BeTrue(), + fmt.Sprintf("failed to find vm image with display name %s", imageDisplayName)) + + return imageName, nil +} + +// Utility function to wait for a VMI to have its Status.Disks populated. +func WaitForVirtualMachineImageStatusDisks(ctx context.Context, config *framework.Config, + client ctrlclient.Client, namespace, imageName string) { + vmImageRegistryFss := utils.IsFssEnabled(ctx, client, + config.GetVariable("VMOPNamespace"), + config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), + config.GetVariable("EnvFSSVMImageRegistry")) + if !vmImageRegistryFss { + namespace = "" + } + + objKey := ctrlclient.ObjectKey{Name: imageName, Namespace: namespace} + + Eventually(func(g Gomega) { + vmi := &vmopv1a3.VirtualMachineImage{} + g.Expect(client.Get(ctx, objKey, vmi)).To(Succeed()) + g.Expect(vmi.Status.Disks).ToNot(BeEmpty()) + }, config.GetIntervals("default", "wait-virtual-machine-image-creation")...).Should(Succeed(), + fmt.Sprintf("failed to wait for vm image %s to have Status.Disks populated", imageName)) +} + +// Utility function to get a ClusterVirtualMachineImage k8s object's name by its display name. +func WaitForClusterVirtualMachineImageName(ctx context.Context, config *framework.Config, + client ctrlclient.Client, imageDisplayName string) (string, error) { + vmImageRegistryFss := utils.IsFssEnabled(ctx, client, + config.GetVariable("VMOPNamespace"), + config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), + config.GetVariable("EnvFSSVMImageRegistry")) + if !vmImageRegistryFss { + return "", fmt.Errorf("cannot get ClusterVirtualMachineImage as image-registry FSS is not enabled") + } + + var cvmiName string + + Eventually(func(g Gomega) bool { + cvmiList, err := utils.ListClusterVirtualMachineImages(ctx, client) + g.Expect(err).ToNot(HaveOccurred(), "Failed to list ClusterVirtualMachineImages") + + for _, cvmiObj := range cvmiList.Items { + if cvmiObj.Status.Name == imageDisplayName { + cvmiName = cvmiObj.Name + return true + } + } + + return false + }, config.GetIntervals("default", "wait-virtual-machine-image-creation")...).Should(BeTrue(), + fmt.Sprintf("failed to find cvmi by display name %s", imageDisplayName)) + + return cvmiName, nil +} + +func GetClusterScopedVirtualMachineImage(ctx context.Context, config *framework.Config, client ctrlclient.Client, name string) (ctrlclient.Object, error) { + vmImageRegistryFss := utils.IsFssEnabled(ctx, client, + config.GetVariable("VMOPNamespace"), + config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), + config.GetVariable("EnvFSSVMImageRegistry")) + if vmImageRegistryFss { + virtualMachineImage := &vmopv1a2.ClusterVirtualMachineImage{} + + err := client.Get(ctx, ctrlclient.ObjectKey{Name: name}, virtualMachineImage) + if err != nil { + return nil, err + } + + return virtualMachineImage, nil + } + + virtualMachineImage := &vmopv1a2.VirtualMachineImage{} + + err := client.Get(ctx, ctrlclient.ObjectKey{Name: name}, virtualMachineImage) + if err != nil { + return nil, err + } + + return virtualMachineImage, nil +} + +func GetClusterScopedVirtualMachineImageV1A1(ctx context.Context, config *framework.Config, client ctrlclient.Client, name string) (ctrlclient.Object, error) { + vmImageRegistryFss := utils.IsFssEnabled(ctx, client, + config.GetVariable("VMOPNamespace"), + config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), + config.GetVariable("EnvFSSVMImageRegistry")) + if vmImageRegistryFss { + virtualMachineImage := &vmopv1a1.ClusterVirtualMachineImage{} + + err := client.Get(ctx, ctrlclient.ObjectKey{Name: name}, virtualMachineImage) + if err != nil { + return nil, err + } + + return virtualMachineImage, nil + } + + virtualMachineImage := &vmopv1a1.VirtualMachineImage{} + + err := client.Get(ctx, ctrlclient.ObjectKey{Name: name}, virtualMachineImage) + if err != nil { + return nil, err + } + + return virtualMachineImage, nil +} + +func GetVMClassInNamespace(ctx context.Context, client ctrlclient.Client, config *config.E2EConfig, ns, vmclassName string) (*vmopv1a2.VirtualMachineClass, error) { + vmclass := &vmopv1a2.VirtualMachineClass{} + + e2eframework.Logf("Getting Namespace scoped VMClass %s in namespace %s", vmclassName, ns) + + err := client.Get(ctx, ctrlclient.ObjectKey{Name: vmclassName, Namespace: ns}, vmclass) + if err != nil { + return nil, err + } + + return vmclass, nil +} + +func MemoryQuantityToMb(q resource.Quantity) int { + return int(math.Ceil(float64(q.Value()) / float64(1024*1024))) +} + +func GetVMClassesNameForUpdate(ctx context.Context, config *framework.Config, client ctrlclient.Client) (oldVMClassName, newVMClassName string, _ error) { + // By default, we use the name directly from vcenter with the assumption that best-effort-small and + // best-effort-medium exist, and we cannot assume the vm class CR presents. + return vmClassBestEffortExtraSmall, vmClassBestEffortSmall, nil +} + +func getVirtualMachinePublishRequestCondition(vmPub *vmopv1a2.VirtualMachinePublishRequest, conditionType string) *metav1.Condition { + for _, condition := range vmPub.Status.Conditions { + if condition.Type == conditionType { + return &condition + } + } + + return nil +} + +// VerifyVirtualMachinePublishRequestCondition waits until the expected condition is met on the VirtualMachinePublishRequest. +func VerifyVirtualMachinePublishRequestCondition( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, vmPubName string, + expectedCondition metav1.Condition) { + Eventually(func(g Gomega) { + vmPub, err := utils.GetVirtualMachinePublishRequest(ctx, client, ns, vmPubName) + g.Expect(err).ToNot(HaveOccurred()) + + actualCondition := getVirtualMachinePublishRequestCondition(vmPub, expectedCondition.Type) + g.Expect(actualCondition).ToNot(BeNil()) + + g.Expect(actualCondition.Status).Should(Equal(expectedCondition.Status)) + + if actualCondition.Status == metav1.ConditionFalse { + g.Expect(actualCondition.Reason).Should(Equal(expectedCondition.Reason)) + } + }, config.GetIntervals("default", "wait-virtual-machine-publish-request-condition")...).Should(Succeed(), "Timed out waiting for Condition: %+v on VirtualMachinePublishRequest: %s", expectedCondition, vmPubName) +} + +// VerifyVirtualMachineGroupPublishRequestCompleted waits until the completed condition is true. +func VerifyVirtualMachineGroupPublishRequestCompleted( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, name string) { + var lastConditions []metav1.Condition + + Eventually(func(g Gomega) bool { + vmGroupPub, err := utils.GetVirtualMachineGroupPublishRequest(ctx, client, ns, name) + if err != nil { + e2eframework.Logf("get vm group publish request error: %v", err) + return false + } + + if !reflect.DeepEqual(lastConditions, vmGroupPub.Status.Conditions) { + lastConditions = vmGroupPub.Status.Conditions + e2eframework.Logf("VirtualMachineGroupPublishRequest %s Conditions: %v", name, lastConditions) + } + + for _, condition := range vmGroupPub.Status.Conditions { + if condition.Type == vmopv1a5.VirtualMachineGroupPublishRequestConditionComplete { + return condition.Status == metav1.ConditionTrue + } + } + + return false + }, config.GetIntervals("default", "wait-virtual-machine-group-publish-request-condition")...).Should(BeTrue(), + "Timed out waiting for VirtualMachineGroupPublishRequest to be completed") +} + +func VerifyVirtualMachineGroupPublishRequestDeleted( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, name string) { + Eventually(func(g Gomega) bool { + _, err := utils.GetVirtualMachineGroupPublishRequest(ctx, client, ns, name) + return apierrors.IsNotFound(err) + }, config.GetIntervals("default", "wait-virtual-machine-group-publish-request-deletion")...).Should(BeTrue(), + "Timed out waiting for VirtualMachineGroupPublishRequest to be deleted") +} + +func VerifyVirtualMachineGroupLinked( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, name string, + expectedMembers sets.Set[vmopv1a5.GroupMember]) { + e2eframework.Logf("%s expected members: %v", name, expectedMembers) + + var lastActualMembers sets.Set[vmopv1a5.GroupMember] + + Eventually(func(g Gomega) bool { + vmGroup, err := utils.GetVirtualMachineGroup(ctx, client, ns, name) + if err != nil { + e2eframework.Logf("get vm group error: %v", err) + return false + } + + actualMembers := make(sets.Set[vmopv1a5.GroupMember]) + + for _, member := range vmGroup.Status.Members { + for _, condition := range member.Conditions { + if condition.Type == vmopv1a5.VirtualMachineGroupMemberConditionGroupLinked && + condition.Status == metav1.ConditionTrue { + actualMembers.Insert(vmopv1a5.GroupMember{ + Name: member.Name, + Kind: member.Kind, + }) + + break + } + } + } + + if !reflect.DeepEqual(lastActualMembers, actualMembers) { + lastActualMembers = actualMembers + e2eframework.Logf("%s actual members: %v", name, lastActualMembers) + } + + return actualMembers.Equal(expectedMembers) + }, config.GetIntervals("default", "wait-virtual-machine-group-condition-update")...).Should(BeTrue(), + "Timed out waiting for VirtualMachineGroup to have expected members with group linked condition") +} + +func VerifyWebConsoleRequestStatus(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, webconsoleName string) { + By("Verify that WebConsoleRequest populates Proxy Address in Status") + Eventually(func(g Gomega) { + webconsole, err := utils.GetVWebConsoleRequest(ctx, client, ns, webconsoleName) + g.Expect(err).ToNot(HaveOccurred()) + + actualStatusProxyAddr := webconsole.Status.ProxyAddr + g.Expect(actualStatusProxyAddr).ToNot(BeNil()) + }, config.GetIntervals("default", "wait-virtual-machine-web-console-request-creation")...).Should(Succeed(), "Timed out waiting for WebConsoleRequest %s Status to populate proxy addr", webconsoleName) +} + +func VerifyVirtualMachineWebConsoleRequestStatus(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmWebconsoleName string) { + By("Verify that VirtualMachineWebConsoleRequest populates Proxy Address in Status") + Eventually(func(g Gomega) { + vmWebconsole, err := utils.GetVirtualMachineWebConsoleRequest(ctx, client, ns, vmWebconsoleName) + g.Expect(err).ToNot(HaveOccurred()) + + actualStatusProxyAddr := vmWebconsole.Status.ProxyAddr + g.Expect(actualStatusProxyAddr).ToNot(BeNil()) + }, config.GetIntervals("default", "wait-virtual-machine-web-console-request-creation")...).Should(Succeed(), "Timed out waiting for VirtualMachineWebConsoleRequest %s Status to populate proxy addr", vmWebconsoleName) +} + +func DeleteVirtualMachinePublishRequest(ctx context.Context, client ctrlclient.Client, ns, vmPubName string) { + vmPub, err := utils.GetVirtualMachinePublishRequest(ctx, client, ns, vmPubName) + Expect(err).ToNot(HaveOccurred()) + Expect(client.Delete(ctx, vmPub)).To(Succeed()) +} + +func WaitForVirtualMachinePublishRequestToBeDeleted(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmPubName string) { + Eventually(func(g Gomega) { + _, err := utils.GetVirtualMachinePublishRequest(ctx, client, ns, vmPubName) + g.Expect(err).To(HaveOccurred()) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }, config.GetIntervals("default", "wait-virtual-machine-publish-request-deletion")...).Should(Succeed(), "Timed out waiting for VirtualMachinePublishRequest %s to be deleted", vmPubName) +} + +func GetVirtualMachinePublishRequestSourceName(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, vmPubName string) (string, error) { + var vmPubSourceName string + + Eventually(func(g Gomega) { + vmPub, err := utils.GetVirtualMachinePublishRequest(ctx, svClusterClient, ns, vmPubName) + g.Expect(err).ToNot(HaveOccurred()) + + vmPubStatusSourceRef := vmPub.Status.SourceRef + g.Expect(vmPubStatusSourceRef).ToNot(BeNil()) + + vmPubSourceName = vmPub.Status.SourceRef.Name + g.Expect(vmPubSourceName).ToNot(BeEmpty()) + }, config.GetIntervals("default", "wait-virtual-machine-publish-request-condition")...).Should(Succeed(), "failed to get vmpub %s source name", vmPubName) + + return vmPubSourceName, nil +} + +func GetVirtualMachinePublishRequestTargetItemName(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, vmPubName string) (string, error) { + var vmPubTargetItemName string + + Eventually(func(g Gomega) { + vmPub, err := utils.GetVirtualMachinePublishRequest(ctx, svClusterClient, ns, vmPubName) + g.Expect(err).ToNot(HaveOccurred()) + + vmPubStatusTargetRef := vmPub.Status.TargetRef + g.Expect(vmPubStatusTargetRef).ToNot(BeNil()) + + vmPubTargetItemName = vmPub.Status.TargetRef.Item.Name + g.Expect(vmPubTargetItemName).ToNot(BeEmpty()) + }, config.GetIntervals("default", "wait-virtual-machine-publish-request-condition")...).Should(Succeed(), "failed to get vmpub %s target item name", vmPubName) + + return vmPubTargetItemName, nil +} + +// GetVirtualMachineNetworkProviderIP returns the IP address of the network provider for the given VM. +// For VDS topology, it returns the IP address of the network interface from net-operator. +// For NSX topology, it returns the IP address of the virtual network interface from ncp. +// For VPC topology, it returns the IP address of the subnetport from nsx operator. +func GetVirtualMachineNetworkProviderIP(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName string) string { + if framework.NetworkTopologyIs(config.InfraConfig.NetworkingTopology, consts.VDS) { + By("Getting VM network provider IP from Net-Operator's networkinterfaces in VDS") + + networkIfList := &netopv1alpha1.NetworkInterfaceList{} + Expect(client.List(ctx, networkIfList, ctrlclient.InNamespace(ns))).To(Succeed()) + Expect(networkIfList.Items).ToNot(BeEmpty(), "no NetworkInterfaces found in namespace %s", ns) + + for _, networkIf := range networkIfList.Items { + for _, ownerRef := range networkIf.OwnerReferences { + if ownerRef.Kind == virtualMachineKind && ownerRef.Name == vmName { + Expect(networkIf.Status.IPConfigs).ToNot(BeEmpty()) + return networkIf.Status.IPConfigs[0].IP + } + } + } + } + + if framework.NetworkTopologyIs(config.InfraConfig.NetworkingTopology, consts.NSX) { + if IsNetworkNsxtVPC(ctx, client, config) { + By("Getting VM network provider IP from NSX Operator's SubnetPort in NSX") + + subnetPortList := &vpcv1alpha1.SubnetPortList{} + Expect(client.List(ctx, subnetPortList, ctrlclient.InNamespace(ns))).To(Succeed()) + Expect(subnetPortList.Items).ToNot(BeEmpty(), "no SubnetPort found in namespace %s", ns) + + for _, subnetPort := range subnetPortList.Items { + for _, ownerRef := range subnetPort.OwnerReferences { + if ownerRef.Kind == virtualMachineKind && ownerRef.Name == vmName { + Expect(subnetPort.Status.NetworkInterfaceConfig.IPAddresses).ToNot(BeEmpty()) + // Note SubnetPort provides IPAddress with CIDR format. + cidr := subnetPort.Status.NetworkInterfaceConfig.IPAddresses[0].IPAddress + ip, _, err := net.ParseCIDR(cidr) + Expect(err).ToNot(HaveOccurred(), "failed to parse CIDR from SubnetPort IPAddress %s", cidr) + + return ip.String() + } + } + } + } else { + By("Getting VM network provider IP from NCP's VirtualNetworkInterfaces in NSX") + + vnetIfList := &ncpv1alpha1.VirtualNetworkInterfaceList{} + Expect(client.List(ctx, vnetIfList, ctrlclient.InNamespace(ns))).To(Succeed()) + Expect(vnetIfList.Items).ToNot(BeEmpty(), "no VirtualNetworkInterfaces found in namespace %s", ns) + + for _, vnetIf := range vnetIfList.Items { + for _, ownerRef := range vnetIf.OwnerReferences { + if ownerRef.Kind == virtualMachineKind && ownerRef.Name == vmName { + Expect(vnetIf.Status.IPAddresses).ToNot(BeEmpty()) + return vnetIf.Status.IPAddresses[0].IP + } + } + } + } + } + + return "" +} + +func waitForVDSNetworkIf(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName string) []NetworkProviderInfo { + var res []NetworkProviderInfo + + Eventually(func() error { + networkIfList := &netopv1alpha1.NetworkInterfaceList{} + + err := client.List(ctx, networkIfList, ctrlclient.InNamespace(ns)) + if err != nil { + return err + } + + if len(networkIfList.Items) == 0 { + return fmt.Errorf("no NetworkInterfaces found in namespace %s", ns) + } + + for _, networkIf := range networkIfList.Items { + for _, ownerRef := range networkIf.OwnerReferences { + if ownerRef.Kind == virtualMachineKind && ownerRef.Name == vmName { + if len(networkIf.Status.IPConfigs) == 0 { + return fmt.Errorf("no IPConfigs found for NetworkInterface %s", networkIf.Name) + } + + res = make([]NetworkProviderInfo, len(networkIf.Status.IPConfigs)) + for i, ipConfig := range networkIf.Status.IPConfigs { + res[i] = NetworkProviderInfo{ + NetworkType: consts.VDSNetworkType, + IPv4: ipConfig.IP, + SubnetMask: ipConfig.SubnetMask, + Gateway: ipConfig.Gateway, + } + } + + return nil + } + } + } + + return fmt.Errorf("no NetworkInterface found for VirtualMachine %s", vmName) + }, config.GetIntervals("default", "wait-virtual-machine-vmip")...).Should(Succeed()) + + return res +} + +func waitForNSXVirtualNetworkIf(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName string) []NetworkProviderInfo { + var res []NetworkProviderInfo + + Eventually(func() error { + vnetIfList := &ncpv1alpha1.VirtualNetworkInterfaceList{} + + err := client.List(ctx, vnetIfList, ctrlclient.InNamespace(ns)) + if err != nil { + return err + } + + if len(vnetIfList.Items) == 0 { + return fmt.Errorf("no VirtualNetworkInterfaces found in namespace %s", ns) + } + + for _, vnetIf := range vnetIfList.Items { + for _, ownerRef := range vnetIf.OwnerReferences { + if ownerRef.Kind == virtualMachineKind && ownerRef.Name == vmName { + if len(vnetIf.Status.IPAddresses) == 0 { + return fmt.Errorf("no IPAddresses found for VirtualNetworkInterface %s", vnetIf.Name) + } + + res = make([]NetworkProviderInfo, len(vnetIf.Status.IPAddresses)) + for i, ipAddr := range vnetIf.Status.IPAddresses { + res[i] = NetworkProviderInfo{ + NetworkType: consts.NSXNetworkType, + IPv4: ipAddr.IP, + SubnetMask: ipAddr.SubnetMask, + Gateway: ipAddr.Gateway, + } + } + + return nil + } + } + } + + return fmt.Errorf("no VirtualNetworkInterface found for VirtualMachine %s", vmName) + }, config.GetIntervals("default", "wait-virtual-machine-vmip")...).Should(Succeed()) + + return res +} + +func waitForSubnetPort(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName string) []NetworkProviderInfo { + var res []NetworkProviderInfo + + Eventually(func() error { + subnetPortList := &vpcv1alpha1.SubnetPortList{} + + err := client.List(ctx, subnetPortList, ctrlclient.InNamespace(ns)) + if err != nil { + return err + } + + if len(subnetPortList.Items) == 0 { + return fmt.Errorf("no SubnetPort found in namespace %s", ns) + } + + for _, subnetPort := range subnetPortList.Items { + for _, ownerRef := range subnetPort.OwnerReferences { + if ownerRef.Kind == virtualMachineKind && ownerRef.Name == vmName { + if len(subnetPort.Status.NetworkInterfaceConfig.IPAddresses) == 0 { + return fmt.Errorf("no IPAddresses found for SubnetPort %s", subnetPort.Name) + } + + res = make([]NetworkProviderInfo, len(subnetPort.Status.NetworkInterfaceConfig.IPAddresses)) + for i, ipAddr := range subnetPort.Status.NetworkInterfaceConfig.IPAddresses { + ip, ipNet, err := net.ParseCIDR(ipAddr.IPAddress) + if err != nil || ipNet == nil { + return fmt.Errorf("failed to parse CIDR from SubnetPort IPAddress %s", ipAddr.IPAddress) + } + + res[i] = NetworkProviderInfo{ + NetworkType: consts.VPCNetworkType, + IPv4: ip.String(), + SubnetMask: net.IP(ipNet.Mask).String(), + Gateway: ipAddr.Gateway, + } + } + + return nil + } + } + } + + return fmt.Errorf("no SubnetPort found for VirtualMachine %s", vmName) + }, config.GetIntervals("default", "wait-virtual-machine-vmip")...).Should(Succeed()) + + return res +} + +// WaitForVMNetworkProviderInfo waits for the network provider (NetworkInterface in VDS, VirtualNetworkInterface in NSX, Subnetport in VPC) +// to be created and contain the network information for the given VM. +func WaitForVMNetworkProviderInfo(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, vmName string) []NetworkProviderInfo { + var res []NetworkProviderInfo + + if framework.NetworkTopologyIs(config.InfraConfig.NetworkingTopology, consts.VDS) { + By("Getting VM network provider IP from Net-Operator's networkinterfaces in VDS") + + res = waitForVDSNetworkIf(ctx, config, client, ns, vmName) + } else if framework.NetworkTopologyIs(config.InfraConfig.NetworkingTopology, consts.NSX) { + if IsNetworkNsxtVPC(ctx, client, config) { + By("Getting VM network provider IP from NSX Operator's SubnetPort in NSX") + + res = waitForSubnetPort(ctx, config, client, ns, vmName) + } else { + By("Getting VM network provider IP from NCP's VirtualNetworkInterfaces in NSX") + + res = waitForNSXVirtualNetworkIf(ctx, config, client, ns, vmName) + } + } + + return res +} + +func WaitOnVirtualMachineGroupCondition( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, groupName string, + expectedCondition metav1.Condition) { + Eventually(func(g Gomega) bool { + vmg, err := utils.GetVirtualMachineGroup(ctx, client, ns, groupName) + g.Expect(err).ToNot(HaveOccurred()) + + for _, c := range vmg.Status.Conditions { + if c.Type == expectedCondition.Type { + g.Expect(c.Status).Should(Equal(expectedCondition.Status)) + + if expectedCondition.Reason != "" { + g.Expect(c.Reason).Should(Equal(expectedCondition.Reason)) + } + + return true + } + } + + return false + }, config.GetIntervals("default", "wait-virtual-machine-group-condition-update")...).Should(BeTrue(), "Timed out waiting for Condition: %+v on VirtualMachineGroup: %q", expectedCondition, groupName) +} + +func WaitOnVirtualMachineGroupMemberCondition( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, groupName, memberName, memberKind string, + expectedCondition metav1.Condition) { + Eventually(func(g Gomega) error { + vmg, err := utils.GetVirtualMachineGroup(ctx, client, ns, groupName) + g.Expect(err).ToNot(HaveOccurred()) + + for _, m := range vmg.Status.Members { + if m.Name == memberName && m.Kind == memberKind { + for _, c := range m.Conditions { + if c.Type == expectedCondition.Type { + g.Expect(c.Status).Should(Equal(expectedCondition.Status)) + + if expectedCondition.Reason != "" { + g.Expect(c.Reason).Should(Equal(expectedCondition.Reason)) + } + + return nil + } + } + + return fmt.Errorf("condition type %s not found for member %s/%s in VirtualMachineGroup %s", expectedCondition.Type, memberKind, memberName, groupName) + } + } + + return fmt.Errorf("member %s/%s not found in VirtualMachineGroup %s", memberKind, memberName, groupName) + }, config.GetIntervals("default", "wait-virtual-machine-group-condition-update")...).Should(Succeed(), "Timed out waiting for member %s/%s condition: %+v on VirtualMachineGroup: %s", memberKind, memberName, expectedCondition, groupName) +} + +func WaitForVirtualMachineGroupToBeDeleted(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, groupName string) { + Eventually(func(g Gomega) { + _, err := utils.GetVirtualMachineGroup(ctx, client, ns, groupName) + g.Expect(err).To(HaveOccurred()) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }, config.GetIntervals("default", "wait-virtual-machine-group-deletion")...).Should(Succeed(), "Timed out waiting for VirtualMachineGroup %s to be deleted", groupName) +} + +func VerifyVirtualMachineSnapshotDeleted( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, name string) { + GinkgoHelper() + + Eventually(func(g Gomega) bool { + _, err := utils.GetVirtualMachineSnapshot(ctx, client, ns, name) + return apierrors.IsNotFound(err) + }, config.GetIntervals("default", "wait-virtual-machine-snapshot-deletion")...).Should(BeTrue(), + "Timed out waiting for VirtualMachineSnapshot to be deleted") +} + +func VerifyVirtualMachineSnapshotCondition( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, name string, + powerState vmopv1a5.VirtualMachinePowerState, + quiesced bool, + children []vmopv1a5.VirtualMachineSnapshotReference, +) { + GinkgoHelper() + + var lastConditions []metav1.Condition + Eventually(func(g Gomega) bool { + vmSnapshot, err := utils.GetVirtualMachineSnapshot(ctx, client, ns, name) + if err != nil { + e2eframework.Logf("get vm snapshot error: %v", err) + return false + } + + if !reflect.DeepEqual(lastConditions, vmSnapshot.Status.Conditions) { + e2eframework.Logf("VirtualMachineSnapshot %s Conditions changed: %v", name, vmSnapshot.Status.Conditions) + lastConditions = vmSnapshot.Status.Conditions + } + + for _, condition := range vmSnapshot.Status.Conditions { + if condition.Type == vmopv1a5.VirtualMachineSnapshotReadyCondition { + return condition.Status == metav1.ConditionTrue + } + } + + return false + }, config.GetIntervals("default", "wait-virtual-machine-snapshot-condition")...).Should(BeTrue(), + "Timed out waiting for VirtualMachineSnapshot to be ready, current conditions: %v", lastConditions) + + Eventually(func(g Gomega) { + vmSnapshot, err := utils.GetVirtualMachineSnapshot(ctx, client, ns, name) + g.Expect(err).To(Succeed()) + g.Expect(vmSnapshot.Status.UniqueID).NotTo(BeEmpty()) + g.Expect(vmSnapshot.Status.PowerState).To(Equal(powerState), + "PowerState is not equal to %v", powerState) + g.Expect(vmSnapshot.Status.Quiesced).To(Equal(quiesced), + "Quiesced is not equal to %v", quiesced) + g.Expect(vmSnapshot.Status.Children).To(HaveLen(len(children)), + "Children is not equal to %v", children) + g.Expect(vmSnapshot.Status.Children).To(ConsistOf(children), + "Children is not equal to %v", children) + g.Expect(vmSnapshot.Status.Storage).NotTo(BeNil(), "Storage is nil") + g.Expect(vmSnapshot.Status.Storage.Used).NotTo(BeNil(), "Used is nil") + g.Expect(vmSnapshot.Status.Storage.Requested).NotTo(BeNil(), "Requested is nil") + }, config.GetIntervals("default", "wait-virtual-machine-snapshot-condition")...).Should(Succeed(), + "Timed out waiting for VirtualMachineSnapshot's condition") +} + +func VerifySnapshotStatusOnVirtualMachine( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, name string, + currentSnapshot *vmopv1a5.VirtualMachineSnapshotReference, + rootSnapshots []vmopv1a5.VirtualMachineSnapshotReference, + powerState vmopv1a5.VirtualMachinePowerState, +) { + GinkgoHelper() + + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, client, ns, name) + g.Expect(err).To(Succeed()) + g.Expect(vm.Spec.CurrentSnapshotName).To(BeEmpty(), + "spec.CurrentSnapshotName should be empty") + g.Expect(vm.Status.CurrentSnapshot).To(Equal(currentSnapshot), + "CurrentSnapshot is not equal to %v", currentSnapshot) + g.Expect(vm.Status.RootSnapshots).To(HaveLen(len(rootSnapshots)), + "RootSnapshots's length is not equal to %v", len(rootSnapshots)) + g.Expect(vm.Status.RootSnapshots).To(ConsistOf(rootSnapshots), + "RootSnapshots is not equal to %v", rootSnapshots) + g.Expect(vm.Status.PowerState).To(Equal(powerState), + "PowerState is not equal to %v", powerState) + }, config.GetIntervals("default", "wait-virtual-machine-snapshot-related-resource")...).Should(Succeed(), + "Timed out waiting for VirtualMachineSnapshot's related resource") +} + +func VerifyVirtualMachineRestartMode( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + ns, name string, + restartMode vmopv1a5.VirtualMachinePowerOpMode, +) { + GinkgoHelper() + + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, client, ns, name) + g.Expect(err).To(Succeed()) + g.Expect(vm.Spec.RestartMode).To(Equal(restartMode)) + }, config.GetIntervals("default", "wait-virtual-machine-snapshot-related-resource")...).Should(Succeed(), + "Timed out waiting for restart mode of VM") +} + +func VerifyVMSnapshotDeletion( + ctx context.Context, + client ctrlclient.Client, + vmSvcE2EConfig *config.E2EConfig, + params manifestbuilders.VirtualMachineSnapshotYaml, +) { + GinkgoHelper() + + Eventually(func() bool { + _, err := utils.GetVirtualMachineSnapshot(ctx, client, params.Namespace, params.Name) + return apierrors.IsNotFound(err) + }, vmSvcE2EConfig.GetIntervals("default", "wait-virtual-machine-snapshot-deletion")...).Should(BeTrue(), + "Timed out waiting for VirtualMachineSnapshot to be deleted") +} + +func VerifyVMSnapshotQuotaUsage( + ctx context.Context, + client ctrlclient.Client, + vmSvcE2EConfig *config.E2EConfig, + ns string, + spuName string, + snapshots ...string, +) { + GinkgoHelper() + + Eventually(func(g Gomega) { + usageTotal := resource.NewQuantity(0, resource.BinarySI) + + for _, snapshot := range snapshots { + snapshotCR, err := utils.GetVirtualMachineSnapshot(ctx, client, ns, snapshot) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(snapshotCR.Status).NotTo(BeNil()) + g.Expect(snapshotCR.Status.Storage).NotTo(BeNil()) + g.Expect(snapshotCR.Status.Storage.Used).NotTo(BeNil()) + usageTotal.Add(*snapshotCR.Status.Storage.Used) + } + + storagePolicyUsage, err := utils.GetStoragePolicyUsage(ctx, client, ns, spuName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(storagePolicyUsage.Status.ResourceTypeLevelQuotaUsage.Used.Value()).To(Equal(usageTotal.Value()), + fmt.Sprintf("expect StoragePolicyUsage %s to have correct used capacity ", spuName)) + }, vmSvcE2EConfig.GetIntervals("default", "wait-virtual-machine-snapshot-quota-usage")...).Should(Succeed()) +} + +func EnsureVMSnapshotDeleted( + ctx context.Context, + client ctrlclient.Client, + vmSvcE2EConfig *config.E2EConfig, + params manifestbuilders.VirtualMachineSnapshotYaml, +) { + GinkgoHelper() + + vmSnapshotYaml := manifestbuilders.GetVirtualMachineSnapshotYaml(params) + e2eframework.Logf("Delete VirtualMachineSnapshot:\n%v", string(vmSnapshotYaml)) + Eventually(func() bool { + err := utils.DeleteVirtualMachineSnapshot(ctx, client, params.Namespace, params.Name) + if err != nil && apierrors.IsNotFound(err) { + return true + } + + return false + }, vmSvcE2EConfig.GetIntervals("default", "wait-virtual-machine-snapshot-deletion")...).Should(BeTrue()) +} + +func VerifyVMDeleted( + ctx context.Context, + client ctrlclient.Client, + vmSvcE2EConfig *config.E2EConfig, + ns, name string) { + Eventually(func() bool { + err := utils.DeleteVirtualMachineA5(ctx, client, ns, name) + if err != nil && apierrors.IsNotFound(err) { + return true + } + + return false + }, vmSvcE2EConfig.GetIntervals("default", "wait-virtual-machine-deletion")...).Should(BeTrue(), + "Timed out waiting for VirtualMachine to be deleted") +} diff --git a/test/e2e/vmservice/skipper/skipper.go b/test/e2e/vmservice/skipper/skipper.go new file mode 100644 index 000000000..8ce5dc7e5 --- /dev/null +++ b/test/e2e/vmservice/skipper/skipper.go @@ -0,0 +1,111 @@ +package skipper + +import ( + "context" + "os" + "strconv" + + . "github.com/onsi/gomega" + "golang.org/x/crypto/ssh" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/appple2e/util" + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + e2essh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" +) + +func SkipUnlessHAFSSEnabled(ctx context.Context, client ctrlclient.Client, config *config.E2EConfig) { + skipUnlessFSSEnabled(ctx, client, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSHA")) +} + +func SkipUnlessVMImageRegistryFSSEnabled(ctx context.Context, client ctrlclient.Client, config *config.E2EConfig) { + skipUnlessFSSEnabled(ctx, client, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMImageRegistry")) +} + +func SkipUnlessNamespacedVMClassFSSEnabled(ctx context.Context, client ctrlclient.Client, config *config.E2EConfig) { + skipUnlessFSSEnabled(ctx, client, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSNamespacedVMClass")) +} + +func SkipUnlessV1a2FSSEnabled(ctx context.Context, client ctrlclient.Client, config *config.E2EConfig) { + skipUnlessFSSEnabled(ctx, client, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSV1alpha2")) +} + +func SkipUnlessNetworkingIsVPC(ctx context.Context, client ctrlclient.Client, config *config.E2EConfig) { + if !vmoperator.IsNetworkNsxtVPC(ctx, client, config) { + framework.SkipInternalf(1, "skip if not VPC networking environment") + } +} + +func SkipUnlessWindowsFSSEnabled(ctx context.Context, client ctrlclient.Client, config *config.E2EConfig) { + skipUnlessFSSEnabled(ctx, client, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSWindowsSysprep")) +} + +func skipUnlessFSSEnabled(ctx context.Context, client ctrlclient.Client, deploymentNS, deploymentName, command, fss string) { + envs, err := utils.GetCommandEnvVars(ctx, client, deploymentNS, deploymentName, command) + Expect(err).To(Succeed(), "%q FSS cannot not be fetched for command %q", fss, command) + + mmEnabled, _ := strconv.ParseBool(envs[fss]) + if !mmEnabled { + framework.SkipInternalf(1, "%q FSS is not enabled for command %q", fss, command) + } +} + +func SkipUnlessInfraIs(clusterInfra, requiredInfra string) { + if !framework.InfraIs(clusterInfra, requiredInfra) { + framework.SkipInternalf(1, "required infrastructure environment: %s for test does not match with provided infrastructure environment:%s", clusterInfra, requiredInfra) + } +} + +func SkipIfStretchSupervisorIsEnabled() { + if os.Getenv("STRETCHED_SUPERVISOR") == "true" { + framework.SkipInternalf(1, "skip the test due to StretchSupervisor is enabled") + } +} + +func SkipUnlessStretchSupervisorIsEnabled() { + // Skip the test for 1CP and 1Worker if the stretch supervisor is enabled + if os.Getenv("STRETCHED_SUPERVISOR") != "true" { + framework.SkipInternalf(1, "skip the test due to StretchSupervisor is not enabled") + } +} + +func SkipUnlessSupervisorCapabilityEnabled(ctx context.Context, vmSvcClusterProxy *common.VMServiceClusterProxy, capabilityName string) { + sshCommandRunner, _ := e2essh.NewSSHCommandRunner( + vcenter.GetVCPNIDFromKubeconfigFile(ctx, vmSvcClusterProxy.GetKubeconfigPath()), + vcenter.VCSSHPort, + testbed.RootUsername, + []ssh.AuthMethod{ + ssh.Password(testbed.RootPassword), + }, + ) + isAsyncSvUpgradeEnabled, _ := util.IsFSSEnabled(sshCommandRunner, utils.SupervisorAsyncUpgradeFSS) + + if !utils.IsSupervisorCapabilityEnabled( + ctx, + vmSvcClusterProxy.GetClientSet(), + vmSvcClusterProxy.GetDynamicClient(), + capabilityName, + isAsyncSvUpgradeEnabled, + ) { + framework.SkipInternalf(1, "skip the test due to Supervisor capability %q is not enabled", capabilityName) + } +} + +func SkipUnlessSnapshotFSSEnabled(ctx context.Context, vmSvcClusterProxy *common.VMServiceClusterProxy, config *config.E2EConfig) { + sshCommandRunner, _ := e2essh.NewSSHCommandRunner( + vcenter.GetVCPNIDFromKubeconfigFile(ctx, vmSvcClusterProxy.GetKubeconfigPath()), + vcenter.VCSSHPort, + testbed.RootUsername, + []ssh.AuthMethod{ssh.Password(testbed.RootPassword)}) + isVMVMSnapshotEnabled, _ := util.IsFSSEnabled(sshCommandRunner, utils.SupervisorVMSnapshotFSS) + + if !isVMVMSnapshotEnabled { + framework.SkipInternalf(1, "skip the test due to %s is disabled.", utils.SupervisorVMSnapshotFSS) + } +} diff --git a/test/e2e/vmservice/vmservice/devops/namespaces.go b/test/e2e/vmservice/vmservice/devops/namespaces.go new file mode 100644 index 000000000..48ff93e5b --- /dev/null +++ b/test/e2e/vmservice/vmservice/devops/namespaces.go @@ -0,0 +1,169 @@ +// Copyright (c) 2023 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package devops + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/dcli" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/kubectl" + libssh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/testutils" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type DevOpsSpecInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *e2eConfig.E2EConfig + WCPClient wcp.WorkloadManagementAPI + WCPNamespaceName string +} + +const ( + devopsSpecName = "devops-namespaces" + randomSSOUserName = "devops-sso-user" + randomSSOUserPassword = "Password!23" +) + +func DevOpsSpec(ctx context.Context, inputGetter func() DevOpsSpecInput) { + var ( + input DevOpsSpecInput + config *e2eConfig.E2EConfig + wcpClient wcp.WorkloadManagementAPI + k8sClient ctrlclient.Client + kubeconfigPath string + namespacedVMClassFSSEnabled bool + vmImageRegistryEnabled bool + namespace string + sshCommandRunner libssh.SSHCommandRunner + supervisorClusterIP string + vCenterAdminCreds dcli.VCenterUserCredentials + user *vcenter.User + userKubeconfigPath string + ) + + BeforeEach(func() { + By("Set up infrastructure related configs") + + input = inputGetter() + Expect(input.Config).NotTo(BeNil(), "Invalid argument. input.Config can't be nil when calling %s spec", devopsSpecName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", devopsSpecName) + + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + Expect(input.ClusterProxy).NotTo(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling %s spec", devopsSpecName) + Expect(input.WCPClient).NotTo(BeNil(), "Invalid argument. input.WCPClient can't be nil when calling %s spec", devopsSpecName) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", devopsSpecName) + config = input.Config + wcpClient = input.WCPClient + namespace = input.WCPNamespaceName + k8sClient = input.ClusterProxy.GetClient() + kubeconfigPath = input.ClusterProxy.GetKubeconfigPath() + vCenterAdminCreds = dcli.VCenterUserCredentials{Username: testbed.AdminUsername, Password: testbed.AdminPassword} + namespacedVMClassFSSEnabled = utils.IsFssEnabled(ctx, k8sClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSNamespacedVMClass")) + vmImageRegistryEnabled = utils.IsFssEnabled(ctx, k8sClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMImageRegistry")) + // Create SSO user + sshCommandRunner, _, supervisorClusterIP = testutils.GetHelpersFromKubeconfig(ctx, kubeconfigPath) + user = vcenter.NewUser(randomSSOUserName, randomSSOUserPassword).WithAdminCreds(vCenterAdminCreds).WithSSHCommandRunner(sshCommandRunner) + kubectlPlugin := testutils.CreateUserAndLogin(user, supervisorClusterIP, "", "") + userKubeconfigPath = kubectlPlugin.KubeconfigPath() + }) + + Context("When Devops have view permission to namespace", func() { + BeforeEach(func() { + // Grant SSO user with view only access to the supervisor namespace. + testutils.SetUserPermissionsOnNamespace(wcpClient, user, wcp.ViewAccessType, namespace) + }) + + AfterEach(func() { + // Remove the user's permissions on the namespace. + err := wcpClient.RemoveNamespacePermissions(wcp.Principal{Type: wcp.UserSubjectType, Name: user.Credentials.Username, Domain: "vsphere.local"}, namespace) + Expect(err).NotTo(HaveOccurred()) + // Delete the SSO user. + vcenter.DeleteUserOrFail(user) + }) + + It("Should have correct view permission", Label("smoke"), func() { + By("able to get resources within assigned namespace") + + if namespacedVMClassFSSEnabled { + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "virtualmachineclass", "-n", namespace) + } else { + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "virtualmachineclassbinding", "-n", namespace) + } + + if vmImageRegistryEnabled { + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "clustervirtualmachineimage") + } + + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "virtualmachineimage", "-n", namespace) + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "virtualmachine", "-n", namespace) + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "virtualmachineservices", "-n", namespace) + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "virtualmachinepublishrequests", "-n", namespace) + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "webconsolerequests", "-n", namespace) + + By("not able to get resources outside assigned namespace") + + if namespacedVMClassFSSEnabled { + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "get", "virtualmachineclass", "-A") + } else { + // VM Class is cluster scoped resource when WCP_Namespaced_VM_Class disabled + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "virtualmachineclass") + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "get", "virtualmachineclassbinding", "-A") + } + + if vmImageRegistryEnabled { + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "clustervirtualmachineimage") + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "get", "virtualmachineimage", "-A") + } else { + // VM Image is cluster scoped resource when WCP_VM_Image_Registry disabled + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "get", "virtualmachineimage") + } + + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "get", "virtualmachine", "-A") + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "get", "virtualmachineservices", "-A") + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "get", "virtualmachinepublishrequests", "-A") + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "get", "webconsolerequests", "-A") + }) + }) + + Context("When Devops have edit permission to namespace", func() { + BeforeEach(func() { + // Grant SSO user with edit access to the supervisor namespace. + testutils.SetUserPermissionsOnNamespace(wcpClient, user, wcp.EditAccessType, namespace) + }) + + AfterEach(func() { + // Remove the user's permissions on the namespace. + err := wcpClient.RemoveNamespacePermissions(wcp.Principal{Type: wcp.UserSubjectType, Name: user.Credentials.Username, Domain: "vsphere.local"}, namespace) + Expect(err).NotTo(HaveOccurred()) + // Delete the SSO user. + vcenter.DeleteUserOrFail(user) + }) + + It("Should have correct edit permission ", func() { + By("able to create within assigned namespace") + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "create", "virtualmachine", "-n", namespace) + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "create", "virtualmachineservices", "-n", namespace) + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "create", "virtualmachinepublishrequests", "-n", namespace) + kubectl.AssertKubectlUserCan(ctx, userKubeconfigPath, "create", "webconsolerequests", "-n", namespace) + + By("not able to create outside assigned namespace") + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "create", "virtualmachine", "-A") + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "create", "virtualmachineservices", "-A") + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "create", "virtualmachinepublishrequests", "-A") + kubectl.AssertKubectlUserCannot(ctx, userKubeconfigPath, "create", "webconsolerequests", "-A") + }) + }) +} diff --git a/test/e2e/vmservice/vmservice/support_bundle.go b/test/e2e/vmservice/vmservice/support_bundle.go new file mode 100644 index 000000000..cc34510cd --- /dev/null +++ b/test/e2e/vmservice/vmservice/support_bundle.go @@ -0,0 +1,73 @@ +package vmservice + +import ( + "fmt" + "io" + "os/exec" + "strings" + + . "github.com/onsi/gomega" + + vmopv1a3 "github.com/vmware-tanzu/vm-operator/api/v1alpha3" + "k8s.io/kubernetes/test/e2e/framework" +) + +const supportBundleImage = "wcp-gc-docker-local.packages.vcfd.broadcom.net/utils/tkc-support-bundler:v1.2.0" + +func runSupportBundleCLI(vm *vmopv1a3.VirtualMachine, svKubeconfig, testName string) error { + output := fmt.Sprintf("${SUPPORT_BUNDLE_DIR}/%s-%s", vm.Name, testName) + + err := runCommand("mkdir -p " + output) + if err != nil { + return fmt.Errorf("unable to make output directory: %w", err) + } + // Note: tkc-support-bundler is specific to TKC. If a VM-specific tool exists, replace this command. + cmdString := fmt.Sprintf("tkc-support-bundler create -k %s -o %s -c %s -n %s -i %s", svKubeconfig, output, vm.Name, vm.Namespace, supportBundleImage) + framework.Logf("Running %s", cmdString) + + err = runCommand(cmdString) + if err != nil { + return fmt.Errorf("unable to collect support-bundle: %w", err) + } + + return nil +} + +func CollectSupportBundle(vm *vmopv1a3.VirtualMachine, svKubeconfig, testName string) { + testName = strings.ReplaceAll(testName, " ", "-") + + supportBundleErr := runSupportBundleCLI(vm, svKubeconfig, testName) + if supportBundleErr != nil { + framework.Logf("Cannot collect support bundle \n %s\n", supportBundleErr.Error()) + } +} + +func runCommand(cmdString string) error { + cmd := exec.Command("/bin/sh", "-c", cmdString) //nolint:gosec // G204: E2E helper runs shell for support-bundle CLI + stdout, err := cmd.StdoutPipe() + Expect(err).ToNot(HaveOccurred()) + stderr, err := cmd.StderrPipe() + Expect(err).ToNot(HaveOccurred()) + + framework.Logf("Starting command %s", cmdString) + + err = cmd.Start() + Expect(err).NotTo(HaveOccurred()) + + stdoutBytes, _ := io.ReadAll(stdout) + if len(stdoutBytes) > 0 { + framework.Logf("Command standard output: %v\n", string(stdoutBytes)) + } + + stderrBytes, _ := io.ReadAll(stderr) + if len(stderrBytes) > 0 { + framework.Logf("Command standard error: %v\n", string(stderrBytes)) + } + + framework.Logf("Waiting for the command to finish") + + err = cmd.Wait() + Expect(err).NotTo(HaveOccurred()) + + return err +} diff --git a/test/e2e/vmservice/vmservice/util.go b/test/e2e/vmservice/vmservice/util.go new file mode 100644 index 000000000..3d3f27803 --- /dev/null +++ b/test/e2e/vmservice/vmservice/util.go @@ -0,0 +1,1903 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmservice + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha1" + "crypto/tls" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "net/url" + "os" + "reflect" + "slices" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vapi/tags" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + "golang.org/x/crypto/ssh" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/kubernetes/test/e2e/framework" + + e2eframework "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + vmopv1a2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + cnsunregistervolumev1alpha1 "github.com/vmware-tanzu/vm-operator/external/vsphere-csi-driver/api/cnsunregistervolume/v1alpha1" + e2essh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" +) + +const ( + trueString = "true" +) + +const ( + VMClassNotFound = "Server error: com.vmware.vapi.std.errors.NotFound" + VMClassInstanceStorage = "e2etest-instance-storage-small" + VMClassE1000 = "e2etest-vmclass-config-e1000" + VMClassMockVGPUConfigSpec = "e2etest-vmclass-config-mockvgpu" + VMClassGPUnDDPIOConfigSpec = "e2etest-vmclass-config-gpuddpio" + VMClassVMX22 = "e2etest-vmclass-22" + VMClassReservedVGPU = "e2etest-vmclass-mockvgpu-reserved" + VMFinalizerName = "vmoperator.vmware.com/virtualmachine" + VMFinalizerNameDeprecated = "virtualmachine.vmoperator.vmware.com" +) + +var ( + vmClassFunctions = map[string]any{ + "e2etest-best-effort-small": CreateSpecE2eTestBestEffortSmall, + "e2etest-guaranteed-xsmall": CreateSpecE2eTestGuaranteedXSmall, + "e2etest-guaranteed-medium": CreateSpecE2eTestGuaranteedMedium, + VMClassE1000: CreateSpecE2eVMClassE1000, + VMClassInstanceStorage: CreateSpecInstanceStorageSmall, + "best-effort-small": CreateSpecBestEffortSmall, + "guaranteed-large": CreateGuaranteedLarge, + "customize": CreateSpecCustomizedVMClass, + VMClassMockVGPUConfigSpec: CreateSpecE2eVMClassMockVGPUConfigSpec, + VMClassGPUnDDPIOConfigSpec: CreateSpecE2eVMClassGPUnDDPIOConfigSpec, + VMClassVMX22: CreateSpecE2eVMClassVMX22, + VMClassReservedVGPU: CreateSpecE2eVMClassReservedVGPU, + } +) + +func EnsureNamespaceHasAccess(wcpClient wcp.WorkloadManagementAPI, vmClassID, ns string) error { + namespaceInfo, err := wcpClient.GetNamespace(ns) + if err != nil { + return err + } + + if !slices.Contains(namespaceInfo.VMServiceSpec.VMClasses, vmClassID) { + framework.Logf("VMClass %s is not accessible to namespace %s, adding it", vmClassID, ns) + namespaceInfo.VMServiceSpec.VMClasses = append(namespaceInfo.VMServiceSpec.VMClasses, vmClassID) + + updateSpec := wcp.NamespaceUpdateVMserviceSpec{VMClasses: &namespaceInfo.VMServiceSpec.VMClasses} + + err := wcpClient.UpdateNamespaceVMServiceSpec(ns, updateSpec) + if err != nil { + framework.Logf("Error update namespace %s VMService spec. Err: %s", ns, err) + return err + } + } + + // Wait for namespaceInfo.configStatus to be RUNNING which ensures successful creation of + // VirtualMachineClass/VirtualMachineClassBinding in the namespace depending on the scope of VirtualMachineClass + wcp.WaitForNamespaceReady(wcpClient, ns) + + return nil +} + +func VerifyVMClassCreate(wcpClient wcp.WorkloadManagementAPI, createSpec wcp.VMClassSpec, expectedSpec wcp.VMClassSpec) { + Expect(wcpClient.CreateVMClass(createSpec)).To(Succeed()) + + var ( + vmClass wcp.VMClassInfo + err error + ) + Eventually(func(g Gomega) { + vmClass, err = wcpClient.GetVMClassInfo(createSpec.ID) + g.Expect(err).ToNot(HaveOccurred()) + }, 5*time.Minute, 5*time.Second).Should(Succeed(), "failed to get created vmClass info %w", err) + VerifyVMClassSpec(vmClass.VMClassSpec, expectedSpec) +} + +func VerifyVMClassDeletion(wcpClient wcp.WorkloadManagementAPI, vmClassID string) { + Expect(wcpClient.DeleteVMClass(vmClassID)).To(Succeed()) + + // Eventually VMClass should be deleted. + Eventually(func(g Gomega) { + _, err := wcpClient.GetVMClassInfo(vmClassID) + g.Expect(err).To(HaveOccurred()) + + var dcliErr wcp.DcliError + g.Expect(errors.As(err, &dcliErr)).Should(BeTrue()) + g.Expect(dcliErr.Response()).To(ContainSubstring(VMClassNotFound)) + }, 5*time.Minute, 10*time.Second).Should(Succeed(), "failed to delete vmClass ", vmClassID) +} + +func VerifyVMClassSpec(actualVMClassSpec wcp.VMClassSpec, expectedVMClassSpec wcp.VMClassSpec) { + if expectedVMClassSpec.Description == nil { + emptyString := "" + expectedVMClassSpec.Description = &emptyString + } + + if expectedVMClassSpec.ConfigSpec == nil { + // wcpsvc sets the ConfigSpec when nil + expectedVMClassSpec.ConfigSpec = actualVMClassSpec.ConfigSpec + } + + Expect(actualVMClassSpec).To(BeComparableTo(expectedVMClassSpec), "unexpected class %s", actualVMClassSpec.ID) +} + +// GenerateVMClassSpecFunction returns a spec generating function based on vmClassName using map 'vmClassFunctions'. +func GenerateVMClassSpecFunction(vmClassName string, params ...any) (result any, err error) { + f := reflect.ValueOf(vmClassFunctions[vmClassName]) + if len(params) != f.Type().NumIn() { + err = errors.New("number of given parameters is out of index") + return + } + + in := make([]reflect.Value, len(params)) + for k, param := range params { + in[k] = reflect.ValueOf(param) + } + + res := f.Call(in) + result = res[0].Interface() + + return +} + +func CreateSpecE2eTestGuaranteedXSmall() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: "e2etest-guaranteed-xsmall", + CPUCount: new(2), + MemoryMB: new(512), + } +} + +func CreateSpecE2eTestBestEffortSmall() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: "e2etest-best-effort-small", + CPUCount: new(2), + MemoryMB: new(1024), + } +} + +func CreateSpecE2eTestGuaranteedMedium() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: "e2etest-guaranteed-medium", + CPUCount: new(2), + MemoryMB: new(8192), + } +} + +func CreateSpecE2eTestGuaranteedXSmallVirtualDevicesVGPUs() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: "e2etest-guaranteed-xsmall-virtual-devices-vgpus", + CPUCount: new(2), + MemoryMB: new(512), + CPUReservation: new(100), + MemoryReservation: new(100), + Devices: wcp.VirtualDevices{ + VGPUDevices: []wcp.VGPUDevice{ + { + ProfileName: "mockup-vmiop", + }, + { + ProfileName: "mockup-vmiop", + }, + }, + }, + } +} + +func CreateSpecE2eTestGuaranteedXSmallCustomizedVirtualDevices(vmClassName string, virtualDevices wcp.VirtualDevices) wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: vmClassName, + CPUCount: new(2), + MemoryMB: new(512), + CPUReservation: new(100), + MemoryReservation: new(100), + Devices: virtualDevices, + } +} + +// CreateSpecBestEffortSmall creates the best-effort-small VM class spec. +func CreateSpecBestEffortSmall() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: "best-effort-small", + CPUCount: new(2), + MemoryMB: new(4096), + } +} + +func CreateGuaranteedLarge() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: "guaranteed-large", + CPUCount: new(4), + MemoryMB: new(16 * 1024), + CPUReservation: new(100), + MemoryReservation: new(100), + } +} + +// CreateSpecCustomizedVMClass creates a customized VM class spec with given parameters. +func CreateSpecCustomizedVMClass(className string, cpuCount, memoryMB int, description string) wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: className, + CPUCount: &cpuCount, + MemoryMB: &memoryMB, + Description: &description, + } +} + +// CreateSpecInstanceStorageSmall creates the instance volume VM class spec. +func CreateSpecInstanceStorageSmall(params ...any) wcp.VMClassSpec { + Expect(params).ToNot(BeEmpty()) + + return wcp.VMClassSpec{ + ID: VMClassInstanceStorage, + CPUCount: new(2), + MemoryMB: new(4096), + InstanceStorage: wcp.InstanceStorage{ + StoragePolicy: params[0].(string), + Volumes: []wcp.InstanceStorageVolume{ + { + Size: 2048, + }, + { + Size: 1024, + }, + }, + }, + } +} + +// CreateSpecE2eVMClassE1000 creates a vm class spec with a e1000 nic. +func CreateSpecE2eVMClassE1000() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: VMClassE1000, + CPUCount: new(2), + MemoryMB: new(4096), + ConfigSpec: &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualE1000{ + VirtualEthernetCard: types.VirtualEthernetCard{ + VirtualDevice: types.VirtualDevice{ + Key: -10000, + }, + }, + }, + }, + }, + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{ + Key: "hello-test-key", + Value: "hello-test-value", + }, + }, + }, + } +} + +func CreateSpecE2eVMClassVMX22() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: VMClassVMX22, + CPUCount: new(2), + MemoryMB: new(4096), + ConfigSpec: &types.VirtualMachineConfigSpec{ + Version: "vmx-22", + }, + } +} + +func CreateSpecE2eVMClassMockVGPUConfigSpec() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: VMClassMockVGPUConfigSpec, + CPUCount: new(2), + MemoryMB: new(512), + CPUReservation: new(100), + MemoryReservation: new(100), + ConfigSpec: &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Key: -20000, + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "mockup-vmiop", + }, + }, + }, + }, + }, + }, + } +} + +func CreateSpecE2eVMClassReservedVGPU() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: VMClassReservedVGPU, + CPUCount: new(1), + MemoryMB: new(1024), + ConfigSpec: &types.VirtualMachineConfigSpec{ + NumCPUs: int32(1), + MemoryMB: int64(1024), + CpuAllocation: &types.ResourceAllocationInfo{ + Reservation: new(int64(1000)), + Limit: new(int64(1000)), + }, + MemoryAllocation: &types.ResourceAllocationInfo{ + Reservation: new(int64(1024)), + Limit: new(int64(1024)), + }, + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Key: -20000, + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "mockup-vmiop", + }, + }, + }, + }, + }, + }, + } +} + +func CreateSpecE2eVMClassGPUnDDPIOConfigSpec() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: VMClassGPUnDDPIOConfigSpec, + CPUCount: new(2), + MemoryMB: new(4096), + CPUReservation: new(100), + MemoryReservation: new(100), + ConfigSpec: &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Key: -20000, + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "mockup-vmiop", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Key: -30000, + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "SampleLabel2", + }, + }, + }, + }, + }, + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{ + Key: "hello-test-key", + Value: "hello-test-value", + }, + }, + }, + } +} + +// CreateSpecE2eVMClassVTPMConfigSpec creates a vm class with a vTPM. +func CreateSpecE2eVMClassVTPMConfigSpec() wcp.VMClassSpec { + return wcp.VMClassSpec{ + ID: "e2etest-best-effort-small-with-vtpm", + CPUCount: new(2), + MemoryMB: new(4096), + ConfigSpec: &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualTPM{ + VirtualDevice: types.VirtualDevice{ + Key: -40000, + }, + }, + }, + }, + }, + } +} + +// EnsureVMClassPresent ensures the VM class exists in the WCP vcenter. +func EnsureVMClassPresent(wcpClient wcp.WorkloadManagementAPI, className string, params ...any) error { + _, err := wcpClient.GetVMClassInfo(className) + if err == nil { + return nil + } + + var dcliErr wcp.DcliError + Expect(errors.As(err, &dcliErr)).Should(BeTrue()) + Expect(dcliErr.Response()).Should(ContainSubstring(VMClassNotFound)) + + classSpec, err := GenerateVMClassSpecFunction(className, params...) + Expect(err).ShouldNot(HaveOccurred()) + + return wcpClient.CreateVMClass(classSpec.(wcp.VMClassSpec)) +} + +// EnsureVMClassAccess ensures the VM Class exists and the namespace has access to it. +func EnsureVMClassAccess(wcpClient wcp.WorkloadManagementAPI, className, namespace string) { + Expect(EnsureVMClassPresent(wcpClient, className)).To(Succeed()) + Expect(EnsureNamespaceHasAccess(wcpClient, className, namespace)).To(Succeed()) +} + +// VerifyCLAssociation ensures that the namespace has access to expected content libraries. +func VerifyCLAssociation(wcpClient wcp.WorkloadManagementAPI, ns string, cls []string) { + updateSpec := wcp.NamespaceUpdateVMserviceSpec{ContentLibraries: &cls} + err := wcpClient.UpdateNamespaceVMServiceSpec(ns, updateSpec) + Expect(err).ShouldNot(HaveOccurred()) + namespaceInfo, err := wcpClient.GetNamespace(ns) + Expect(err).ShouldNot(HaveOccurred()) + + expectedCls := namespaceInfo.VMServiceSpec.ContentLibraries + Expect(VerifyExpectedCls(expectedCls, cls)).Should(BeTrue()) +} + +// VerifyExpectedCls checks whether all the contentLibraries in cls are present in expectedCls. +// expectedCls contains tkgCl in addition to required contentLibraries. +func VerifyExpectedCls(expectedCls []string, cls []string) bool { + for _, cl := range cls { + found := slices.Contains(expectedCls, cl) + + if !found { + return false + } + } + + return true +} + +// CheckCLDisassociation verifies that the namespace doesn't have access to given content libraries. +func CheckCLDisassociation(wcpClient wcp.WorkloadManagementAPI, ns string, cls []string) { + namespaceInfo, err := wcpClient.GetNamespace(ns) + Expect(err).ShouldNot(HaveOccurred()) + + currentCls := namespaceInfo.VMServiceSpec.ContentLibraries + Expect(VerifyDetachedCls(currentCls, cls)).Should(BeTrue()) +} + +// VerifyDetachedCls checks whether all the contentLibraries in detachedCls are not present in cls. +func VerifyDetachedCls(cls []string, detachedCls []string) bool { + for _, deCl := range detachedCls { + if slices.Contains(cls, deCl) { + return false + } + } + + return true +} + +func VerifyConfigMapCreation(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, configMapName string) { + By("Verify that we have a ConfigMap CRD") + Eventually(func(g Gomega) { + cm, err := utils.GetConfigMap(ctx, svClusterClient, ns, configMapName) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(cm).ShouldNot(BeNil()) + }, config.GetIntervals("default", "wait-config-map-creation")...).Should(Succeed(), "Timed out waiting for ConfigMap %s to be created in namespace %s", configMapName, ns) +} + +func VerifySecretCreation(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, secretName string) { + By("Verify that we have a Secret CRD") + Eventually(func(g Gomega) { + cm, err := utils.GetSecret(ctx, svClusterClient, ns, secretName) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(cm).ShouldNot(BeNil()) + }, config.GetIntervals("default", "wait-secret-creation")...).Should(Succeed(), "Timed out waiting for Secret %s to be created in namespace %s", secretName, ns) +} + +func VerifySecurityPolicyCreation(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, sName string) { + By("Verify that we have a Security Policy CRD") + Eventually(func(g Gomega) { + sp, err := utils.GetSecurityPolicy(ctx, svClusterClient, ns, sName) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(sp).ShouldNot(BeNil()) + }, config.GetIntervals("default", "wait-security-policy-creation")...).Should(Succeed(), "Timed out waiting for SecurityPolicy %s to be created in namespace %s", sName, ns) +} + +func VerifySubnetOrSubnetSetCreation(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, name, kind string) { + By("Verify that we have a Subnet or SubnetSet CRD") + Eventually(func(g Gomega) { + switch kind { + case "Subnet": + subnet, err := utils.GetSubnet(ctx, svClusterClient, ns, name) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(subnet).ToNot(BeNil()) + case "SubnetSet": + subnetSet, err := utils.GetSubnetSet(ctx, svClusterClient, ns, name) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(subnetSet).ToNot(BeNil()) + default: + g.Expect(false).To(BeTrue(), "unknown kind: %s", kind) + } + }, config.GetIntervals("default", "wait-subnet-creation")...).Should(Succeed(), "Timed out waiting for %s %s to be created in namespace %s", kind, name, ns) +} + +func VerifyWebConsoleRequestCreation(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, webconsoleName string) { + By("Verify that we have a WebConsoleRequest CRD") + Eventually(func(g Gomega) { + webconsole, err := utils.GetVWebConsoleRequest(ctx, svClusterClient, ns, webconsoleName) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(webconsole).ShouldNot(BeNil()) + }, config.GetIntervals("default", "wait-virtual-machine-web-console-request-creation")...).Should(Succeed(), "Timed out waiting for WebConsoleRequest %s to be created in namespace %s", webconsoleName, ns) +} + +func VerifyVirtualMachineWebConsoleRequestCreation(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, vmWebconsoleName string) { + By("Verify that we have a VirtualMachineWebConsoleRequest CRD") + Eventually(func(g Gomega) { + vmWebconsole, err := utils.GetVirtualMachineWebConsoleRequest(ctx, svClusterClient, ns, vmWebconsoleName) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(vmWebconsole).ShouldNot(BeNil()) + }, config.GetIntervals("default", "wait-virtual-machine-web-console-request-creation")...).Should(Succeed(), "Timed out waiting for VirtualMachineWebConsoleRequest %s to be created in namespace %s", vmWebconsoleName, ns) +} + +func VerifyVMInZone(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, zone, vmName string) { + By("Verify that VM is placed in zone") + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachine(ctx, svClusterClient, ns, vmName) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(vm.Status.Zone).Should(Equal(zone)) + }, config.GetIntervals("default", "wait-config-map-creation")...).Should(Succeed(), "Timed out waiting for VirtualMachine %s to be created") +} + +// CreateLocalContentLibrary Utility function to create a local content library with given name. +func CreateLocalContentLibrary(clName string, wcpClient wcp.WorkloadManagementAPI) string { + datastores, err := wcpClient.ListDatastores() + Expect(err).NotTo(HaveOccurred(), "failed to list datastores") + Expect(len(datastores)).NotTo(BeZero(), "no datastores found") + + // Choose the first datastore we see. + dsForCL := datastores[0] + clID, err := wcpClient.CreateLocalContentLibrary(clName, wcp.StorageBackingInfo{ + StorageBackings: []wcp.BackingInfo{ + { + DatastoreID: dsForCL.Datastore, + Type: "DATASTORE", + }, + }, + }) + Expect(err).NotTo(HaveOccurred(), "failed to create a new content library for publishing") + Expect(clID).NotTo(BeEmpty(), "new publishing content library ID is empty") + + return clID +} + +func GetK8sContentLibraryNameByUUID(ctx context.Context, config *config.E2EConfig, svClusterClient ctrlclient.Client, ns, contentLibraryID string) (string, error) { + var k8sContentLibraryName string + + Eventually(func() (string, error) { + k8sCLObj, err := utils.GetContentLibraryByUUID(ctx, svClusterClient, ns, contentLibraryID) + if err != nil { + return "", err + } + + k8sContentLibraryName = k8sCLObj.Name + + return k8sContentLibraryName, nil + }, config.GetIntervals("default", "wait-content-library-name")...).ShouldNot(BeEmpty(), "failed to get the k8s content library object by UUID: %s", contentLibraryID) + + return k8sContentLibraryName, nil +} + +// GetContentLibraryUUIDByName returns the content library UUID by a given name. +func GetContentLibraryUUIDByName(clName string, wcpClient wcp.WorkloadManagementAPI) string { + libraries, err := wcpClient.ListContentLibraries() + Expect(err).NotTo(HaveOccurred()) + vmserviceCLID, err := wcpClient.FetchContentLibraryIDByName(clName, libraries) + Expect(err).NotTo(HaveOccurred()) + Expect(vmserviceCLID).NotTo(BeEmpty()) + + return vmserviceCLID +} + +// Wait for a max of 2 minutes for the given pod to be ready else return an error. +func WaitForPodReady(ctx context.Context, config *config.E2EConfig, client ctrlclient.Client, ns, name string) { + Eventually(func() bool { + pod, err := utils.GetPod(ctx, client, ns, name) + if err != nil { + framework.Logf("retry due to: %v", err) + return false + } + + for i := range pod.Status.Conditions { + if pod.Status.Conditions[i].Type == corev1.PodReady { + if pod.Status.Conditions[i].Status == corev1.ConditionTrue { + framework.Logf("Pod: %s, ready state: %v", pod.Name, corev1.PodReady) + return true + } + } + } + + return false + }, config.GetIntervals("default", "wait-pod-ready")...).Should(BeTrue(), "Timed out waiting for Pod %s/%s to be ready", ns, name) +} + +// VerifyLoginAndRunCmdsInNSXSetup verifies ssh login to a given VM's Ip address and run commands with +// corresponding expected outputs. +// For NSX setup, VMs will be inaccessible directly. NSX-t setup uses a PodVM as a jumpbox to create connection with VM. +func VerifyLoginAndRunCmdsInNSXSetup(ctx context.Context, config *config.E2EConfig, clusterProxy *common.VMServiceClusterProxy, + namespace string, podVMName string, vmIP string, cmds []string, expectedOutput []string) { + framework.Logf("will attempt to ssh into %s using jumpbox podvm", vmIP) + + Eventually(func() bool { + stdout, err := clusterProxy.Exec(ctx, "-it", "jumpbox", "-n", namespace, "--", "sshpass", "-V") + if err == nil && stdout != nil { + return true + } + // The Exec function will output an error message on each failure. + // Add a log here to clarify that retries are expected behavior. + framework.Logf("sshpass not yet installed on jumpbox PodVM, retrying...") + + return false + }, config.GetIntervals("default", "wait-jumpbox-sshpass-ready")...).Should(BeTrue(), "Timed out waiting for sshpass installation on jumpbox PodVM") + + framework.Logf("sshpass installed on jumpbox PodVM; verifying ssh login to VM with 'ip addr' command") + + sshHostField := fmt.Sprintf("%s@%s", consts.DefaultVMUserName, vmIP) + Eventually(func(g Gomega) bool { + _, err := clusterProxy.Exec(ctx, "-it", podVMName, "-n", namespace, "--", "rm", "-rf", "/root/.ssh/known_hosts") + g.Expect(err).NotTo(HaveOccurred(), "failed to remove SSH known_hosts file in jumpbox PodVM") + stdout, err := clusterProxy.Exec(ctx, "-it", podVMName, "-n", namespace, "--", "sshpass", "-p", consts.DefaultVMPassword, "ssh", "-o", "StrictHostKeyChecking no", sshHostField, "ip", "addr") + g.Expect(err).NotTo(HaveOccurred(), "failed to SSH into VM to run 'ip addr' command") + + if strings.Contains(string(stdout), vmIP) { + framework.Logf("'ip addr' command output:\n%s", string(stdout)) + return true + } + + return false + }, config.GetIntervals("default", "login-retry-timeout")...).Should(BeTrue(), "timeout SSH into VM or 'ip addr' command output does not contain VM IP %q", vmIP) + + framework.Logf("ssh login verified; running requested commands on VM") + Expect(len(cmds)).To(Equal(len(expectedOutput)), "number of commands and expected outputs must be the same") + + for i, cmd := range cmds { + framework.Logf("running cmd %q via jumpbox PodVM", cmd) + Eventually(func(g Gomega) { + _, err := clusterProxy.Exec(ctx, "-it", podVMName, "-n", namespace, "--", "rm", "-rf", "~/.ssh/known_hosts") + g.Expect(err).NotTo(HaveOccurred()) + stdout, err := clusterProxy.Exec(ctx, "-it", podVMName, "-n", namespace, "--", "sshpass", "-p", consts.DefaultVMPassword, "ssh", "-o", "StrictHostKeyChecking no", sshHostField, cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(string(stdout)).To(ContainSubstring(expectedOutput[i])) + }, config.GetIntervals("default", "login-retry-timeout")...).Should(Succeed(), "failed to run cmd %q or expected output %q not found from cmdOut", cmd, expectedOutput[i]) + } +} + +// VerifyLoginAndRunCmdsInVDSSetup verifies ssh login to a given VM's Ip address and run commands with +// corresponding expected outputs. +// For VDS setup, VMs will be inaccessible directly, +// the gateway VM will be used as a jumpbox to create connection with VM. +func VerifyLoginAndRunCmdsInVDSSetup(config *config.E2EConfig, vmIP string, cmds []string, expectedOutput []string) { + By(fmt.Sprintf("Verify that ssh login to VM with vmIP (ipv4): %s succeeds", vmIP)) + + httpProxy := os.Getenv(consts.HTTPProxyEnv) + Expect(httpProxy).NotTo(BeEmpty(), fmt.Sprintf("%s env var is not set", consts.HTTPProxyEnv)) + + gatewayVMIP := strings.Split(httpProxy, ":")[0] + + gatewayCmdRunner, err := e2essh.NewSSHCommandRunner(gatewayVMIP, consts.SshPort, testbed.GatewayUsername, []ssh.AuthMethod{ssh.Password(testbed.GatewayPassword)}) + Expect(err).NotTo(HaveOccurred()) + Expect(gatewayCmdRunner).NotTo(BeNil()) + + findTCPForwardingEntity := fmt.Sprintf("line=$(sed -n '/%s/=' %s | tail -1)", consts.AllowTCPForwardingKey, consts.SshdConfig) + enableTCPForwarding := fmt.Sprintf("sed -i \"${line}s/no/yes/\" %s", consts.SshdConfig) + enableTCPForwardingCmd := fmt.Sprintf("%s;%s;%s", findTCPForwardingEntity, enableTCPForwarding, consts.CmdRestartSSHD) + _, err = gatewayCmdRunner.RunCommand(enableTCPForwardingCmd) + Expect(err).NotTo(HaveOccurred()) + time.Sleep(10 * time.Second) + + gw := e2essh.Gateway{ + Hostname: gatewayVMIP, + Username: testbed.GatewayUsername, + Port: consts.SshPort, + AuthMethods: []ssh.AuthMethod{ssh.Password(testbed.GatewayPassword)}, + } + + var cmdRunner e2essh.SSHCommandRunner + + Eventually(func(g Gomega) { + cmdRunner, err = e2essh.NewSSHCommandRunnerWithinGateway(vmIP, consts.SshPort, consts.DefaultVMUserName, []ssh.AuthMethod{ssh.Password(consts.DefaultVMPassword)}, gw) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cmdRunner).ToNot(BeNil()) + }, config.GetIntervals("default", "login-retry-timeout")...).Should(Succeed(), "ssh login to VM did not succeed in time.") + + cmdOutput, err := cmdRunner.RunCommand("ip addr") + Expect(err).NotTo(HaveOccurred()) + + cmdOutputString := string(cmdOutput) + Expect(cmdOutputString).To(ContainSubstring(vmIP)) + + if len(cmds) > 0 && len(expectedOutput) > 0 { + for i, cmd := range cmds { + By(fmt.Sprintf("Verify running cmd: %s on VM with vmIP (ipv4): %s contains expected output: %s", cmd, vmIP, expectedOutput[i])) + cmdOutput, err = cmdRunner.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + + cmdOutputString = string(cmdOutput) + Expect(cmdOutputString).To(ContainSubstring(expectedOutput[i])) + } + } + + disableForwarding := fmt.Sprintf("sed -i \"${line}s/yes/no/\" %s", consts.SshdConfig) + disableForwardingCmd := fmt.Sprintf("%s;%s;%s", findTCPForwardingEntity, disableForwarding, consts.CmdRestartSSHD) + _, err = gatewayCmdRunner.RunCommand(disableForwardingCmd) + Expect(err).NotTo(HaveOccurred()) +} + +// decodeGzipBase64 decodes a gzip-compressed and base64-encoded string. +func decodeGzipBase64(encoded string) (string, error) { + // Decode base64 + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return "", fmt.Errorf("failed to decode base64: %w", err) + } + + // Decompress gzip + reader, err := gzip.NewReader(bytes.NewReader(decoded)) + if err != nil { + return "", fmt.Errorf("failed to create gzip reader: %w", err) + } + + defer func() { _ = reader.Close() }() + + decompressed, err := io.ReadAll(reader) + if err != nil { + return "", fmt.Errorf("failed to decompress gzip: %w", err) + } + + return string(decompressed), nil +} + +// WaitForBackupToComplete waits for the VM backup process to complete by verifying +// that all PVCs in the VM spec have corresponding entries in the backup data stored +// in the VM's ExtraConfig. This is exported for use in tests that perform in-place restores. +func WaitForBackupToComplete( + ctx context.Context, + vm *vmopv1a2.VirtualMachine, + clusterProxy *common.VMServiceClusterProxy, + config *config.E2EConfig, +) { + waitForBackupToComplete(ctx, vm, clusterProxy, config) +} + +// waitForBackupToComplete waits for the VM backup process to complete by verifying +// that all PVCs in the VM spec have corresponding entries in the backup data stored +// in the VM's ExtraConfig. +func waitForBackupToComplete( + ctx context.Context, + vm *vmopv1a2.VirtualMachine, + clusterProxy *common.VMServiceClusterProxy, + config *config.E2EConfig, +) { + By("Waiting for backup to complete for all PVCs") + + // Get list of PVC names from VM spec + expectedPVCNames := make(map[string]struct{}) + + for _, vol := range vm.Spec.Volumes { + if vol.PersistentVolumeClaim != nil { + expectedPVCNames[vol.PersistentVolumeClaim.ClaimName] = struct{}{} + } + } + + // If there are no PVCs, no need to wait + if len(expectedPVCNames) == 0 { + framework.Logf("No PVCs found in VM spec, skipping backup wait") + return + } + + framework.Logf("Expected PVCs to be backed up: %v", strings.Join(slices.Collect(maps.Keys(expectedPVCNames)), ", ")) + + // Get vCenter client + vCenterClient := vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + defer vcenter.LogoutVimClient(vCenterClient) + + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vm.Status.UniqueID} + propCollector := property.DefaultCollector(vCenterClient) + + // Wait for backup to complete + Eventually(func() bool { + var vmMO mo.VirtualMachine + + err := propCollector.RetrieveOne(ctx, vmMoRef, []string{"config.extraConfig"}, &vmMO) + if err != nil { + framework.Logf("Failed to retrieve VM ExtraConfig: %v", err) + return false + } + + // Check if Config is populated + if vmMO.Config == nil { + framework.Logf("VM Config is nil") + return false + } + + // Check if PVC disk data exists in ExtraConfig + ecList := object.OptionValueList(vmMO.Config.ExtraConfig) + + pvcDiskDataEncoded, found := ecList.GetString(PVCDiskDataExtraConfigKey) + if !found { + framework.Logf("PVC disk data not found in ExtraConfig yet") + return false + } + + // Decode the backup data + pvcDiskDataJSON, err := decodeGzipBase64(pvcDiskDataEncoded) + if err != nil { + framework.Logf("Failed to decode PVC disk data: %v", err) + return false + } + + // Parse the JSON to get list of PVCDiskData + var pvcDiskDataList []PVCDiskData + if err := json.Unmarshal([]byte(pvcDiskDataJSON), &pvcDiskDataList); err != nil { + framework.Logf("Failed to unmarshal PVC disk data: %v", err) + return false + } + + // Check if all expected PVCs are in the backup + backedUpPVCs := make(map[string]struct{}) + for _, pvcData := range pvcDiskDataList { + backedUpPVCs[pvcData.PVCName] = struct{}{} + } + + framework.Logf("Backed up PVCs: %v", strings.Join(slices.Collect(maps.Keys(backedUpPVCs)), ", ")) + + // Verify all expected PVCs are backed up + for pvcName := range expectedPVCNames { + if _, found := backedUpPVCs[pvcName]; !found { + framework.Logf("PVC %s not yet backed up", pvcName) + return false + } + } + + framework.Logf("All PVCs have been backed up successfully") + + return true + }, config.GetIntervals("default", "wait-backup-to-complete")...).Should(BeTrue(), "backup did not complete for all PVCs") +} + +// UnregisterPVCVolumes unregisters all PVCs in the provided list using CnsUnregisterVolume. +// This simulates a backup/restore scenario where the VM on vCenter still has its disks, +// but the Kubernetes PVC/PV objects are gone. +func UnregisterPVCVolumes( + ctx context.Context, + svClusterClient ctrlclient.Client, + namespace string, + vmName string, + pvcNames []string, + config *config.E2EConfig, +) { + By("Unregister volumes using CnsUnregisterVolume for each PVC") + + // Create CnsUnregisterVolume resource for each PVC + for _, pvcName := range pvcNames { + framework.Logf("Creating CnsUnregisterVolume for PVC: %s", pvcName) + + // Generate a unique name for the CnsUnregisterVolume resource + unregisterName := fmt.Sprintf("%s-unreg-%s", vmName, capiutil.RandomString(6)) + + // Create the CnsUnregisterVolume object using the typed API + cnsUnregister := &cnsunregistervolumev1alpha1.CnsUnregisterVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: unregisterName, + Namespace: namespace, + }, + Spec: cnsunregistervolumev1alpha1.CnsUnregisterVolumeSpec{ + PVCName: pvcName, + RetainFCD: false, + ForceUnregister: true, + }, + } + + // Create the CnsUnregisterVolume resource + Expect(svClusterClient.Create(ctx, cnsUnregister)).To(Succeed(), "failed to create CnsUnregisterVolume for PVC %s", pvcName) + framework.Logf("Successfully created CnsUnregisterVolume: %s for PVC: %s", unregisterName, pvcName) + + // Wait for the CnsUnregisterVolume status to show unregistered: true + Eventually(func() bool { + cnsUnregisterStatus := &cnsunregistervolumev1alpha1.CnsUnregisterVolume{} + + err := svClusterClient.Get(ctx, ctrlclient.ObjectKey{ + Namespace: namespace, + Name: unregisterName, + }, cnsUnregisterStatus) + if err != nil { + framework.Logf("Error getting CnsUnregisterVolume %s: %v", unregisterName, err) + return false + } + + return cnsUnregisterStatus.Status.Unregistered + }, config.GetIntervals("default", "wait-pvc-deletion")...).Should(BeTrue(), "CnsUnregisterVolume %s should have status.unregistered=true", unregisterName) + + framework.Logf("PVC %s has been successfully unregistered (CnsUnregisterVolume status confirmed)", pvcName) + + // Optionally verify the PVC is deleted (commented out - relying on CnsUnregisterVolume status instead) + // TODO: VMSVC-3346: Investigate why PV / PVCs are not being cleaned up. + // Eventually(func() bool { + // pvc := &corev1.PersistentVolumeClaim{} + // pvcKey := ctrlclient.ObjectKey{ + // Namespace: namespace, + // Name: pvcName, + // } + // err := svClusterClient.Get(ctx, pvcKey, pvc) + // return apierrors.IsNotFound(err) + // }, config.GetIntervals("default", "wait-pvc-deletion")...).Should(BeTrue(), "PVC %s should be deleted by CnsUnregisterVolume", pvcName) + } +} + +// DeleteVMResource will delete only the K8s VM to simulate a restored VM in vSphere. +// It uses CnsUnregisterVolume to remove PVCs while keeping disks attached to the VM. +func DeleteVMResource( + ctx context.Context, + vmName, vmNamespace string, + bootstrapResourceYAML []byte, + clusterProxy *common.VMServiceClusterProxy, + config *config.E2EConfig, + svClusterClient ctrlclient.Client) string { + By("Get VM before powering off") + + vm, err := utils.GetVirtualMachine(ctx, svClusterClient, vmNamespace, vmName) + Expect(err).ToNot(HaveOccurred()) + + // Wait for backup to complete before powering off and deleting the VM + waitForBackupToComplete(ctx, vm, clusterProxy, config) + + By("Power off the VM") + vmoperator.UpdateVirtualMachinePowerState(ctx, config, svClusterClient, vmNamespace, vmName, string(vmopv1a2.VirtualMachinePowerStateOff)) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmNamespace, vmName, string(vmopv1a2.VirtualMachinePowerStateOff)) + + By("Add the pause annotation to VM") + + vm, err = utils.GetVirtualMachine(ctx, svClusterClient, vmNamespace, vmName) + Expect(err).ToNot(HaveOccurred()) + + if vm.Annotations == nil { + vm.Annotations = make(map[string]string) + } + + vm.Annotations[vmopv1a2.PauseAnnotation] = trueString + Expect(svClusterClient.Update(ctx, vm)).To(Succeed()) + + // Collect all PVC names from the VM spec + var pvcNames []string + + for _, volume := range vm.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + pvcNames = append(pvcNames, volume.PersistentVolumeClaim.ClaimName) + } + } + + // Unregister all PVCs using the helper function + UnregisterPVCVolumes(ctx, svClusterClient, vmNamespace, vmName, pvcNames, config) + + By("Delete only the K8s VM (vSphere VM should remain with the paused annotation applied)") + vmoperator.DeleteVirtualMachine(ctx, svClusterClient, vmNamespace, vmName) + + // The finalizer should be removed after the VM is being deleted to avoid + // its being added again during the normal reconciliation by the controller. + By("Remove the VMOP finalizer to ensure deletion of K8s VM") + Eventually(func() bool { + vm, err = utils.GetVirtualMachine(ctx, svClusterClient, vmNamespace, vmName) + if apierrors.IsNotFound(err) { + // VM is already deleted, nothing to do. + return true + } + + if err != nil { + return false + } + + if vm.DeletionTimestamp.IsZero() { + err = fmt.Errorf("VM %s/%s does not have deletion timestamp set", vmNamespace, vmName) + return false + } + + controllerutil.RemoveFinalizer(vm, VMFinalizerName) + // Also remove the deprecated finalizer if it exists to ensure backward compatibility. + controllerutil.RemoveFinalizer(vm, VMFinalizerNameDeprecated) + err = svClusterClient.Update(ctx, vm) + + return err == nil + }, 30*time.Second, 3*time.Second).Should(BeTrue(), "failed to remove finalizer from VM '%s/%s', most recent error: %v", vmNamespace, vmName, err) + + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, vmNamespace, vmName) + + if bootstrapResourceYAML != nil { + By("Delete the bootstrap resource") + Expect(clusterProxy.DeleteWithArgs(ctx, bootstrapResourceYAML)).To(Succeed(), "failed to delete VM bootstrap resource") + } + + return vm.Status.UniqueID +} + +// InvokeRegisterVM invokes the RegisterVM API. +// Waits for the Task to complete, returning TaskInfo result. +func InvokeRegisterVM( + ctx context.Context, + vmMoID, vmNamespace string, + clusterProxy *common.VMServiceClusterProxy, + wcpClient wcp.WorkloadManagementAPI) (*types.TaskInfo, error) { + By("Invoke the RegisterVM API") + + taskID, err := wcpClient.RegisterVM(vmNamespace, vmMoID) + Expect(err).ToNot(HaveOccurred()) + Expect(taskID).ToNot(BeEmpty()) + + By("Wait for the registerVM API task to complete") + + taskID = fromVmodl1ID(taskID) + vCenterClient := vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + taskMoref := types.ManagedObjectReference{Type: "Task", Value: taskID} + task := object.NewTask(vCenterClient, taskMoref) + + return task.WaitForResult(ctx, nil) +} + +// VerifyPostRegisterVM verifies expected VM state after a successful call to RegisterVM. +func VerifyPostRegisterVM( + ctx context.Context, + vmName, vmNamespace string, + bootstrapResourceYAML []byte, + expectedRestoredPVCCount int, + clusterProxy *common.VMServiceClusterProxy, + config *config.E2EConfig, + svClusterClient ctrlclient.Client, + wcpClient wcp.WorkloadManagementAPI) { + By("Verify that the VM has been created in PoweredOff state") + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, vmNamespace, vmName) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmNamespace, vmName, string(vmopv1a2.VirtualMachinePowerStateOff)) + + vm, err := utils.GetVirtualMachine(ctx, svClusterClient, vmNamespace, vmName) + Expect(err).ToNot(HaveOccurred()) + + By("Restored VM must have expected number of restored volumes") + + actualRestoredPVCCount := 0 + + for _, vol := range vm.Spec.Volumes { + // Volume and PVC both contain "restored-" prefix, so validate that. + if vol.PersistentVolumeClaim != nil && + strings.HasPrefix(vol.PersistentVolumeClaim.ClaimName, "restored-") { + actualRestoredPVCCount++ + } + } + + Expect(actualRestoredPVCCount).To(Equal(expectedRestoredPVCCount), "Restored VM must have expected number of restored volumes, expected %d, got %d", + expectedRestoredPVCCount, actualRestoredPVCCount) + + By("Power on the VM") + vmoperator.UpdateVirtualMachinePowerState(ctx, config, svClusterClient, vmNamespace, vmName, string(vmopv1a2.VirtualMachinePowerStateOn)) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmNamespace, vmName, string(vmopv1a2.VirtualMachinePowerStateOn)) + + By("Verify that the VM has an IP assigned") + vmoperator.WaitForVirtualMachineIP(ctx, config, svClusterClient, vmNamespace, vmName) + restoredVMIp := vmoperator.GetVirtualMachineIP(ctx, svClusterClient, vmNamespace, vmName) + framework.Logf("restored vm ip is: %s ", restoredVMIp) + + By("Verify that the restored VM IP matches the IP in the corresponding network provider resource") + + newIP := vmoperator.GetVirtualMachineNetworkProviderIP(ctx, config, svClusterClient, vmNamespace, vmName) + // In some cases, a restored VM could keep old IP until GOSC runs again to update the guest network with new IP. + // Wait for the VM to have the new IP set in the guest network. + Eventually(func() bool { + vmIP := vmoperator.GetVirtualMachineIP(ctx, svClusterClient, vmNamespace, vmName) + return vmIP == newIP + }, config.GetIntervals("default", "wait-virtual-machine-vmip")...).Should(BeTrue(), fmt.Sprintf("restored VM doesn't have the new IP from network provider: %s", newIP)) +} + +// VerifyRegisterVMOnlyClassicDisk verifies the register VM API with the following steps: +// - Power off the VM and delete only the K8s VM to simulate a restored VM in vSphere. +// - Remove the bootstrap resource if provided. +// - Invoke the RegisterVM API to register the VM on Supervisor. +// - Verify that the VM is created in poweredOff state. +// - Remove the paused annotation that was added earlier to ensure the VM is fully reconciled. +// - Power on the VM and verify that the VM has an IPV4 assigned. +// - Verify that the VM IP matches the IP in the VM's network provider resource. +// The source VM only has classic disk, no PVCs. +func VerifyRegisterVMOnlyClassicDisk( + ctx context.Context, + vmName, vmNamespace string, + bootstrapResourceYAML []byte, + clusterProxy *common.VMServiceClusterProxy, + config *config.E2EConfig, + svClusterClient ctrlclient.Client, + wcpClient wcp.WorkloadManagementAPI) { + By("Verifying Register VM...") + + vmMoID := DeleteVMResource(ctx, vmName, vmNamespace, bootstrapResourceYAML, clusterProxy, config, svClusterClient) + + taskInfo, err := InvokeRegisterVM(ctx, vmMoID, vmNamespace, clusterProxy, wcpClient) + + By("Verify task state is success") + Expect(err).ToNot(HaveOccurred()) + Expect(taskInfo).ToNot(BeNil()) + Expect(taskInfo.Error).To(BeNil()) + Expect(taskInfo.State).To(Equal(types.TaskInfoStateSuccess)) + + // We only expect one restored volume that will be the classic disk + // converted to a PVC because of all disks PVC. + expectedRestoredPVCCount := 1 + VerifyPostRegisterVM( + ctx, + vmName, + vmNamespace, + bootstrapResourceYAML, + expectedRestoredPVCCount, + clusterProxy, + config, + svClusterClient, + wcpClient, + ) + + By("Finish verifying Register VM") +} + +// fromVmodl1ID given Vmodl1 identifier for a managed object, returns its MoID. +// Taken from WCP service. +func fromVmodl1ID(id string) string { + if ix := strings.LastIndex(id, ":"); ix > 0 { + return id[0:ix] + } + // TODO: The function should return empty string on error. + // Returning the original ID till VKAL-2139 is resolved. + // Also, we need to return error in this case. + return id +} + +func LabelVM(ctx context.Context, config *config.E2EConfig, clusterProxy *common.VMServiceClusterProxy, vmName, namespace, labelKey, labelVal string) error { + framework.Logf("Labeling VM %s with %s=%s", vmName, labelKey, labelVal) + + label := fmt.Sprintf("%s=%s", labelKey, labelVal) + + err := clusterProxy.Label(ctx, "vm", vmName, "-n", namespace, label) + if err != nil { + return fmt.Errorf("failed to label VM %s: %w", vmName, err) + } + + return nil +} + +// DeployVMWithCloudInit deploys a VM with the default cloud-init config passed. +func DeployVMWithCloudInit( + ctx context.Context, + vmSvcClusterProxy *common.VMServiceClusterProxy, + clusterResources *config.Resources, + ns, vmName, groupName string, + pvcs []manifestbuilders.PVC) { + secretName := "cloud-config-data-" + capiutil.RandomString(4) + secret := manifestbuilders.Secret{ + Namespace: ns, + Name: secretName, + } + secretYaml := manifestbuilders.GetSecretYamlCloudConfig(secret) + Expect(vmSvcClusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create the Secret with cloud-config data", string(secretYaml)) + + linuxImageDisplayName := GetDefaultImageDisplayName(clusterResources) + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: ns, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + SecretName: secretName, + PVCs: pvcs, + } + if groupName != "" { + vmParameters.GroupName = groupName + } + + vmYaml := manifestbuilders.GetVirtualMachineYamlA5(vmParameters) + framework.Logf("Create VirtualMachine:\n%s", string(vmYaml)) + Expect(vmSvcClusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create VM:\n%s", string(vmYaml)) +} + +func CreateVMGroup( + ctx context.Context, + vmSvcClusterProxy *common.VMServiceClusterProxy, + params manifestbuilders.VirtualMachineGroupYaml, +) { + vmGroupYaml := manifestbuilders.GetVirtualMachineGroupYaml(params) + framework.Logf("Create VirtualMachineGroup:\n%s", string(vmGroupYaml)) + Expect(vmSvcClusterProxy.CreateWithArgs(ctx, vmGroupYaml)).To(Succeed()) +} + +func DeleteVMGroup( + ctx context.Context, + vmSvcClusterProxy *common.VMServiceClusterProxy, + vmSvcE2EConfig *config.E2EConfig, + params manifestbuilders.VirtualMachineGroupYaml, +) { + vmGroupYaml := manifestbuilders.GetVirtualMachineGroupYaml(params) + framework.Logf("Delete VirtualMachineGroup:\n%s", string(vmGroupYaml)) + Expect(vmSvcClusterProxy.Delete(ctx, vmGroupYaml)).To(Succeed()) + + Eventually(func() bool { + _, err := utils.GetVirtualMachineGroup(ctx, vmSvcClusterProxy.GetClient(), params.Namespace, params.Name) + return apierrors.IsNotFound(err) + }, vmSvcE2EConfig.GetIntervals("default", "wait-virtual-machine-group-deletion")...).To(BeTrue()) +} + +func CreateVMGroupPub( + ctx context.Context, + vmSvcClusterProxy *common.VMServiceClusterProxy, + params manifestbuilders.VirtualMachineGroupPublishRequestYaml, + errMsg string, +) { + vmGroupPubYaml := manifestbuilders.GetVirtualMachineGroupPublishRequestYaml(params) + framework.Logf("Create VirtualMachineGroupPublishRequest:\n%s", string(vmGroupPubYaml)) + + if len(errMsg) == 0 { + Expect(vmSvcClusterProxy.CreateWithArgs(ctx, vmGroupPubYaml)).To(Succeed()) + } else { + _, stderr, err := vmSvcClusterProxy.CreateRawWithArgs(ctx, vmGroupPubYaml) + Expect(err).To(HaveOccurred()) + Expect(string(stderr)).To(ContainSubstring(errMsg)) + } +} + +func UpdateVMGroupPub( + ctx context.Context, + vmSvcClusterProxy *common.VMServiceClusterProxy, + params manifestbuilders.VirtualMachineGroupPublishRequestYaml) { + vmGroupPubYaml := manifestbuilders.GetVirtualMachineGroupPublishRequestYaml(params) + framework.Logf("Update VirtualMachineGroupPublishRequest:\n%s", string(vmGroupPubYaml)) + Expect(vmSvcClusterProxy.ApplyWithArgs(ctx, vmGroupPubYaml)).To(Succeed()) +} + +func CreateVMSnapshot( + ctx context.Context, + vmSvcClusterProxy *common.VMServiceClusterProxy, + params manifestbuilders.VirtualMachineSnapshotYaml, +) { + vmSnapshotYaml := manifestbuilders.GetVirtualMachineSnapshotYaml(params) + framework.Logf("Create VirtualMachineSnapshot:\n%s", string(vmSnapshotYaml)) + Expect(vmSvcClusterProxy.CreateWithArgs(ctx, vmSnapshotYaml)).To(Succeed()) +} + +func CreateSnapshotInVC( + ctx context.Context, + clusterProxy *common.VMServiceClusterProxy, + vmSnapshotName string, + vmName string, + vmNamespace string, +) { + vCenterClient := vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + defer vcenter.LogoutVimClient(vCenterClient) + + vm, err := utils.GetVirtualMachineA5(ctx, clusterProxy.GetClient(), vmNamespace, vmName) + Expect(err).NotTo(HaveOccurred()) + + vmMoID := vm.Status.UniqueID + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoID} + vmObj := object.NewVirtualMachine(vCenterClient, vmMoRef) + + task, err := vmObj.CreateSnapshot(ctx, vmSnapshotName, "description", true, true) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + + _, err = task.WaitForResult(ctx) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + + var vmMO mo.VirtualMachine + + propCollector := property.DefaultCollector(vCenterClient) + ExpectWithOffset(1, propCollector.RetrieveOne(ctx, vmMoRef, []string{"snapshot"}, &vmMO)).To(Succeed()) + ExpectWithOffset(1, vmMO.Snapshot).ToNot(BeNil()) +} + +func UpdateVMSnapshot( + ctx context.Context, + vmSvcClusterProxy *common.VMServiceClusterProxy, + vmSvcE2EConfig *config.E2EConfig, + params manifestbuilders.VirtualMachineSnapshotYaml, +) { + vmSnapshotYaml := manifestbuilders.GetVirtualMachineSnapshotYaml(params) + framework.Logf("Update VirtualMachineSnapshot:\n%s", string(vmSnapshotYaml)) + Eventually(func() error { + return vmSvcClusterProxy.ApplyWithArgs(ctx, vmSnapshotYaml) + }, vmSvcE2EConfig.GetIntervals("default", "wait-virtual-machine-snapshot-update")...).Should(Succeed()) +} + +func RevertVMSnapshot( + ctx context.Context, + vmSvcClusterProxy *common.VMServiceClusterProxy, + vmSvcE2EConfig *config.E2EConfig, + vmName, vmNamespace string, + currentSnapshot *vmopv1a5.VirtualMachineSnapshotReference, +) { + GinkgoHelper() + + Expect(currentSnapshot.Name).NotTo(BeEmpty()) + + vm, err := utils.GetVirtualMachineA5(ctx, vmSvcClusterProxy.GetClient(), vmNamespace, vmName) + Expect(err).NotTo(HaveOccurred()) + + vmPatch := vm.DeepCopy() + vmPatch.Spec.CurrentSnapshotName = currentSnapshot.Name + Expect(vmSvcClusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))).To(Succeed()) + framework.Logf("Revert to VirtualMachineSnapshot:\n%s", currentSnapshot.Name) + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, vmSvcClusterProxy.GetClient(), vmNamespace, vmName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(vm.Status.CurrentSnapshot).To(Equal(currentSnapshot)) + }, vmSvcE2EConfig.GetIntervals("default", "wait-virtual-machine-snapshot-revert")...).Should(Succeed()) +} + +func UpdateVMRestartMode( + ctx context.Context, + vmSvcClusterProxy *common.VMServiceClusterProxy, + vmSvcE2EConfig *config.E2EConfig, + vmName, vmNamespace string, + restartMode vmopv1a5.VirtualMachinePowerOpMode, +) { + GinkgoHelper() + + vm, err := utils.GetVirtualMachineA5(ctx, vmSvcClusterProxy.GetClient(), vmNamespace, vmName) + Expect(err).NotTo(HaveOccurred()) + + vmPatch := vm.DeepCopy() + vmPatch.Spec.RestartMode = restartMode + Expect(vmSvcClusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))).To(Succeed()) + framework.Logf("Update VM RestartMode:\n%s", restartMode) + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, vmSvcClusterProxy.GetClient(), vmNamespace, vmName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(vm.Spec.RestartMode).To(Equal(restartMode)) + }, vmSvcE2EConfig.GetIntervals("default", "wait-virtual-machine-restart-mode-update")...).Should(Succeed()) +} + +func GetDefaultImageDisplayName(clusterResources *config.Resources) string { + if os.Getenv("RUN_CANONICAL_TEST") == "true" { + Expect(clusterResources.UbuntuImageDisplayName).ToNot(BeEmpty(), "Invalid argument. UbuntuImageDisplayName can't be empty") + return clusterResources.UbuntuImageDisplayName + } else { + Expect(clusterResources.PhotonImageDisplayName).ToNot(BeEmpty(), "Invalid argument. PhotonImageDisplayName can't be empty") + return clusterResources.PhotonImageDisplayName + } +} + +// GetDefaultImageGuestID returns the guest ID for the default linux image. +func GetDefaultImageGuestID() string { + if os.Getenv("RUN_CANONICAL_TEST") == "true" { + return "ubuntu64Guest" + } + + return "vmwarePhoton64Guest" +} + +// VerifyVMTagsAndPolicyAssignment verifies the expected tags and policies assigned to the VM. +func VerifyVMTagsAndPolicyAssignment( + ctx context.Context, + config *config.E2EConfig, + client ctrlclient.Client, + mgr *tags.Manager, + ns, vmName string, + policyNameToTagID map[string]string, + expectedPolicyNames []string) { + GinkgoHelper() + + expectedTagIDs := make([]string, len(expectedPolicyNames)) + for i, policyName := range expectedPolicyNames { + expectedTagIDs[i] = policyNameToTagID[policyName] + } + + Eventually(func(g Gomega) { + // Verify the K8s VM CR has the expected policies in status. + vm, err := utils.GetVirtualMachineA5(ctx, client, ns, vmName) + g.Expect(err).NotTo(HaveOccurred(), "failed to get K8s VM CR") + + vmStatusPolicyNames := make([]string, len(vm.Status.Policies)) + for i, policy := range vm.Status.Policies { + vmStatusPolicyNames[i] = policy.Name + } + + g.Expect(vmStatusPolicyNames).To(ConsistOf(expectedPolicyNames)) + + // Verify the vCenter VM has the expected tags assigned. + g.Expect(vm.Status.UniqueID).NotTo(BeEmpty(), "VM unique ID should be present") + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vm.Status.UniqueID} + list, err := mgr.ListAttachedTags(ctx, vmMoRef) + g.Expect(err).NotTo(HaveOccurred(), "failed to list attached tags for VM %s", vmMoRef.Value) + g.Expect(list).To(ConsistOf(expectedTagIDs)) + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...).Should(Succeed()) +} + +func getThumbprint(urlStr string) (string, error) { + u, err := url.Parse(urlStr) + if err != nil { + return "", err + } + + host := u.Host + if !strings.Contains(host, ":") { + host += ":443" + } + + conn, err := tls.Dial("tcp", host, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return "", err + } + + defer func() { _ = conn.Close() }() + + cert := conn.ConnectionState().PeerCertificates[0] + hash := sha1.Sum(cert.Raw) + + hexHash := make([]string, 0, len(hash)) + for _, b := range hash { + hexHash = append(hexHash, fmt.Sprintf("%02X", b)) + } + + return strings.Join(hexHash, ":"), nil +} + +// EnsureVMServiceContentLibrary ensures that the vmservice content library exists. +func EnsureVMServiceContentLibrary(ctx context.Context, wcpClient wcp.WorkloadManagementAPI, subscriptionURL string) string { + clName := consts.VMServiceCLName + libraries, err := wcpClient.ListContentLibraries() + Expect(err).NotTo(HaveOccurred()) + + clID, err := wcpClient.FetchContentLibraryIDByName(clName, libraries) + if err == nil && clID != "" { + e2eframework.Byf("Content library %q already exists with ID %q", clName, clID) + return clID + } + + e2eframework.Byf("Creating VM Service content library %q", clName) + + datastores, err := wcpClient.ListDatastores() + Expect(err).NotTo(HaveOccurred()) + + var datastoreID string + + for _, dsName := range []string{"vsanDatastore", "sharedVmfs-0", "nfs0-1"} { + for _, ds := range datastores { + if ds.Name == dsName { + datastoreID = ds.Datastore + break + } + } + + if datastoreID != "" { + break + } + } + + Expect(datastoreID).NotTo(BeEmpty(), "Failed to find a suitable datastore for content library") + + storageBackings := wcp.StorageBackingInfo{ + StorageBackings: []wcp.BackingInfo{ + { + DatastoreID: datastoreID, + Type: "DATASTORE", + }, + }, + } + + thumbprint, err := getThumbprint(subscriptionURL) + if err != nil { + framework.Logf("Warning: failed to get thumbprint for %s: %v", subscriptionURL, err) + } + + clID, err = wcpClient.CreateSubscribedContentLibrary(clName, subscriptionURL, thumbprint, true, storageBackings) + Expect(err).NotTo(HaveOccurred(), "Failed to create subscribed content library") + + e2eframework.Byf("Created content library %q with ID %q", clName, clID) + + // Wait for sync + e2eframework.Byf("Waiting for VMService Content library synchronization") + + err = wcpClient.SyncSubscribedContentLibrary(clID) + Expect(err).NotTo(HaveOccurred(), "Failed to sync subscribed content library") + + // Wait for the library to be synced + Eventually(func() bool { + items, err := wcpClient.ListContentLibraryItems(clID) + if err != nil { + return false + } + + return len(items) > 0 + }, "5m", "10s").Should(BeTrue(), "Content library items should be synced") + + e2eframework.Byf("VMService Content library synchronization finished") + + return clID +} + +const PVCDiskDataExtraConfigKey = "vmservice.virtualmachine.pvc.disk.data" + +type PVCDiskData struct { + FileName string + PVCName string + AccessModes []string + UUID string +} + +// ESXConfig contains configuration for ESX host operations. +type ESXConfig struct { + HostIPs []string + Username string + Password string + Build string +} + +// RBACRole defines a custom cluster role for e2e testing. +type RBACRole struct { + Name string + APIGroup string + Resource string + Verbs string +} + +var ( + // vGPUConfiguredHosts tracks which hosts have already been configured to avoid duplicate work. + vGPUConfiguredHosts = make(map[string]bool) +) + +// EnsureVGPUConfiguration configures vGPUs on ESX hosts if needed for tests that require vGPU functionality. +// This function is idempotent - it tracks which hosts have been configured and skips already configured hosts. +// This replaces the Python logic from esx_helper.py that was used in gce2e_prerequisite.py. +// +// Usage: Call this function in BeforeEach or at the start of tests that need vGPU functionality. +func EnsureVGPUConfiguration(config ESXConfig) error { + // Check if we should skip vGPU configuration + if config.Username == "" || config.Password == "" || config.Build == "" { + framework.Logf("Skipping vGPU configuration due to missing ESX credentials or build information") + return nil + } + + if strings.Contains(os.Getenv("TEST_SKIP"), "vGPU") { + framework.Logf("Skipping vGPU configuration due to vGPU in TEST_SKIP environment variable") + return nil + } + + // Parse build number to construct VIB path + buildParts := strings.Split(config.Build, "-") + if len(buildParts) != 2 { + return fmt.Errorf("invalid build format %s, expected format: -", config.Build) + } + + buildType := buildParts[0] + buildNumber := buildParts[1] + + // Convert build type (ob -> bora, keep sb as sb) + if buildType == "ob" { + buildType = "bora" + } + + vibPath := fmt.Sprintf("http://build-squid.vcfd.broadcom.net/build/mts/release/%s-%s/publish/test-vmx.vib", buildType, buildNumber) + + // Commands to run on each ESX host + commands := []string{ + fmt.Sprintf("esxcli software vib install -v %s", vibPath), + "esxcli graphics host refresh", + } + + // Configure vGPUs on each ESX host (skip if already configured) + for _, hostIP := range config.HostIPs { + // Check if this host was already configured + if vGPUConfiguredHosts[hostIP] { + framework.Logf("vGPU already configured on ESX host %s, skipping", hostIP) + continue + } + + e2eframework.Byf("Configuring vGPUs on ESX host: %s", hostIP) + + // Create SSH connection to the ESX host + authMethods := []ssh.AuthMethod{ssh.Password(config.Password)} + + cmdRunner, err := e2essh.NewSSHCommandRunner(hostIP, 22, config.Username, authMethods) + if err != nil { + framework.Logf("Warning: Failed to connect to ESX host %s: %v", hostIP, err) + continue + } + + // Execute commands on the ESX host + hostConfigured := true + + for _, cmd := range commands { + framework.Logf("Running command on ESX host %s: %s", hostIP, cmd) + + _, err := cmdRunner.RunCommand(cmd) + if err != nil { + // Log warning but don't fail - this is best effort + // Not all ESX hosts need to have vGPU installed + framework.Logf("Warning: Failed to run command '%s' on ESX host %s: %v", cmd, hostIP, err) + + hostConfigured = false + } + } + + // Mark host as configured if all commands succeeded + if hostConfigured { + vGPUConfiguredHosts[hostIP] = true + framework.Logf("Successfully configured vGPUs on ESX host: %s", hostIP) + } + } + + return nil +} + +// ParseESXHosts parses comma-separated ESX host IPs into a slice. +func ParseESXHosts(hostIPs string) []string { + if hostIPs == "" { + return nil + } + + return strings.Split(hostIPs, ",") +} + +// NewESXConfigFromEnv creates an ESXConfig from environment variables. +// Returns nil if required environment variables are not set. +// Environment variables: +// - ESX_IPS: Comma-separated list of ESX host IPs +// - ESX_USER: Username for ESX hosts +// - ESX_PWD: Password for ESX hosts +// - ESX_BUILD: ESX build number (e.g., "ob-12345" or "sb-67890") +func NewESXConfigFromEnv() *ESXConfig { + esxIPs := os.Getenv("ESX_IPS") + esxUser := os.Getenv("ESX_USER") + esxPwd := os.Getenv("ESX_PWD") + esxBuild := os.Getenv("ESX_BUILD") + + if esxIPs == "" || esxUser == "" || esxPwd == "" || esxBuild == "" { + // ESX configuration not provided + return nil + } + + return &ESXConfig{ + HostIPs: ParseESXHosts(esxIPs), + Username: esxUser, + Password: esxPwd, + Build: esxBuild, + } +} + +// kubeCreateAlreadyExists returns true if kubectl "create" failed only because the object already exists. +func kubeCreateAlreadyExists(stderr []byte) bool { + s := string(stderr) + return strings.Contains(s, "AlreadyExists") || strings.Contains(s, "already exists") +} + +// SetupClusterRoleBindings creates the necessary cluster role bindings for vm-operator e2e tests. +// This replaces the Python logic from roles_helper.py that was called by gce2e_prerequisite.py. +// It SSHes into the supervisor control plane as root to run kubectl with admin.conf, exactly as +// the Python script did with run_cmd_vc on the supervisor control plane IP. +func SetupClusterRoleBindings(clusterProxy *common.VMServiceClusterProxy) error { + ctx := context.TODO() + + e2eframework.Byf("Setting up cluster role bindings for e2e tests") + + // Get an admin cluster proxy that uses /etc/kubernetes/admin.conf from the control plane VM, + // giving the cluster-admin privileges. + adminProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + if err != nil { + return fmt.Errorf("failed to get admin cluster proxy: %w", err) + } + defer adminProxy.Dispose(ctx) + + // Define the custom roles needed for e2e testing (from roles_helper.py) + customRoles := []RBACRole{ + { + Name: "gce2e-crds", + APIGroup: "apiextensions.k8s.io", + Resource: "customresourcedefinitions", + Verbs: "get,list,watch", + }, + { + Name: "gce2e-cns-batch-attachments", + APIGroup: "cns.vmware.com", + Resource: "cnsnodevmbatchattachments", + Verbs: "get,list,watch", + }, + { + Name: "gce2e-import-operations", + APIGroup: "mobility-operator.vmware.com", + Resource: "importoperations", + Verbs: "create,update,patch,delete,get,list,watch", + }, + } + + kubeconfigPath := adminProxy.GetKubeconfigPath() + + // 1. Create cluster-admin binding for cluster-administrator@vsphere.local (best effort) + clusterAdminArgs := []string{ + "--kubeconfig", kubeconfigPath, + "--insecure-skip-tls-verify", + "create", "clusterrolebinding", "cluster-administrator:cluster-admin", + "--user", "sso:cluster-administrator@vsphere.local", + "--clusterrole", "cluster-admin", + } + clusterAdminCmd := e2eframework.NewCommand( + e2eframework.WithCommand("kubectl"), + e2eframework.WithArgs(clusterAdminArgs...), + ) + + _, _, err = clusterAdminCmd.Run(ctx) + if err != nil { + // This is best effort - may already exist + framework.Logf("Info: cluster-admin binding may already exist: %v", err) + } + + // 2. Create custom cluster roles and bindings for Administrator@vsphere.local + for _, role := range customRoles { + // Create the cluster role + createRoleArgs := []string{ + "--kubeconfig", kubeconfigPath, + "--insecure-skip-tls-verify", + "create", "clusterrole", role.Name, + "--verb", role.Verbs, + "--resource", role.Resource + "." + role.APIGroup, + } + createRoleCmd := e2eframework.NewCommand( + e2eframework.WithCommand("kubectl"), + e2eframework.WithArgs(createRoleArgs...), + ) + + _, stderr, err := createRoleCmd.Run(ctx) + if err != nil { + if kubeCreateAlreadyExists(stderr) { + framework.Logf("Info: cluster role %q already exists, skipping create", role.Name) + } else { + return fmt.Errorf("failed to create cluster role %s: %w\nstderr: %s", role.Name, err, string(stderr)) + } + } + + // Create the cluster role binding + createBindingArgs := []string{ + "--kubeconfig", kubeconfigPath, + "--insecure-skip-tls-verify", + "create", "clusterrolebinding", role.Name, + "--user", "sso:Administrator@vsphere.local", + "--clusterrole", role.Name, + } + createBindingCmd := e2eframework.NewCommand( + e2eframework.WithCommand("kubectl"), + e2eframework.WithArgs(createBindingArgs...), + ) + + _, stderr, err = createBindingCmd.Run(ctx) + if err != nil { + if kubeCreateAlreadyExists(stderr) { + framework.Logf("Info: cluster role binding %q already exists, skipping create", role.Name) + } else { + return fmt.Errorf("failed to create cluster role binding %s: %w\nstderr: %s", role.Name, err, string(stderr)) + } + } + } + + framework.Logf("Successfully set up cluster role bindings for e2e tests") + + return nil +} + +// CleanupClusterRoleBindings removes the cluster role bindings created by SetupClusterRoleBindings. +// This can be called in test cleanup to avoid leaving test artifacts behind. +func CleanupClusterRoleBindings(clusterProxy *common.VMServiceClusterProxy) error { + ctx := context.TODO() + + e2eframework.Byf("Cleaning up cluster role bindings for e2e tests") + + // Use admin proxy with /etc/kubernetes/admin.conf for elevated privileges. + adminProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + if err != nil { + return fmt.Errorf("failed to get admin cluster proxy: %w", err) + } + defer adminProxy.Dispose(ctx) + + // Define the same custom roles for cleanup + customRoles := []RBACRole{ + {Name: "gce2e-crds"}, + {Name: "gce2e-cns-batch-attachments"}, + {Name: "gce2e-import-operations"}, + } + + kubeconfigPath := adminProxy.GetKubeconfigPath() + + // Remove cluster-admin binding (best effort) + deleteClusterAdminArgs := []string{ + "--kubeconfig", kubeconfigPath, + "--insecure-skip-tls-verify", + "delete", "clusterrolebinding", "cluster-administrator:cluster-admin", + } + deleteClusterAdminCmd := e2eframework.NewCommand( + e2eframework.WithCommand("kubectl"), + e2eframework.WithArgs(deleteClusterAdminArgs...), + ) + + _, _, err = deleteClusterAdminCmd.Run(ctx) + if err != nil { + framework.Logf("Warning: Failed to delete cluster-admin binding: %v", err) + } + + // Remove custom cluster roles and bindings + for _, role := range customRoles { + // Delete the cluster role binding + deleteBindingArgs := []string{ + "--kubeconfig", kubeconfigPath, + "--insecure-skip-tls-verify", + "delete", "clusterrolebinding", role.Name, + } + deleteBindingCmd := e2eframework.NewCommand( + e2eframework.WithCommand("kubectl"), + e2eframework.WithArgs(deleteBindingArgs...), + ) + + _, _, err := deleteBindingCmd.Run(ctx) + if err != nil { + framework.Logf("Warning: Failed to delete cluster role binding %s: %v", role.Name, err) + } + + // Delete the cluster role + deleteRoleArgs := []string{ + "--kubeconfig", kubeconfigPath, + "--insecure-skip-tls-verify", + "delete", "clusterrole", role.Name, + } + deleteRoleCmd := e2eframework.NewCommand( + e2eframework.WithCommand("kubectl"), + e2eframework.WithArgs(deleteRoleArgs...), + ) + + _, _, err = deleteRoleCmd.Run(ctx) + if err != nil { + framework.Logf("Warning: Failed to delete cluster role %s: %v", role.Name, err) + } + } + + framework.Logf("Successfully cleaned up cluster role bindings for e2e tests") + + return nil +} diff --git a/test/e2e/vmservice/vmservice/viadmin/contentlibraries.go b/test/e2e/vmservice/vmservice/viadmin/contentlibraries.go new file mode 100644 index 000000000..c91b5a101 --- /dev/null +++ b/test/e2e/vmservice/vmservice/viadmin/contentlibraries.go @@ -0,0 +1,98 @@ +// Copyright (c) 2019-2023 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package viadmin + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + capiutil "sigs.k8s.io/cluster-api/util" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type VIAdminCLSpecInput struct { + Config *e2eConfig.E2EConfig + ClusterProxy wcpframework.WCPClusterProxyInterface + ArtifactFolder string + WCPClient wcp.WorkloadManagementAPI + SkipCleanup bool +} + +func VIAdminCLSpec(ctx context.Context, inputGetter func() VIAdminCLSpecInput) { + const ( + specName = "vmcl" + ) + + var ( + input VIAdminCLSpecInput + wcpClient wcp.WorkloadManagementAPI + clusterProxy *common.VMServiceClusterProxy + config *e2eConfig.E2EConfig + nsContext wcpframework.NamespaceContext + cls []string + ) + + BeforeEach(func() { + var err error + + input = inputGetter() + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + wcpClient = input.WCPClient + config = input.Config + vmClassNames, contentLibraryNames := []string{}, []string{} + vmsvcSpecs := wcp.NewVMServiceSpecDetails(vmClassNames, contentLibraryNames) + + // VIAdminCLSpec will update the WCP namespace by overwriting its content library association. + // Therefore, we are not using the default namespace and creating a new one for each test spec. + nsContext, err = clusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, + config.InfraConfig.ManagementClusterConfig.Resources.StorageClassName, + config.InfraConfig.ManagementClusterConfig.Resources.WorkerStorageClassName, + fmt.Sprintf("%s-%s", specName, capiutil.RandomString(6)), + input.ArtifactFolder) + Expect(err).NotTo(HaveOccurred(), "failed to create wcp namespace") + + // By default, there are at least two content libraries. + // One vmservice content library, one TKG content library. + cls, err = wcpClient.ListContentLibraries() + Expect(err).NotTo(HaveOccurred(), "failed to list content libraries") + Expect(len(cls)).Should(BeNumerically(">=", 2)) + }) + + AfterEach(func() { + clusterProxy.DeleteWCPNamespace(nsContext) + }) + + Context("When testing content library association workflow with valid params", func() { + It("Should associate single valid content library", Label("smoke"), func() { + vmservice.VerifyCLAssociation(wcpClient, nsContext.GetNamespace().Name, cls[:1]) + }) + + It("Should associate multiple valid content library", func() { + vmservice.VerifyCLAssociation(wcpClient, nsContext.GetNamespace().Name, cls) + }) + + It("Should associate then disassociate content library", func() { + // Associate content libraries. + vmservice.VerifyCLAssociation(wcpClient, nsContext.GetNamespace().Name, cls) + + // Disassociate content libraries and verify removed CLs are not associated to the namespace. + vmservice.VerifyCLAssociation(wcpClient, nsContext.GetNamespace().Name, cls[0:1]) + vmservice.CheckCLDisassociation(wcpClient, nsContext.GetNamespace().Name, cls[1:]) + /* TODO (dramdass): Figure out how/if dcli supports update to empty list or use set instead of update + cls = []string{""} + VerifyCLAssociation(wcpClient, nsContext.GetNamespace().Name, cls) + */ + }) + }) +} diff --git a/test/e2e/vmservice/vmservice/viadmin/registervm.go b/test/e2e/vmservice/vmservice/viadmin/registervm.go new file mode 100644 index 000000000..700dc093a --- /dev/null +++ b/test/e2e/vmservice/vmservice/viadmin/registervm.go @@ -0,0 +1,1051 @@ +// Copyright (c) 2024-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package viadmin + +import ( + "context" + "errors" + "fmt" + "path" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + vmopv1a3 "github.com/vmware-tanzu/vm-operator/api/v1alpha3" + backupapi "github.com/vmware-tanzu/vm-operator/pkg/backup/api" + "github.com/vmware/govmomi/alarm" + "github.com/vmware/govmomi/cns" + cnstypes "github.com/vmware/govmomi/cns/types" + "github.com/vmware/govmomi/event" + "github.com/vmware/govmomi/fault" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + "github.com/vmware/govmomi/vslm" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/appple2e/lib" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/dcli" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/testutils" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + config "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" + e2eframework "k8s.io/kubernetes/test/e2e/framework" +) + +const trueString = "true" + +type VIAdminRegisterVMSpecInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *config.E2EConfig + WCPClient wcp.WorkloadManagementAPI + WCPNamespaceName string + LinuxVMName string +} + +func VIAdminRegisterVMSpec(ctx context.Context, inputGetter func() VIAdminRegisterVMSpecInput) { + const ( + specName = "register-vm" + ) + + var ( + input VIAdminRegisterVMSpecInput + wcpClient wcp.WorkloadManagementAPI + config *config.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterClient ctrlclient.Client + svClusterClientSet *kubernetes.Clientset + vmServiceBackupRestoreEnabled bool + incrementalRestoreEnabled bool + linuxImageDisplayName string + ) + + BeforeEach(func() { + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", specName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", specName) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", specName) + Expect(input.LinuxVMName).ToNot(BeEmpty(), "Invalid argument. input.LinuxVMName can't be empty when calling %s spec", specName) + + wcpClient = input.WCPClient + config = input.Config + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterClient = clusterProxy.GetClient() + svClusterClientSet = clusterProxy.GetClientSet() + + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(config.InfraConfig.ManagementClusterConfig.Resources) + + vmServiceBackupRestoreEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMServiceBackupRestore")) + incrementalRestoreEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSIncrementalRestore")) + }) + + Context("Authorization test", func() { + var ( + vCenterHostname string + authTestWCPClient wcp.WorkloadManagementAPI + vimClient *vim25.Client + user *vcenter.User + testUserWithoutPrivilege = "test-user-without-privilege" + password = "Password!23" + testUserWithoutPrivilegeWithDomain = "test-user-without-privilege@vsphere.local" + ) + + BeforeEach(func() { + vCenterAdminCreds := dcli.VCenterUserCredentials{Username: testbed.AdminUsername, Password: testbed.AdminPassword} + vCenterHostname = vcenter.GetVCPNIDFromKubeconfig(context.TODO(), clusterProxy.GetKubeconfigPath()) + Expect(vCenterHostname).NotTo(BeZero(), "Unable to determine VC PNID") + + sshCommandRunner, _, _ := testutils.GetHelpersFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + user = vcenter.NewUser(testUserWithoutPrivilege, password).WithAdminCreds(vCenterAdminCreds).WithSSHCommandRunner(sshCommandRunner) + + var err error + + err = user.Create() + Expect(err).ToNot(HaveOccurred()) + + authTestWCPClient, err = wcp.NewWCPAPIClient(vCenterHostname, testUserWithoutPrivilegeWithDomain, password, testbed.RootUsername, testbed.RootPassword) + Expect(err).NotTo(HaveOccurred()) + vimClient, err = vcenter.NewVimClient(vCenterHostname, testbed.AdminUsername, testbed.AdminPassword) + Expect(err).NotTo(HaveOccurred()) + err = vcenter.AddToGroup(ctx, vimClient, testUserWithoutPrivilege, "ReadOnlyUsers") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + // Delete the SSO user. + vcenter.DeleteUserOrFail(user) + }) + + It("A user without namespaces.Configure privilege should not be able to invoke RegisterVM API", Label("smoke"), func() { + if !vmServiceBackupRestoreEnabled { + Skip("WCP_VMService_BackupRestore FSS is not enabled") + } + + By("Invoke the RegisterVM API") + + taskID, err := authTestWCPClient.RegisterVM(input.WCPNamespaceName, "fake-vm-moid") + Expect(taskID).To(BeEmpty()) + Expect(err).To(HaveOccurred()) + + var dcliErr wcp.DcliError + Expect(errors.As(err, &dcliErr)).Should(BeTrue()) + Expect(dcliErr.Response()).Should(ContainSubstring(lib.VapiUnauthorizedErrMsg)) + }) + }) + + Context("RegisterVM with invalid params", func() { + It("If the VM does not exist, returns not found error", func() { + if !vmServiceBackupRestoreEnabled { + Skip("WCP_VMService_BackupRestore FSS is not enabled") + } + + By("Invoke the RegisterVM API") + + taskID, err := wcpClient.RegisterVM(input.WCPNamespaceName, "non-exist-vm-moid") + Expect(err).To(HaveOccurred()) + + var dcliErr wcp.DcliError + Expect(errors.As(err, &dcliErr)).Should(BeTrue()) + Expect(dcliErr.Response()).Should(ContainSubstring(lib.VapiNotFoundErrMsg)) + Expect(taskID).To(BeEmpty()) + }) + + It("If the namespace does not exist, returns not found error", func() { + if !vmServiceBackupRestoreEnabled { + Skip("WCP_VMService_BackupRestore FSS is not enabled") + } + + By("Invoke the RegisterVM API") + + taskID, err := wcpClient.RegisterVM("non-existent-namespace", "vm-moid") + Expect(err).To(HaveOccurred()) + + var dcliErr wcp.DcliError + Expect(errors.As(err, &dcliErr)).Should(BeTrue()) + Expect(dcliErr.Response()).Should(ContainSubstring(lib.VapiNotFoundErrMsg)) + Expect(taskID).To(BeEmpty()) + }) + + It("If the VM is already registered, returns already in desired state error", func() { + if !vmServiceBackupRestoreEnabled { + Skip("WCP_VMService_BackupRestore FSS is not enabled") + } + + if incrementalRestoreEnabled { + Skip("WCP_VMService_Incremental_Restore FSS is enabled") + } + + By("Get an existing VM Service VM MoID in Supervisor") + vmoperator.WaitForVirtualMachineMOID(ctx, config, svClusterClient, input.WCPNamespaceName, input.LinuxVMName) + existingVM, err := utils.GetVirtualMachine(ctx, svClusterClient, input.WCPNamespaceName, input.LinuxVMName) + Expect(err).ToNot(HaveOccurred()) + Expect(existingVM.Status.UniqueID).ToNot(BeEmpty()) + + By("Invoke the RegisterVM API") + + taskID, err := wcpClient.RegisterVM(input.WCPNamespaceName, existingVM.Status.UniqueID) + Expect(err).To(HaveOccurred()) + + var dcliErr wcp.DcliError + Expect(errors.As(err, &dcliErr)).Should(BeTrue()) + Expect(dcliErr.Response()).Should(ContainSubstring(lib.VapiAlreadyInDesiredStateErrMsg)) + Expect(taskID).To(BeEmpty()) + }) + }) + + Context("Incremental Restore - Register VM with pre-existing VM CR", func() { + It("Should register VM successfully", func() { + if !incrementalRestoreEnabled { + Skip("WCP_VMService_Incremental_Restore FSS is not enabled") + } + + vCenterClient := vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + defer vcenter.LogoutVimClient(vCenterClient) + + vmName := fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + secretName := vmName + "-cloud-config-data" + secret := manifestbuilders.Secret{ + Namespace: input.WCPNamespaceName, + Name: secretName, + } + secretYaml := manifestbuilders.GetSecretYamlCloudConfig(secret) + Expect(clusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create the Secret with cloud-config data", string(secretYaml)) + + resources := config.InfraConfig.ManagementClusterConfig.Resources + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + VMClassName: resources.VMClassName, + StorageClassName: resources.StorageClassName, + ResourcePolicy: resources.VMResourcePolicyName, + ImageName: linuxImageDisplayName, + Bootstrap: manifestbuilders.Bootstrap{ + CloudInit: &manifestbuilders.CloudInit{ + RawCloudConfig: &manifestbuilders.KeySelector{ + Key: "user-data", + Name: secretName, + }, + }, + }, + PowerState: "PoweredOn", + } + vmYaml := manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create Linux VM:\n%s", string(vmYaml)) + // End create new VM + + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmoperator.WaitForVirtualMachineMOID(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + + existingVM, err := utils.GetVirtualMachine(ctx, svClusterClient, input.WCPNamespaceName, vmName) + Expect(err).ToNot(HaveOccurred()) + + // Wait for backup to complete before reading the backup data + vmservice.WaitForBackupToComplete(ctx, existingVM, clusterProxy, config) + + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: existingVM.Status.UniqueID} + vmObj := object.NewVirtualMachine(vCenterClient, vmMoRef) + + var vmMO mo.VirtualMachine + + var ( + backupVersion string + resourceYAML string + ) + // VM Operator starts recording backup when disk promotion and volume registration has happened. + propCollector := property.DefaultCollector(vCenterClient) + Expect(propCollector.RetrieveOne(ctx, vmMoRef, []string{"config.extraConfig"}, &vmMO)).To(Succeed()) + Expect(vmMO.Config).ToNot(BeNil(), "VM Config should not be nil") + + ecList := object.OptionValueList(vmMO.Config.ExtraConfig) + resourceYAML, _ = ecList.GetString(backupapi.VMResourceYAMLExtraConfigKey) + backupVersion, _ = ecList.GetString(backupapi.BackupVersionExtraConfigKey) + + Expect(resourceYAML).ToNot(BeEmpty()) + Expect(backupVersion).ToNot(BeEmpty()) + + By("Power off the VM") + vmoperator.UpdateVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOff") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOff") + + By("Add the pause annotation to VM") + + vm, err := utils.GetVirtualMachine(ctx, svClusterClient, input.WCPNamespaceName, vmName) + Expect(err).ToNot(HaveOccurred()) + + if vm.Annotations == nil { + vm.Annotations = make(map[string]string) + } + + vm.Annotations[vmopv1a3.PauseAnnotation] = trueString + Expect(svClusterClient.Update(ctx, vm)).To(Succeed()) + + // Collect all PVC names from the VM spec + var pvcNames []string + + for _, volume := range vm.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + pvcNames = append(pvcNames, volume.PersistentVolumeClaim.ClaimName) + } + } + + // Unregister all PVCs using the helper function + vmservice.UnregisterPVCVolumes(ctx, svClusterClient, input.WCPNamespaceName, vmName, pvcNames, config) + + // reconfigBeforeRegister changes the VM's resource.yaml, backupVersion to the given value. + reconfigBeforeRegister := func(value []string) { + vmSpec := types.VirtualMachineConfigSpec{ + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{Key: backupapi.VMResourceYAMLExtraConfigKey, Value: value[0]}, + &types.OptionValue{Key: backupapi.BackupVersionExtraConfigKey, Value: value[1]}, + }, + } + + task, err := vmObj.Reconfigure(ctx, vmSpec) + Expect(err).NotTo(HaveOccurred()) + Expect(task.Wait(ctx)).To(Succeed()) + } + + By(fmt.Sprintf("Reconfigure VM with saved vm yaml %v: %s\n, backupVersion %v: %s\n", + backupapi.VMResourceYAMLExtraConfigKey, resourceYAML, + backupapi.BackupVersionExtraConfigKey, backupVersion)) + reconfigBeforeRegister([]string{resourceYAML, backupVersion}) + + taskInfo, err := vmservice.InvokeRegisterVM(ctx, existingVM.Status.UniqueID, existingVM.Namespace, clusterProxy, wcpClient) + + By("Verify task state is success") + Expect(err).ToNot(HaveOccurred()) + Expect(taskInfo).ToNot(BeNil()) + Expect(taskInfo.Error).To(BeNil()) + Expect(taskInfo.State).To(Equal(types.TaskInfoStateSuccess)) + + vmservice.VerifyPostRegisterVM(ctx, existingVM.Name, existingVM.Namespace, nil, len(existingVM.Spec.Volumes), clusterProxy, config, svClusterClient, wcpClient) + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).To(Succeed(), "failed to delete virtualmachine") + }) + }) + + Context("Incremental Restore - Register VM with pre-existing VM CR and PVCs", func() { + It("Should register VM successfully", func() { + if !incrementalRestoreEnabled { + Skip("WCP_VMService_Incremental_Restore FSS is not enabled") + } + + vCenterClient := vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + defer vcenter.LogoutVimClient(vCenterClient) + + vmName := fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + secretName := vmName + "-cloud-config-data" + secret := manifestbuilders.Secret{ + Namespace: input.WCPNamespaceName, + Name: secretName, + } + secretYaml := manifestbuilders.GetSecretYamlCloudConfig(secret) + Expect(clusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create the Secret with cloud-config data", string(secretYaml)) + + resources := config.InfraConfig.ManagementClusterConfig.Resources + pvcNameA := vmName + "-pvc-a" + testutils.AssertCreatePVC(svClusterClientSet, pvcNameA, input.WCPNamespaceName, resources.StorageClassName) + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + VMClassName: resources.VMClassName, + StorageClassName: resources.StorageClassName, + ResourcePolicy: resources.VMResourcePolicyName, + ImageName: linuxImageDisplayName, + Bootstrap: manifestbuilders.Bootstrap{ + CloudInit: &manifestbuilders.CloudInit{ + RawCloudConfig: &manifestbuilders.KeySelector{ + Key: "user-data", + Name: secretName, + }, + }, + }, + PowerState: "PoweredOn", + PVCNames: []string{pvcNameA}, + } + vmYaml := manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create Linux VM:\n%s", string(vmYaml)) + // End create new VM + + // Wait for IP, a valid moID and the PVC attachment. + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmoperator.WaitForVirtualMachineMOID(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmoperator.WaitForPVCAttachment(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, pvcNameA) + + existingVM, err := utils.GetVirtualMachine(ctx, svClusterClient, input.WCPNamespaceName, vmName) + Expect(err).ToNot(HaveOccurred()) + + // Wait for backup to complete before reading the backup data + vmservice.WaitForBackupToComplete(ctx, existingVM, clusterProxy, config) + + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: existingVM.Status.UniqueID} + vmObj := object.NewVirtualMachine(vCenterClient, vmMoRef) + + var vmMO mo.VirtualMachine + + // take a copy of the backed up vm resource and pvc backup data. + By("Save original VM resource, backup version and PVC backup from ExtraConfig") + + propCollector := property.DefaultCollector(vCenterClient) + Expect(propCollector.RetrieveOne(ctx, vmMoRef, []string{"config.extraConfig"}, &vmMO)).To(Succeed()) + Expect(vmMO.Config).ToNot(BeNil(), "VM Config should not be nil") + + var ( + backupVersion string + resourceYAML string + ) + + ecList := object.OptionValueList(vmMO.Config.ExtraConfig) + resourceYAML, _ = ecList.GetString(backupapi.VMResourceYAMLExtraConfigKey) + pvcBackup, _ := ecList.GetString(backupapi.PVCDiskDataExtraConfigKey) + backupVersion, _ = ecList.GetString(backupapi.BackupVersionExtraConfigKey) + + Expect(resourceYAML).ToNot(BeEmpty()) + Expect(pvcBackup).ToNot(BeEmpty()) + Expect(backupVersion).ToNot(BeEmpty()) + + // Create and attach another pvc to the VM. + pvcNameB := vmName + "-pvc-b" + testutils.AssertCreatePVC(svClusterClientSet, pvcNameB, input.WCPNamespaceName, resources.StorageClassName) + + // Use v1alpha3 here to make sure this doesn't blow up in product branches older than v1a5. + By(fmt.Sprintf("Updating the VM with two PVCs: '%v'", vmParameters.PVCNames)) + + vm, err := utils.GetVirtualMachineA3(ctx, svClusterClient, input.WCPNamespaceName, vmName) + Expect(err).ToNot(HaveOccurred()) + + vm.Spec.Volumes = append(vm.Spec.Volumes, vmopv1a3.VirtualMachineVolume{ + Name: pvcNameB, + VirtualMachineVolumeSource: vmopv1a3.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1a3.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcNameB, + }, + }, + }, + }) + Expect(svClusterClient.Update(ctx, vm)).To(Succeed()) + + vmoperator.WaitForPVCAttachment(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, pvcNameB) + // Both PVC A and B are now attached to VM. + + By("Power off the VM") + vmoperator.UpdateVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOff") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOff") + + By("Add the pause annotation to VM") + + vm, err = utils.GetVirtualMachineA3(ctx, svClusterClient, input.WCPNamespaceName, vmName) + Expect(err).ToNot(HaveOccurred()) + + if vm.Annotations == nil { + vm.Annotations = make(map[string]string) + } + + vm.Annotations[vmopv1a3.PauseAnnotation] = trueString + Expect(svClusterClient.Update(ctx, vm)).To(Succeed()) + + // Collect all PVC names from the VM spec + var pvcNames []string + + for _, volume := range vm.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + pvcNames = append(pvcNames, volume.PersistentVolumeClaim.ClaimName) + } + } + + // Unregister all PVCs using the helper function + vmservice.UnregisterPVCVolumes(ctx, svClusterClient, input.WCPNamespaceName, vmName, pvcNames, config) + + // reconfigBeforeRegister changes the VM's resource.yaml, backupVersion and PVC properties to the given value. + reconfigBeforeRegister := func(value []string) { + vmSpec := types.VirtualMachineConfigSpec{ + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{Key: backupapi.VMResourceYAMLExtraConfigKey, Value: value[0]}, + &types.OptionValue{Key: backupapi.PVCDiskDataExtraConfigKey, Value: value[1]}, + &types.OptionValue{Key: backupapi.BackupVersionExtraConfigKey, Value: value[2]}, + }, + } + + task, err := vmObj.Reconfigure(ctx, vmSpec) + Expect(err).NotTo(HaveOccurred()) + Expect(task.Wait(ctx)).To(Succeed()) + } + + By(fmt.Sprintf("Reconfigure VM with saved vm yaml %v: %s\n, backupVersion %v: %s\n, and PVC backup with one PVC (pvc-a) %v: %s\n", + backupapi.VMResourceYAMLExtraConfigKey, resourceYAML, + backupapi.BackupVersionExtraConfigKey, backupVersion, + backupapi.PVCDiskDataExtraConfigKey, pvcBackup)) + reconfigBeforeRegister([]string{resourceYAML, pvcBackup, backupVersion}) + + // Call registerVM on existing VM CR currently having two PVCs (a and b) with backup VM yaml having one PVC (a) + taskInfo, err := vmservice.InvokeRegisterVM(ctx, existingVM.Status.UniqueID, existingVM.Namespace, clusterProxy, wcpClient) + + By("Verify task state is success") + Expect(err).ToNot(HaveOccurred()) + Expect(taskInfo).ToNot(BeNil()) + Expect(taskInfo.Error).To(BeNil()) + Expect(taskInfo.State).To(Equal(types.TaskInfoStateSuccess)) + + // Expected registered VM should have pvc-a-restored in vm.spec.volumes + // There should be two restored volumes: one from classic disk, and one for pvc-a since pvc-b was added after backup. + expectedRestoredPVCCount := 2 + vmservice.VerifyPostRegisterVM(ctx, existingVM.Name, existingVM.Namespace, nil, expectedRestoredPVCCount, clusterProxy, config, svClusterClient, wcpClient) + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).To(Succeed(), "failed to delete virtualmachine") + }) + }) + + Context("RegisterVM Alarm", func() { + // Predefined Alarm definition added in main/9.0 (CLN 13918662) + // If using a VC without the predefined alarm, create with: + // govc alarm.create -n WCPRegisterVMFailedAlarm \ + // -d "registervm failed (for gce2e)" \ + // -green com.vmware.wcp.RegisterVM.success \ + // -yellow com.vmware.wcp.RegisterVM.failure + // Note: "alarm." prefix can only be used in predefined SystemName + const ( + alarmName = "WCPRegisterVMFailedAlarm" + eventPrefix = "com.vmware.wcp.RegisterVM." + eventSuccess = eventPrefix + "success" + eventFailure = eventPrefix + "failure" + ) + + alarmMatches := func(info types.AlarmInfo) bool { + return info.SystemName == "alarm."+alarmName || info.Name == alarmName + } + + // Test summary: + // - DeleteVMResource, removing the K8s CR + // - Reconfig VM's resource.yaml to invalid + // - InvokeRegisterVM, expecting to fail and trigger alarm + // - Reconfig VM's resource.yaml to valid + // - InvokeRegisterVM, expecting to succeed and clear triggered alarm + // - VerifyPostRegisterVM, expecting VM is powered on, has IP, etc + It("Should trigger on failure", func() { + if !vmServiceBackupRestoreEnabled { + Skip("WCP_VMService_BackupRestore FSS is not enabled") + } + + vCenterClient := vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + defer vcenter.LogoutVimClient(vCenterClient) + + alarmManager := alarm.NewManager(vCenterClient) + alarms, err := alarmManager.GetAlarm(ctx, vCenterClient.ServiceContent.RootFolder) + Expect(err).NotTo(HaveOccurred()) + + var wcpAlarm *mo.Alarm + + for _, alarm := range alarms { + if alarmMatches(alarm.Info) { + wcpAlarm = &alarm + break + } + } + + if wcpAlarm == nil { + Skip(alarmName + " not defined in this vCenter") + } + + // Create a new VM (copy-n-paste of vmservicee2e.deployVMWithCloudInit) + vmName := fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + vmsvcClusterProxy := input.ClusterProxy.(*common.VMServiceClusterProxy) + secretName := vmName + "-cloud-config-data" + secret := manifestbuilders.Secret{ + Namespace: input.WCPNamespaceName, + Name: secretName, + } + secretYaml := manifestbuilders.GetSecretYamlCloudConfig(secret) + Expect(vmsvcClusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create the Secret with cloud-config data", string(secretYaml)) + + resources := config.InfraConfig.ManagementClusterConfig.Resources + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + VMClassName: resources.VMClassName, + StorageClassName: resources.StorageClassName, + ResourcePolicy: resources.VMResourcePolicyName, + ImageName: linuxImageDisplayName, + Bootstrap: manifestbuilders.Bootstrap{ + CloudInit: &manifestbuilders.CloudInit{ + RawCloudConfig: &manifestbuilders.KeySelector{ + Key: "user-data", + Name: secretName, + }, + }, + }, + PowerState: "PoweredOn", + } + vmYaml := manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(vmsvcClusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create Linux VM:\n%s", string(vmYaml)) + + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + existingVM, err := utils.GetVirtualMachine(ctx, svClusterClient, input.WCPNamespaceName, vmName) + Expect(err).ToNot(HaveOccurred()) + + // Wait for backup to complete before reading the backup data + vmservice.WaitForBackupToComplete(ctx, existingVM, clusterProxy, config) + + // Delete the VM Service VM CR, keeping the vCenter VM in inventory. + vmMoID := vmservice.DeleteVMResource(ctx, existingVM.Name, existingVM.Namespace, nil, clusterProxy, config, svClusterClient) + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoID} + vmObj := object.NewVirtualMachine(vCenterClient, vmMoRef) + + var vmMO mo.VirtualMachine + + var resourceYAML string + + By("Save original VM ExtraConfig") + + Eventually(func(g Gomega) { + propCollector := property.DefaultCollector(vCenterClient) + g.Expect(propCollector.RetrieveOne(ctx, vmMoRef, []string{"config.extraConfig"}, &vmMO)).To(Succeed()) + g.Expect(vmMO.Config).ToNot(BeNil(), "VM Config should not be nil") + ecList := object.OptionValueList(vmMO.Config.ExtraConfig) + resourceYAML, _ = ecList.GetString(backupapi.VMResourceYAMLExtraConfigKey) + g.Expect(resourceYAML).ToNot(BeEmpty()) + }, config.GetIntervals("default", "wait-backup-to-complete")...). + Should(Succeed(), "Waiting for VM resource to be saved in ExtraConfig") + + // Create EventHistoryCollector for verifying events + eventSpec := types.EventFilterSpec{ + EventTypeId: []string{eventSuccess, eventFailure}, + Entity: &types.EventFilterSpecByEntity{ + Entity: vmMoRef, + Recursion: types.EventFilterSpecRecursionOptionSelf, + }, + } + + eventCollector, err := event.NewManager(vCenterClient).CreateCollectorForEvents(ctx, eventSpec) + Expect(err).NotTo(HaveOccurred()) + + defer func() { _ = eventCollector.Destroy(ctx) }() + + // latestEvents returns any new events of type spec.EventTypeId + latestEvents := func() (map[string][]types.EventEx, error) { + alarmEvents := make(map[string][]types.EventEx) + + for { + events, err := eventCollector.ReadNextEvents(ctx, 10) + if err != nil { + return nil, err + } + + if len(events) == 0 { // no more new events + break + } + + for i := range events { + // spec.EventTypeId filters out other types + event := events[i].(*types.EventEx) + alarmEvents[event.EventTypeId] = append(alarmEvents[event.EventTypeId], *event) + + // Fields below set by the client PostEvent + // calls in vapi/impl/wcp/registervm.go + Expect(event.Message).ToNot(BeEmpty()) + Expect(event.EventTypeId).To(HavePrefix(eventPrefix)) + // This message set by VC for predefined alarms only, see: + // vpx/vpxd/extensions/VirtualCenter/locale/en/event.vmsg + if wcpAlarm.Info.SystemName != "" { + Expect(event.FullFormattedMessage).ToNot(BeEmpty()) + } + } + } + + return alarmEvents, nil + } + + // triggeredAlarm gets the current triggeredAlarmState property and related info + triggeredAlarm := func() *alarm.StateInfo { + options := alarm.StateInfoOptions{Event: true} + alarmStates, err := alarmManager.GetStateInfo(ctx, vmMoRef, options) + Expect(err).NotTo(HaveOccurred()) + + for _, state := range alarmStates { + if alarmMatches(*state.Info) { + return &state + } + } + + return nil + } + + // reconfigResourceYAML changes the VM's resource.yaml property to the given value + reconfigResourceYAML := func(value any) { + vmSpec := types.VirtualMachineConfigSpec{ + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{Key: backupapi.VMResourceYAMLExtraConfigKey, Value: value}, + }, + } + + task, err := vmObj.Reconfigure(ctx, vmSpec) + Expect(err).NotTo(HaveOccurred()) + Expect(task.Wait(ctx)).To(Succeed()) + } + + By("Checking events before registervm") + + alarmEvents, err := latestEvents() + Expect(err).NotTo(HaveOccurred()) + Expect(alarmEvents).To(HaveLen(0)) + By("Checking triggered alarms before registervm") + Expect(triggeredAlarm()).To(BeNil()) + + By("Reconfigure VM with invalid " + backupapi.VMResourceYAMLExtraConfigKey) + reconfigResourceYAML("invalid-yaml") + + taskInfo, err := vmservice.InvokeRegisterVM(ctx, vmMoID, existingVM.Namespace, clusterProxy, wcpClient) + + By("Verify task state is error") + Expect(err).NotTo(BeNil()) + Expect(taskInfo.Error).NotTo(BeNil()) + Expect(taskInfo.State).To(Equal(types.TaskInfoStateError)) + + By("Verify failure event was emitted after registervm failure") + Eventually(func(g Gomega) { + alarmEvents, err = latestEvents() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(alarmEvents).To(HaveLen(1)) + g.Expect(alarmEvents[eventFailure]).To(HaveLen(1)) + }, config.GetIntervals("default", "wait-config-map-creation")...).Should(Succeed(), "Timed out waiting for failure event") + + By("Verify alarm was triggered by failure event") + + warningAlarm := triggeredAlarm() + Expect(warningAlarm).ToNot(BeNil()) + Expect(warningAlarm.OverallStatus).To(Equal(types.ManagedEntityStatusYellow)) + Expect(warningAlarm.Event).ToNot(BeNil()) + event := warningAlarm.Event.(*types.EventEx) + Expect(event).ToNot(BeNil()) + Expect(event.EventTypeId).To(Equal(eventFailure)) + Expect(warningAlarm.EventKey).To(Equal(alarmEvents[eventFailure][0].Key)) + + By("Reconfigure VM with original ExtraConfig") + reconfigResourceYAML(resourceYAML) + + taskInfo, err = vmservice.InvokeRegisterVM(ctx, vmMoID, existingVM.Namespace, clusterProxy, wcpClient) + + By("Verify task state is success") + Expect(err).ToNot(HaveOccurred()) + Expect(taskInfo).ToNot(BeNil()) + Expect(taskInfo.Error).To(BeNil()) + Expect(taskInfo.State).To(Equal(types.TaskInfoStateSuccess)) + + vmservice.VerifyPostRegisterVM(ctx, existingVM.Name, existingVM.Namespace, nil, len(existingVM.Spec.Volumes), clusterProxy, config, svClusterClient, wcpClient) + + By("Verify success event was emitted after successful registervm") + Eventually(func(g Gomega) { + alarmEvents, err = latestEvents() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(alarmEvents).To(HaveLen(1)) + g.Expect(alarmEvents[eventSuccess]).To(HaveLen(1)) + }, config.GetIntervals("default", "wait-config-map-creation")...).Should(Succeed(), "Timed out waiting for success event") + By("Verify triggered alarm was cleared by success event") + Expect(triggeredAlarm()).To(BeNil()) + + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).To(Succeed(), "failed to delete virtualmachine") + }) + }) + + Context("Restore disk only", func() { + It("Should register restored disk", func() { + if !vmServiceBackupRestoreEnabled { + Skip("WCP_VMService_BackupRestore FSS is not enabled") + } + + if !incrementalRestoreEnabled { + Skip("WCP_VMService_Incremental_Restore FSS is not enabled") + } + + adminClusterProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + Expect(err).ToNot(HaveOccurred()) + + defer adminClusterProxy.Dispose(ctx) + + vCenterHostname := vcenter.GetVCPNIDFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + adminClient, err := adminClusterProxy.GetAdminClient() + Expect(err).ToNot(HaveOccurred()) + vmopSecret, err := utils.GetSecret(ctx, adminClient, "vmware-system-vmop", "wcp-vmop-sa-vc-auth") + Expect(err).ToNot(HaveOccurred()) + vmopvCenterClient, err := vcenter.NewVimClient(vCenterHostname, string(vmopSecret.Data["username"]), string(vmopSecret.Data["password"])) + Expect(err).ToNot(HaveOccurred()) + + defer vcenter.LogoutVimClient(vmopvCenterClient) + + vCenterClient := vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + defer vcenter.LogoutVimClient(vCenterClient) + + // Create a new VM + vmNamespace := input.WCPNamespaceName + vmName := fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + vmsvcClusterProxy := input.ClusterProxy.(*common.VMServiceClusterProxy) + secretName := vmName + "-cloud-config-data" + secret := manifestbuilders.Secret{ + Namespace: vmNamespace, + Name: secretName, + } + secretYaml := manifestbuilders.GetSecretYamlCloudConfig(secret) + Expect(vmsvcClusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create the Secret with cloud-config data", string(secretYaml)) + + resources := config.InfraConfig.ManagementClusterConfig.Resources + pvcNameA := vmName + "-pvc-a" + testutils.AssertCreatePVC(svClusterClientSet, pvcNameA, input.WCPNamespaceName, resources.StorageClassName) + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: vmNamespace, + Name: vmName, + VMClassName: resources.VMClassName, + StorageClassName: resources.StorageClassName, + ResourcePolicy: resources.VMResourcePolicyName, + ImageName: linuxImageDisplayName, + Bootstrap: manifestbuilders.Bootstrap{ + CloudInit: &manifestbuilders.CloudInit{ + RawCloudConfig: &manifestbuilders.KeySelector{ + Key: "user-data", + Name: secretName, + }, + }, + }, + PowerState: "PoweredOn", + PVCNames: []string{pvcNameA}, + } + vmYaml := manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(vmsvcClusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create Linux VM:\n%s", string(vmYaml)) + // End create new VM + + // Wait for IP, a valid moID and the PVC attachment. + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmoperator.WaitForVirtualMachineMOID(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmoperator.WaitForPVCAttachment(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, pvcNameA) + + existingVM, err := utils.GetVirtualMachine(ctx, svClusterClient, vmNamespace, vmName) + Expect(err).ToNot(HaveOccurred()) + + // Wait for backup to complete before powering off the VM + vmservice.WaitForBackupToComplete(ctx, existingVM, clusterProxy, config) + + vmMoID := existingVM.Status.UniqueID + + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoID} + vmObj := object.NewVirtualMachine(vmopvCenterClient, vmMoRef) + + By("Power off the VM") + vmoperator.UpdateVirtualMachinePowerState(ctx, config, svClusterClient, vmNamespace, vmName, "PoweredOff") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmNamespace, vmName, "PoweredOff") + + By("Add the pause annotation to VM") + + vm, err := utils.GetVirtualMachine(ctx, svClusterClient, input.WCPNamespaceName, vmName) + Expect(err).ToNot(HaveOccurred()) + + if vm.Annotations == nil { + vm.Annotations = make(map[string]string) + } + + vm.Annotations[vmopv1a3.PauseAnnotation] = trueString + Expect(svClusterClient.Update(ctx, vm)).To(Succeed()) + + var vmMO mo.VirtualMachine + + propCollector := property.DefaultCollector(vCenterClient) + Expect(propCollector.RetrieveOne(ctx, vmMoRef, []string{"config.files"}, &vmMO)).To(Succeed()) + + // getVolumeHandle fetches the PVC, gets its PV, and returns the VolumeHandle + getVolumeHandle := func(g Gomega, pvcName, namespace string) string { + // Get the PVC + pvc := &corev1.PersistentVolumeClaim{} + pvcKey := ctrlclient.ObjectKey{ + Namespace: namespace, + Name: pvcName, + } + err := svClusterClient.Get(ctx, pvcKey, pvc) + g.Expect(err).ToNot(HaveOccurred(), "Failed to get PVC %s in namespace %s", pvcName, namespace) + g.Expect(pvc.Spec.VolumeName).ToNot(BeEmpty(), "PVC %s does not have a bound volume", pvcName) + + // Get the PV + pv := &corev1.PersistentVolume{} + pvKey := ctrlclient.ObjectKey{ + Name: pvc.Spec.VolumeName, + } + err = svClusterClient.Get(ctx, pvKey, pv) + g.Expect(err).ToNot(HaveOccurred(), "Failed to get PV %s", pvc.Spec.VolumeName) + + // Get the VolumeHandle from the CSI PersistentVolumeSource + g.Expect(pv.Spec.CSI).ToNot(BeNil(), "PV %s does not have a CSI source", pv.Name) + g.Expect(pv.Spec.CSI.VolumeHandle).ToNot(BeEmpty(), "PV %s does not have a VolumeHandle", pv.Name) + + return pv.Spec.CSI.VolumeHandle + } + + var ( + datastorePath, vmPath object.DatastorePath + disk *types.VirtualDisk + backing *types.VirtualDiskFlatVer2BackingInfo + ) + + findDisk := func(g Gomega, pvcName string, shouldExist bool) { + volumeHandle := getVolumeHandle(g, pvcName, vmNamespace) + + deviceList, err := vmObj.Device(ctx) + g.Expect(err).ToNot(HaveOccurred()) + + found := false + + for _, device := range deviceList.SelectByType((*types.VirtualDisk)(nil)) { + // Find the disk that matches the VolumeHandle from the PVC/PV + if vDiskID := device.(*types.VirtualDisk).VDiskId; vDiskID != nil { + if vDiskID.Id == volumeHandle { + disk = device.(*types.VirtualDisk) + backing = disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo) + found = datastorePath.FromString(backing.FileName) + + break + } + } + } + + g.Expect(found).To(Equal(shouldExist)) + } + + vmPath.FromString(vmMO.Config.Files.VmPathName) + vmHome := path.Dir(vmPath.Path) + + findDisk(Default, pvcNameA, true) + + dir := path.Dir(datastorePath.Path) + Expect(dir).ToNot(Equal(vmHome)) // "fcd" (or vsan object id) initially + + cnsClient, err := cns.NewClient(ctx, vCenterClient) + Expect(err).ToNot(HaveOccurred()) + + queryVolume := func() string { + filter := cnstypes.CnsQueryFilter{ + VolumeIds: []cnstypes.CnsVolumeId{cnstypes.CnsVolumeId(*disk.VDiskId)}, + } + res, err := cnsClient.QueryVolume(ctx, &filter) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Volumes).To(HaveLen(1)) + + return res.Volumes[0].StoragePolicyId + } + storageProfileID := queryVolume() + + ds := object.NewDatastore(vCenterClient, *backing.Datastore) + + fcdManager := vslm.NewObjectManager(vCenterClient) + // Validate FCD backing + _, err = fcdManager.Retrieve(ctx, ds, disk.VDiskId.Id) + Expect(err).ToNot(HaveOccurred()) + + Expect(ds.FindInventoryPath(ctx)).To(Succeed()) + + dc, err := find.NewFinder(vCenterClient).Datacenter(ctx, ds.DatacenterPath) + Expect(err).ToNot(HaveOccurred()) + + fileManager := ds.NewFileManager(dc, false) + + // "Delete" existing disk + Expect(vmObj.RemoveDevice(ctx, true, disk)).To(Succeed()) + findDisk(Default, pvcNameA, false) + + // Create "new" disk + dst := path.Join(vmHome, path.Base(datastorePath.Path)) + Expect(fileManager.Copy(ctx, datastorePath.Path, dst)).To(Succeed()) + Expect(fileManager.Delete(ctx, datastorePath.Path)).To(Succeed()) + + // Expect to fail w/ orphaned FCD + _, err = fcdManager.Retrieve(ctx, ds, disk.VDiskId.Id) + Expect(err).To(HaveOccurred()) + Expect(fault.Is(err, &types.NotFound{})).To(BeTrue()) + + // The Volume still exists + queryVolume() + + // Attach new disk with storage profile (required for cns) + datastorePath.Path = dst + backing.FileName = datastorePath.String() + + profile := []types.BaseVirtualMachineProfileSpec{ + &types.VirtualMachineDefinedProfileSpec{ + ProfileId: storageProfileID, + }, + } + + // Attach existing vmdk, rather than create a new backing + disk.CapacityInKB = 0 + disk.CapacityInBytes = 0 + Expect(vmObj.AddDeviceWithProfile(ctx, profile, disk)).To(Succeed()) + + // Since we deleted (moved) the disk backing, this causes the FCD and CNS Volume objects to be + // removed on the vSphere side.. emulating what Veeam's restore flow does. + task, err := fcdManager.ReconcileDatastoreInventory(ctx, ds.Reference()) + Expect(err).ToNot(HaveOccurred()) + err = task.Wait(ctx) + Expect(err).ToNot(HaveOccurred()) + + taskInfo, err := vmservice.InvokeRegisterVM(ctx, vmMoID, existingVM.Namespace, clusterProxy, wcpClient) + + By("Verify task state is success") + Expect(err).ToNot(HaveOccurred()) + Expect(taskInfo).ToNot(BeNil()) + Expect(taskInfo.Error).To(BeNil()) + Expect(taskInfo.State).To(Equal(types.TaskInfoStateSuccess)) + + e2eframework.Logf("VM has been restored: %v", vm) + + // Fetch the restored VM and verify that it has the expected number of volumes. + restoredVM, err := utils.GetVirtualMachineA3(ctx, svClusterClient, existingVM.Namespace, existingVM.Name) + Expect(err).ToNot(HaveOccurred()) + Expect(len(restoredVM.Spec.Volumes)).To(Equal(2)) // one base disk and one PVC + + var restoredVol *vmopv1a3.VirtualMachineVolume + + for _, vol := range restoredVM.Spec.Volumes { + // The volume with restored- prefix is the one that was restored. + if vol.PersistentVolumeClaim != nil && strings.HasPrefix(vol.PersistentVolumeClaim.ClaimName, "restored-") { + restoredVol = &vol + break + } + } + + Expect(restoredVol).ToNot(BeNil()) + + findDisk(Default, restoredVol.PersistentVolumeClaim.ClaimName, true) + + dir = path.Dir(datastorePath.Path) + Expect(dir).To(Equal(vmHome)) + + // Validate FCD backing is restored + _, err = fcdManager.Retrieve(ctx, ds, disk.VDiskId.Id) + Expect(err).ToNot(HaveOccurred()) + + // Validate new Volume is created + queryVolume() + + // We can't use len(existingVM.Spec.Volumes) here because we are only restoring one disk. + vmservice.VerifyPostRegisterVM(ctx, existingVM.Name, existingVM.Namespace, nil, 1, clusterProxy, config, svClusterClient, wcpClient) + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).To(Succeed(), "failed to delete virtualmachine") + }) + }) +} diff --git a/test/e2e/vmservice/vmservice/viadmin/virtualmachineclasses.go b/test/e2e/vmservice/vmservice/viadmin/virtualmachineclasses.go new file mode 100644 index 000000000..686657542 --- /dev/null +++ b/test/e2e/vmservice/vmservice/viadmin/virtualmachineclasses.go @@ -0,0 +1,312 @@ +// Copyright (c) 2020 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package viadmin + +import ( + "context" + errpkg "errors" + "slices" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/vim25/types" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/kubectl" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + config "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type VIAdminVMClassSpecInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *config.E2EConfig + WCPClient wcp.WorkloadManagementAPI +} + +const VMClassInvalidArg = "Server error: com.vmware.vapi.std.errors.InvalidArgument" + +func VIAdminVMClassSpec(ctx context.Context, inputGetter func() VIAdminVMClassSpecInput) { + var ( + input VIAdminVMClassSpecInput + wcpClient wcp.WorkloadManagementAPI + createSpecE2eTestBestEffortSmall, createSpecE2eTestGuaranteedXSmall wcp.VMClassSpec + createSpecE2eTestGuaranteedXSmallVirtualDevicesVGPUs wcp.VMClassSpec + vmClassAsConfigDaynDateFssEnabled bool + namespacedVMClassFSSEnabled bool + vmImageRegistryEnabled bool + config *config.E2EConfig + ) + + BeforeEach(func() { + input = inputGetter() + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + wcpClient = input.WCPClient + config = input.Config + clusterProxy := input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterClient := clusterProxy.GetClient() + namespacedVMClassFSSEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSNamespacedVMClass")) + vmImageRegistryEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMImageRegistry")) + vmClassAsConfigDaynDateFssEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMClassAsConfigDaynDate")) + createSpecE2eTestBestEffortSmall = vmservice.CreateSpecE2eTestBestEffortSmall() + createSpecE2eTestGuaranteedXSmall = vmservice.CreateSpecE2eTestGuaranteedXSmall() + + vmservice.VerifyVMClassCreate(wcpClient, createSpecE2eTestBestEffortSmall, createSpecE2eTestBestEffortSmall) + vmservice.VerifyVMClassCreate(wcpClient, createSpecE2eTestGuaranteedXSmall, createSpecE2eTestGuaranteedXSmall) + }) + + AfterEach(func() { + vmservice.VerifyVMClassDeletion(wcpClient, createSpecE2eTestBestEffortSmall.ID) + vmservice.VerifyVMClassDeletion(wcpClient, createSpecE2eTestGuaranteedXSmall.ID) + }) + + It("VI Admin should have privs to view and edit resources in all namespaces by default", Label("smoke"), func() { + viAdminKubeConfig := config.InfraConfig.KubeconfigPath + if namespacedVMClassFSSEnabled { + kubectl.AssertKubectlUserCan(ctx, viAdminKubeConfig, "get", "virtualmachineclass", "-A") + } else { + // VM Class is cluster scoped resource when WCP_Namespaced_VM_Class disabled + kubectl.AssertKubectlUserCan(ctx, viAdminKubeConfig, "get", "virtualmachineclass") + kubectl.AssertKubectlUserCan(ctx, viAdminKubeConfig, "get", "virtualmachineclassbinding", "-A") + } + + if vmImageRegistryEnabled { + kubectl.AssertKubectlUserCan(ctx, viAdminKubeConfig, "get", "clustervirtualmachineimage") + } + + kubectl.AssertKubectlUserCan(ctx, viAdminKubeConfig, "get", "virtualmachineimage", "-A") + kubectl.AssertKubectlUserCan(ctx, viAdminKubeConfig, "get", "virtualmachine", "-A") + kubectl.AssertKubectlUserCan(ctx, viAdminKubeConfig, "get", "virtualmachineservices", "-A") + kubectl.AssertKubectlUserCan(ctx, viAdminKubeConfig, "get", "virtualmachinepublishrequests", "-A") + kubectl.AssertKubectlUserCan(ctx, viAdminKubeConfig, "get", "webconsolerequests", "-A") + }) + + Context("When testing VMClass workflow with valid params", func() { + It("Should update vmClass with new cpuCount", func() { + cpuCount := 4 + updateSpec := wcp.VMClassSpec{ID: createSpecE2eTestBestEffortSmall.ID, CPUCount: &cpuCount} + expectedSpec := createSpecE2eTestBestEffortSmall + expectedSpec.CPUCount = updateSpec.CPUCount + VerifyVMClassUpdate(wcpClient, expectedSpec, updateSpec) + }) + + It("Should update vmClass with new description", func() { + description := "Added description for vmClass" + updateSpec := wcp.VMClassSpec{ID: createSpecE2eTestBestEffortSmall.ID, Description: &description} + expectedSpec := createSpecE2eTestBestEffortSmall + expectedSpec.Description = updateSpec.Description + VerifyVMClassUpdate(wcpClient, expectedSpec, updateSpec) + }) + + // Add a new block here so we can skip vGPU related tests by setting TEST_SKIP to vGPU. + Context("VM classes with vGPU", func() { + BeforeEach(func() { + // Configure vGPUs on ESX hosts if configuration is available + esxConfig := vmservice.NewESXConfigFromEnv() + if esxConfig != nil { + err := vmservice.EnsureVGPUConfiguration(*esxConfig) + Expect(err).ToNot(HaveOccurred(), "failed to configure vGPUs on ESX hosts") + } else { + e2eframework.Logf("ESX configuration not provided, proceeding with vGPU VM class tests without ESX vGPU setup") + } + + createSpecE2eTestGuaranteedXSmallVirtualDevicesVGPUs = vmservice.CreateSpecE2eTestGuaranteedXSmallVirtualDevicesVGPUs() + + expectedSpec := createSpecE2eTestGuaranteedXSmallVirtualDevicesVGPUs + if vmClassAsConfigDaynDateFssEnabled { + expectedSpec.ConfigSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "mockup-vmiop", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "mockup-vmiop", + }, + }, + }, + }, + }, + } + } + + vmservice.VerifyVMClassCreate(wcpClient, createSpecE2eTestGuaranteedXSmallVirtualDevicesVGPUs, expectedSpec) + }) + + AfterEach(func() { + vmservice.VerifyVMClassDeletion(wcpClient, createSpecE2eTestGuaranteedXSmallVirtualDevicesVGPUs.ID) + }) + + It("Should update vmClass with new devices", func() { + vgpuDevices := []wcp.VGPUDevice{ + { + ProfileName: "mockup-vmiop", + }, + } + virtualDevices := wcp.VirtualDevices{ + VGPUDevices: vgpuDevices, + } + updateSpec := wcp.VMClassSpec{ID: createSpecE2eTestGuaranteedXSmallVirtualDevicesVGPUs.ID, Devices: virtualDevices} + expectedSpec := createSpecE2eTestGuaranteedXSmallVirtualDevicesVGPUs + + expectedSpec.Devices = virtualDevices + if vmClassAsConfigDaynDateFssEnabled { + expectedSpec.ConfigSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "mockup-vmiop", + }, + }, + }, + }, + }, + } + } + + VerifyVMClassUpdate(wcpClient, expectedSpec, updateSpec) + }) + }) + + It("Should list all the vmClasses created in the VC", func() { + expectedVMClasses := []wcp.VMClassSpec{createSpecE2eTestBestEffortSmall, createSpecE2eTestGuaranteedXSmall} + VerifyListVMClass(wcpClient, expectedVMClasses) + }) + }) + Context("When testing VMClasses workflow with invalid params", func() { + It("Invalid name", func() { + invalidSpec := createSpecE2eTestBestEffortSmall + invalidSpec.ID = "invalid_name" + VerifyVMClassCreateWithInvalidParams(wcpClient, invalidSpec) + }) + It("Zero CPU Count", func() { + invalidSpec := createSpecE2eTestBestEffortSmall + invalidSpec.ID = "zero-cpu-count" + invalidSpec.CPUCount = new(0) + VerifyVMClassCreateWithInvalidParams(wcpClient, invalidSpec) + }) + It("Zero Memory", func() { + invalidSpec := createSpecE2eTestBestEffortSmall + invalidSpec.ID = "zero-memory-count" + invalidSpec.MemoryMB = new(0) + VerifyVMClassCreateWithInvalidParams(wcpClient, invalidSpec) + }) + It("Invalid Memory", func() { + invalidSpec := createSpecE2eTestBestEffortSmall + invalidSpec.ID = "invalid-memory-count" + invalidSpec.MemoryMB = new(3) + VerifyVMClassCreateWithInvalidParams(wcpClient, invalidSpec) + }) + It("Invalid CPU Reservation", func() { + invalidSpec := createSpecE2eTestBestEffortSmall + invalidSpec.ID = "invalid-cpu-reservation" + invalidSpec.CPUReservation = new(101) + VerifyVMClassCreateWithInvalidParams(wcpClient, invalidSpec) + }) + It("Negative CPU Reservation", func() { + invalidSpec := createSpecE2eTestBestEffortSmall + invalidSpec.ID = "invalid-cpu-reservation" + invalidSpec.CPUReservation = new(-1) + VerifyVMClassCreateWithInvalidParams(wcpClient, invalidSpec) + }) + It("Invalid Memory Reservation", func() { + invalidSpec := createSpecE2eTestBestEffortSmall + invalidSpec.ID = "invalid-memory-reservation" + invalidSpec.MemoryReservation = new(101) + VerifyVMClassCreateWithInvalidParams(wcpClient, invalidSpec) + }) + It("Negative Memory Reservation", func() { + invalidSpec := createSpecE2eTestBestEffortSmall + invalidSpec.ID = "invalid-memory-reservation" + invalidSpec.MemoryReservation = new(-1) + VerifyVMClassCreateWithInvalidParams(wcpClient, invalidSpec) + }) + + // These validation are currently disabled. See https://p4swarm.eng.vmware.com/perforce_1666/changes/12922354. + XIt("Invalid vGPU Profile", func() { + invalidSpec := createSpecE2eTestBestEffortSmall + invalidSpec.ID = "invalid-vgpu-profile" + invalidSpec.MemoryReservation = new(100) + invalidSpec.Devices.VGPUDevices = []wcp.VGPUDevice{ + { + ProfileName: "dummy-profile", + }, + } + VerifyVMClassCreateWithInvalidParams(wcpClient, invalidSpec) + }) + XIt("Invalid DDPIO Device", func() { + invalidSpec := createSpecE2eTestBestEffortSmall + invalidSpec.ID = "invalid-ddpio-devices" + invalidSpec.MemoryReservation = new(100) + invalidSpec.Devices.DynamicDirectPathIODevices = []wcp.DynamicDirectPathIODevice{ + { + DeviceID: 1, + VendorID: 2, + }, + } + VerifyVMClassCreateWithInvalidParams(wcpClient, invalidSpec) + }) + }) +} + +func VerifyVMClassCreateWithInvalidParams(wcpClient wcp.WorkloadManagementAPI, createSpec wcp.VMClassSpec) { + err := wcpClient.CreateVMClass(createSpec) + Expect(err).Should(HaveOccurred()) + + var dcliErr wcp.DcliError + Expect(errpkg.As(err, &dcliErr)).Should(BeTrue()) + Expect(dcliErr.Response()).Should(ContainSubstring(VMClassInvalidArg)) +} + +func VerifyVMClassUpdate(wcpClient wcp.WorkloadManagementAPI, expectedSpec, updateSpec wcp.VMClassSpec) { + err := wcpClient.UpdateVMClass(updateSpec) + Expect(err).ShouldNot(HaveOccurred()) + + updatedVMClass := wcp.VMClassInfo{} + + Eventually(func(g Gomega) { + updatedVMClass, err = wcpClient.GetVMClassInfo(expectedSpec.ID) + g.Expect(err).ToNot(HaveOccurred()) + }, 30*time.Second, 3*time.Second).Should(Succeed()) + vmservice.VerifyVMClassSpec(updatedVMClass.VMClassSpec, expectedSpec) +} + +func VerifyListVMClass(wcpClient wcp.WorkloadManagementAPI, expectedVMClassSpecs []wcp.VMClassSpec) { + listVMClasses, err := wcpClient.ListVMClasses() + Expect(err).ToNot(HaveOccurred()) + + var foundClassNames []string + + for _, vmClass := range listVMClasses { + idx := slices.IndexFunc(expectedVMClassSpecs, func(c wcp.VMClassSpec) bool { return c.ID == vmClass.ID }) + if idx >= 0 { + Expect(foundClassNames).ToNot(ContainElement(vmClass.ID)) + foundClassNames = append(foundClassNames, vmClass.ID) + + vmservice.VerifyVMClassSpec(vmClass.VMClassSpec, expectedVMClassSpecs[idx]) + } + } + + Expect(expectedVMClassSpecs).To(HaveLen(len(foundClassNames))) +} + diff --git a/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go b/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go new file mode 100644 index 000000000..a2b5bfe97 --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/virtualmachinelcm.go @@ -0,0 +1,1030 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vapi/tags" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1a2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + imgregv1a1 "github.com/vmware-tanzu/vm-operator/external/image-registry-operator/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +const ( + poweredOnState = "PoweredOn" + poweredOffState = "PoweredOff" +) + +type VMSpecInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *e2eConfig.E2EConfig + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + SkipCleanup bool + WCPNamespaceName string + LinuxVMName string +} + +func VMSpec(ctx context.Context, inputGetter func() VMSpecInput) { + const ( + specName = "vm-lcm" + ) + + var ( + input VMSpecInput + wcpClient wcp.WorkloadManagementAPI + vCenterClient *vim25.Client + propCollector *property.Collector + config *e2eConfig.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterClient ctrlclient.Client + clusterResources *e2eConfig.Resources + tmpNamespaceCtx wcpframework.NamespaceContext + vmYaml []byte + vmName string + instanceStorageFssEnabled bool + vmClassAsConfigDaynDateFssEnabled bool + namespacedVMClassFSSEnabled bool + vmResizeCPUMemoryFssEnabled bool + isoSupportFSSEnabled bool + linuxImageDisplayName string + linuxImageGuestID string + ) + + BeforeEach(func() { + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", specName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", specName) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", specName) + Expect(input.LinuxVMName).ToNot(BeEmpty(), "Invalid argument. input.LinuxVMName can't be empty when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + wcpClient = input.WCPClient + config = input.Config + clusterResources = config.InfraConfig.ManagementClusterConfig.Resources + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterClient = clusterProxy.GetClient() + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + linuxImageGuestID = vmservice.GetDefaultImageGuestID() + + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, clusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, specName)) + DeferCleanup(cancelPodWatches) + + vCenterClient = vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + propCollector = property.DefaultCollector(vCenterClient) + + instanceStorageFssEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSInstanceStorage")) + vmClassAsConfigDaynDateFssEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMClassAsConfigDaynDate")) + namespacedVMClassFSSEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSNamespacedVMClass")) + vmResizeCPUMemoryFssEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMResizeCPUMemory")) + isoSupportFSSEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSIsoSupport")) + + vmYaml = nil + tmpNamespaceCtx = wcpframework.NamespaceContext{} + vmName = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + }) + + AfterEach(func() { + vmNamespaceName := input.WCPNamespaceName + if tmpNamespaceCtx.GetNamespace() != nil { + vmNamespaceName = tmpNamespaceCtx.GetNamespace().Name + } + + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), vmNamespaceName, vmName, "vm") + } + + // Delete the virtual machine if it was created. + if len(vmYaml) > 0 { + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).To(Succeed(), "failed to delete virtualmachine") + // Verify that virtual machine does not exist. + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, vmNamespaceName, vmName) + } + + // Delete the temporary namespace if it was created. + if tmpNamespaceCtx.GetNamespace() != nil { + clusterProxy.DeleteWCPNamespace(tmpNamespaceCtx) + wcp.WaitForNamespaceDeleted(wcpClient, tmpNamespaceCtx.GetNamespace().Name) + } + + if vCenterClient != nil { + vcenter.LogoutVimClient(vCenterClient) + } + }) + + It("Should create expected resources for a single VirtualMachine", Label("smoke"), func() { + // Use the Linux VM deployed in the vmservice_suite_test.go to avoid + // creating another VM with similar config in this read only test. + vmName = input.LinuxVMName + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + + By("Verifying VM gets expected BIOS and instance UUID") + virtualMachine, err := utils.GetVirtualMachineA3(ctx, svClusterClient, input.WCPNamespaceName, vmName) + e2eframework.ExpectNoError(err) + Expect(virtualMachine.Spec.BiosUUID).To(Equal(virtualMachine.Status.BiosUUID)) + + vmMoid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vmName) + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoid} + var vmMO mo.VirtualMachine + Expect(propCollector.RetrieveOne(ctx, vmMoRef, []string{"config"}, &vmMO)).To(Succeed()) + Expect(virtualMachine.Spec.BiosUUID).To(Equal(vmMO.Config.Uuid)) + Expect(virtualMachine.Spec.InstanceUUID).To(Equal(vmMO.Config.InstanceUuid)) + }) + + It("Should create expected resources for a single poweredOff VirtualMachine when VM_Class_as_Config_DaynDate Enabled", func() { + if !vmClassAsConfigDaynDateFssEnabled { + Skip("VM_Class_as_Config_DaynDate FSS is not enabled") + } + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOff", + } + + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmoperator.WaitForVirtualMachineMOID(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmMoid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vmName) + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoid} + + var vmMO mo.VirtualMachine + err := propCollector.RetrieveOne(ctx, vmMoRef, []string{"config"}, &vmMO) + e2eframework.ExpectNoError(err) + + hw := vmMO.Config.Hardware + var vmClass *vmopv1a2.VirtualMachineClass + // Depending on namespacedVMClassFSS, return namespaced VM Class or cluster scoped VM Class + vmClass, err = utils.GetVirtualMachineClass(ctx, svClusterClient, clusterResources.VMClassName, input.WCPNamespaceName, namespacedVMClassFSSEnabled) + e2eframework.ExpectNoError(err) + + Expect(hw.NumCPU).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(hw.MemoryMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + + It("Should resize powered off VirtualMachine when VM_Resize_CPU_Memory is Enabled", func() { + if !vmResizeCPUMemoryFssEnabled { + Skip("VM_Resize_CPU_Memory FSS is not enabled") + } + + const newVMClassName = "guaranteed-large" + + Expect(vmservice.EnsureNamespaceHasAccess(input.WCPClient, clusterResources.VMClassName, input.WCPNamespaceName)).To(Succeed()) + Expect(vmservice.EnsureVMClassPresent(wcpClient, newVMClassName)).To(Succeed()) + Expect(vmservice.EnsureNamespaceHasAccess(input.WCPClient, newVMClassName, input.WCPNamespaceName)).To(Succeed()) + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PowerState: "PoweredOff", + } + + vmYaml = manifestbuilders.GetVirtualMachineYaml(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + By(fmt.Sprintf("Verify that a single VirtualMachine '%s/%s' is created", input.WCPNamespaceName, vmName)) + vmoperator.WaitForVirtualMachineMOID(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmMoid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vmName) + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoid} + + var vmMO mo.VirtualMachine + err := propCollector.RetrieveOne(ctx, vmMoRef, []string{"config"}, &vmMO) + e2eframework.ExpectNoError(err) + + // Verify initial CPU and memory. + vmClass, err := utils.GetVirtualMachineClass(ctx, svClusterClient, clusterResources.VMClassName, input.WCPNamespaceName, namespacedVMClassFSSEnabled) + e2eframework.ExpectNoError(err) + Expect(vmMO.Config.Hardware.NumCPU).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(vmMO.Config.Hardware.MemoryMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + + // Change VM's ClassName. + vmoperator.UpdateVirtualMachineClassName(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, newVMClassName) + + // Wait for Reconfigure. + vmoperator.WaitForVirtualMachineStatusClassUpdated(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, newVMClassName) + + classConfigSyncedCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachineClassConfigurationSynced, + Status: metav1.ConditionTrue, + } + vmoperator.WaitOnVirtualMachineConditionUpdate(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, classConfigSyncedCondition) + + newVMClass, err := utils.GetVirtualMachineClass(ctx, svClusterClient, newVMClassName, input.WCPNamespaceName, namespacedVMClassFSSEnabled) + e2eframework.ExpectNoError(err) + // Assert that we can tell that a resize happened. + Expect(vmClass.Spec.Hardware.Cpus).ToNot(Equal(newVMClass.Spec.Hardware.Cpus)) + Expect(vmClass.Spec.Hardware.Memory).ToNot(Equal(newVMClass.Spec.Hardware.Memory)) + + By("VC VM configuration should have been updated") + err = propCollector.RetrieveOne(ctx, vmMoRef, []string{"config"}, &vmMO) + e2eframework.ExpectNoError(err) + Expect(vmMO.Config.Hardware.NumCPU).To(BeEquivalentTo(newVMClass.Spec.Hardware.Cpus)) + Expect(vmMO.Config.Hardware.MemoryMB).To(BeEquivalentTo(newVMClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + // Make sure the VM reflects the guaranteed class. + cpuAlloc := vmMO.Config.CpuAllocation + Expect(cpuAlloc).ToNot(BeNil()) + Expect(cpuAlloc.Limit).To(HaveValue(BeEquivalentTo(-1))) + Expect(cpuAlloc.Reservation).To(HaveValue(BeNumerically(">", int64(0)))) + memAlloc := vmMO.Config.MemoryAllocation + Expect(memAlloc).ToNot(BeNil()) + Expect(memAlloc.Limit).To(HaveValue(BeEquivalentTo(-1))) + Expect(memAlloc.Reservation).To(HaveValue(BeNumerically(">", int64(0)))) + + vmservice.VerifyVMClassDeletion(wcpClient, newVMClassName) + }) + + It("Should create expected resources for a single VirtualMachine when VM_Class_as_Config_DaynDate Enabled", func() { + if !vmClassAsConfigDaynDateFssEnabled { + Skip("VM_Class_as_Config_DaynDate FSS is not enabled") + } + + Expect(vmservice.EnsureVMClassPresent(wcpClient, vmservice.VMClassE1000)).To(Succeed()) + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: vmservice.VMClassE1000, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + + err := vmservice.EnsureNamespaceHasAccess(input.WCPClient, vmservice.VMClassE1000, input.WCPNamespaceName) + e2eframework.ExpectNoError(err) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmMoid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vmName) + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoid} + + var vmMO mo.VirtualMachine + err = propCollector.RetrieveOne(ctx, vmMoRef, []string{"config"}, &vmMO) + e2eframework.ExpectNoError(err) + + hw := vmMO.Config.Hardware + // verify nic type + virtualDevices := object.VirtualDeviceList(hw.Device) + currentEthCards := virtualDevices.SelectByType((*types.VirtualE1000)(nil)) + Expect(len(currentEthCards)).To(Equal(1)) + + extraConfig := vmMO.Config.ExtraConfig + ecMap := make(map[string]string) + for _, ec := range extraConfig { + if optionValue := ec.GetOptionValue(); optionValue != nil { + ecMap[optionValue.Key] = optionValue.Value.(string) + } + } + Expect(ecMap).To(HaveKeyWithValue("hello-test-key", "hello-test-value")) + + // verify cpu and memory + var vmClass *vmopv1a2.VirtualMachineClass + // Depending on namespacedVMClassFSS, return namespaced VM Class or cluster scoped VM Class + vmClass, err = utils.GetVirtualMachineClass(ctx, svClusterClient, vmservice.VMClassE1000, input.WCPNamespaceName, namespacedVMClassFSSEnabled) + e2eframework.ExpectNoError(err) + Expect(hw.NumCPU).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(hw.MemoryMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + + vmservice.VerifyVMClassDeletion(wcpClient, vmservice.VMClassE1000) + }) + + It("Should create expected resources for a single VirtualMachine with hardware version 22", func() { + Expect(vmservice.EnsureVMClassPresent(wcpClient, vmservice.VMClassVMX22)).To(Succeed()) + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: vmservice.VMClassVMX22, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + Annotations: map[string]string{ + "vmoperator.vmware.com/fast-deploy": "\"false\"", + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + + err := vmservice.EnsureNamespaceHasAccess(input.WCPClient, vmservice.VMClassVMX22, input.WCPNamespaceName) + e2eframework.ExpectNoError(err) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmMoid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vmName) + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoid} + + var vmMO mo.VirtualMachine + err = propCollector.RetrieveOne(ctx, vmMoRef, []string{"config"}, &vmMO) + e2eframework.ExpectNoError(err) + + hwVersion := vmMO.Config.Version + hw := vmMO.Config.Hardware + // verify hardware version is set + Expect(hwVersion).To(Equal("vmx-22")) + + // verify cpu and memory + var vmClass *vmopv1a2.VirtualMachineClass + vmClass, err = utils.GetVirtualMachineClass(ctx, svClusterClient, vmservice.VMClassVMX22, input.WCPNamespaceName, namespacedVMClassFSSEnabled) + e2eframework.ExpectNoError(err) + Expect(hw.NumCPU).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(hw.MemoryMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + + vmservice.VerifyVMClassDeletion(wcpClient, vmservice.VMClassVMX22) + }) + + It("Should create expected resources for a VirtualMachine from poweredOff to poweredOn", Label("smoke"), func() { + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOff", + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOff") + + By("Verify that the VirtualMachine can be powered on and also powered off after") + vmParameters.PowerState = poweredOnState + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + e2eframework.Logf("Updating the VM's PowerState to '%v'", vmParameters.PowerState) + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to power-on virtualmachine", string(vmYaml)) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOn") + vmoperator.WaitForVirtualMachineIP(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + + // Power off the VM and verify that this doesn't cause any issues. + vmParameters.PowerState = poweredOffState + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + e2eframework.Logf("Updating the VM's PowerState to '%v'", vmParameters.PowerState) + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to power-off virtualmachine", string(vmYaml)) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOff") + }) + + It("Should create expected resources for a VirtualMachine on Namespaces recreated with identical names", func() { + // Create a new namespace here to avoid overwriting the existing namespace also used by other specs. + tmpNamespaceName := fmt.Sprintf("%s-%s", specName, capiutil.RandomString(6)) + vmserviceCLID := vmservice.GetContentLibraryUUIDByName(consts.VMServiceCLName, wcpClient) + clIDs := []string{vmserviceCLID} + vmClassNames := []string{clusterResources.VMClassName} + vmsvcSpecs := wcp.NewVMServiceSpecDetails(vmClassNames, clIDs) + var err error + tmpNamespaceCtx, err = clusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, clusterResources.StorageClassName, clusterResources.WorkerStorageClassName, tmpNamespaceName, input.ArtifactFolder) + Expect(err).NotTo(HaveOccurred(), "failed to create wcp namespace") + wcp.WaitForNamespaceReady(wcpClient, tmpNamespaceName) + + // Ensure the Linux VMI name is present in the temp namespace. + vmiName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VMI name in namespace %q", tmpNamespaceName) + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + + // Delete the virtual machine and temporary namespace to recreate the latter with the same name. + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).To(Succeed(), "failed to delete virtualmachine") + // Reset variable since temporary VM and namespace deletion is handled in AfterEach. + vmYaml = nil + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, tmpNamespaceName, vmName) + clusterProxy.DeleteWCPNamespace(tmpNamespaceCtx) + wcp.WaitForNamespaceDeleted(wcpClient, tmpNamespaceName) + + // Recreate the namespace with the same name and spec. + tmpNamespaceCtx, err = clusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, clusterResources.StorageClassName, clusterResources.WorkerStorageClassName, tmpNamespaceName, input.ArtifactFolder) + Expect(err).ToNot(HaveOccurred(), "failed to create wcp namespace") + wcp.WaitForNamespaceReady(wcpClient, tmpNamespaceName) + + // Ensure the Linux VMI name is present in the temp namespace. + vmiName, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VMI name in namespace %q", tmpNamespaceName) + vmParameters.ImageName = vmiName + + // Create a new VM and verify the creation is successful. + vmName = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + vmParameters.Name = vmName + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + }) + + It("Should emit a condition for a VMClass that doesn't exist in the cluster for a Virtual Machine creation", func() { + vmClassName := "test-vmClass" + expectedCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachineConditionClassReady, + Status: metav1.ConditionFalse, + Reason: "NotFound", + Message: fmt.Sprintf("Failed to get VirtualMachineClass %s", vmClassName), + } + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: vmClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + By("Verify that we have a single VirtualMachine with expected condition") + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, expectedCondition) + }) + + It("Should emit a condition for a VMClass that is not attached to the namespace for a Virtual Machine creation", func() { + vmClassName := "e2etest-guaranteed-medium" + // Create another workload and attach 'e2etest-guaranteed-medium' VM class to it, + // otherwise 'e2etest-guaranteed-medium' CR would be deleted if it has zero workload associations + Expect(vmservice.EnsureVMClassPresent(wcpClient, vmClassName)).To(Succeed()) + newNamespaceName := fmt.Sprintf("%s-%s", specName, capiutil.RandomString(6)) + vmClassNames := []string{vmClassName} + clIDs := []string{} + vmsvcSpecs := wcp.NewVMServiceSpecDetails(vmClassNames, clIDs) + // Not assigning to the tmpNamespaceCtx because no VMs will be deployed in this namespace. + // So that in AfterEach, it will delete the VM from the correct namespace (input.WCPNamespaceName). + newNamespaceCtx, err := clusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, clusterResources.StorageClassName, clusterResources.WorkerStorageClassName, newNamespaceName, input.ArtifactFolder) + Expect(err).ToNot(HaveOccurred(), "failed to create wcp namespace") + wcp.WaitForNamespaceReady(wcpClient, newNamespaceName) + + vmClassInfo, err := wcpClient.GetVMClassInfo(vmClassName) + Expect(err).ToNot(HaveOccurred()) + Expect(vmClassInfo.Namespaces).ShouldNot(ContainElement(input.WCPNamespaceName)) + expectedCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachineConditionClassReady, + Status: metav1.ConditionFalse, + Reason: "NotFound", + Message: fmt.Sprintf("Namespace does not have access to VirtualMachineClass. className: %s, namespace: %s", vmClassName, input.WCPNamespaceName), + } + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: vmClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + By("Verify that we have a single VirtualMachine with expected condition") + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, expectedCondition) + + clusterProxy.DeleteWCPNamespace(newNamespaceCtx) + // VM class should be deleted as it has no workload associations. + vmservice.VerifyVMClassDeletion(wcpClient, vmClassName) + }) + + It("Should Create Expected Resources For a Single Virtual Machine with Instance Storage", func() { + // Instance storage can't be tested on kind cluster + skipper.SkipUnlessInfraIs(config.InfraConfig.InfraName, "wcp") + + // We don't need to check framework.NetworkTopologyIs(config.InfraConfig.NetworkingTopology, consts.VDS) + // as Instance Storage FSS is disabled on VDS + if !instanceStorageFssEnabled { + Skip("Instance Storage FSS is not enabled") + } + + isVsanDEnabled, err := vcenter.IsVSANDEnabledCluster(ctx, vCenterClient, clusterProxy.GetKubeconfigPath()) + Expect(err).ShouldNot(HaveOccurred()) + if !isVsanDEnabled { + Skip("Cluster is not VSAND enabled") + } + + storageProfileName := "gc-e2e-vsand-profile" + var vSANDDStoragePolicyIDInt any + vSANDDStoragePolicyID, err := vcenter.GetOrCreateVsanDirectStoragePolicyID(ctx, vCenterClient, storageProfileName) + vSANDDStoragePolicyIDInt = vSANDDStoragePolicyID + Expect(err).ShouldNot(HaveOccurred()) + Expect(vSANDDStoragePolicyID).ShouldNot(BeEmpty()) + + // Create a new namespace here to avoid overwriting the existing namespace also used by other specs. + tmpNamespaceName := fmt.Sprintf("%s-%s", specName, capiutil.RandomString(6)) + vmserviceCLID := vmservice.GetContentLibraryUUIDByName(consts.VMServiceCLName, wcpClient) + clIDs := []string{vmserviceCLID} + vmClassNames := []string{clusterResources.VMClassName} + vmsvcSpecs := wcp.NewVMServiceSpecDetails(vmClassNames, clIDs) + tmpNamespaceCtx, err = clusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, clusterResources.StorageClassName, clusterResources.WorkerStorageClassName, tmpNamespaceName, input.ArtifactFolder) + Expect(err).NotTo(HaveOccurred(), "failed to create wcp namespace %s", tmpNamespaceName) + wcp.WaitForNamespaceReady(wcpClient, tmpNamespaceName) + + nsDetails, err := wcpClient.GetNamespace(tmpNamespaceName) + Expect(err).ShouldNot(HaveOccurred()) + + storageSpec := []wcp.StorageSpec{ + {Policy: vSANDDStoragePolicyID, Limit: 1024 * 100}, + {Policy: nsDetails.VMStorageSpec[0].Policy, Limit: 1024 * 100}, + } + Expect(wcpClient.SetNamespaceStorageSpecs(tmpNamespaceName, storageSpec)).Should(Succeed()) + + wcp.WaitForNamespaceReady(wcpClient, tmpNamespaceName) + + Expect(vmservice.EnsureVMClassPresent(wcpClient, vmservice.VMClassInstanceStorage, vSANDDStoragePolicyIDInt)).To(Succeed()) + Expect(vmservice.EnsureNamespaceHasAccess(input.WCPClient, vmservice.VMClassInstanceStorage, tmpNamespaceName)).To(Succeed()) + + vmiName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get VMI name in namespace: %s", tmpNamespaceName) + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: vmservice.VMClassInstanceStorage, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachineInstanceStorageAnnotations(ctx, config, svClusterClient, tmpNamespaceName, vmName) + + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + + By("Verify that instance volumes are attached to virtual machine as expected") + var vmcInfo *vmopv1a2.VirtualMachineClass + // Depending on namespacedVMClassFSS, return namespaced VM Class or cluster scoped VM Class + vmcInfo, err = utils.GetVirtualMachineClass(ctx, svClusterClient, vmservice.VMClassInstanceStorage, tmpNamespaceName, namespacedVMClassFSSEnabled) + + Expect(err).NotTo(HaveOccurred()) + vmcISVols := vmcInfo.Spec.Hardware.InstanceStorage.Volumes + + vmMoid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, tmpNamespaceName, vmName) + vmdetails, err := wcpClient.GetVirtualMachine(vmMoid) + Expect(err).NotTo(HaveOccurred()) + e2eframework.Logf("%v", vmdetails) + + // TODO validate desired state in a separate fn + var findInVMCISVols = func(vmDiskCapacity int64) bool { + for _, vol := range vmcISVols { + if vol.Size.Value() == vmDiskCapacity { + return true + } + } + return false + } + var vmInstanceDisksCount int + for _, deviceInfo := range vmdetails.Disks { + // Skip vsanDatastore + // TODO find vsand datastore using DS type rather than pattern matching + if !strings.Contains(deviceInfo.Backing.VMDKFile, "vSAND_") { + continue + } + Expect(findInVMCISVols(deviceInfo.Capacity)).Should(BeTrue()) + vmInstanceDisksCount++ + } + + Expect(vmInstanceDisksCount).Should(BeEquivalentTo(len(vmcISVols))) + }) + + It("Should create expected resources for a VirtualMachine deployed from ISO", func() { + skipper.SkipUnlessInfraIs(config.InfraConfig.InfraName, "wcp") + if !isoSupportFSSEnabled { + Skip("ISO Support FSS is not enabled") + } + + if os.Getenv("RUN_CANONICAL_TEST") == "true" { + Skip("These tests will be skipped for Canonical OVA testing.") + } + + By("Get the ISO-type image CR name") + isoImageDisplayName := "ubuntu-24.04-live-server-amd64" + isoImageName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, isoImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VMI name in namespace %q", input.WCPNamespaceName) + + By("Create a VM with CD-ROM attached and backed by the ISO-type image") + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + GuestID: "ubuntu64Guest", + Cdrom: []manifestbuilders.Cdrom{ + { + Name: "cdrom1", + ImageName: isoImageName, + ImageKind: "VirtualMachineImage", + Connected: true, + AllowGuestControl: true, + }, + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create VM with CD-ROM:\n %s", string(vmYaml)) + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOn") + + By("Verifying network provider CR has an IP assigned for the VM") + netInfo := vmoperator.WaitForVMNetworkProviderInfo(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + Expect(netInfo).NotTo(BeEmpty(), "VM network info from provider is empty") + + By(fmt.Sprintf("Verifying VM status field has the expected network config info: %+v", netInfo)) + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA3(ctx, svClusterClient, input.WCPNamespaceName, vmName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(vm.Status.Network).NotTo(BeNil()) + g.Expect(vm.Status.Network.Config).NotTo(BeNil()) + + // Check the DNS field is populated with nameservers. + dns := vm.Status.Network.Config.DNS + g.Expect(dns).NotTo(BeNil()) + g.Expect(dns.Nameservers).NotTo(BeEmpty()) + + // Check the Interface field is populated and matches the network provider info. + interfaces := vm.Status.Network.Config.Interfaces + g.Expect(interfaces).To(HaveLen(len(netInfo))) + for i, ifc := range interfaces { + vmIP := ifc.IP + g.Expect(vmIP.Addresses).NotTo(BeEmpty()) + // Parse the VM CIDR notation to compare both the IP and subnet mask. + cidr := vmIP.Addresses[0] + ip, ipNet, err := net.ParseCIDR(cidr) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ip.String()).To(Equal(netInfo[i].IPv4)) + subnetMask := net.IP(ipNet.Mask).String() + g.Expect(subnetMask).To(Equal(netInfo[i].SubnetMask)) + // Check the Gateway field matches either the IPv4 or IPv6 gateway. + g.Expect(netInfo[i].Gateway).To(Or(Equal(vmIP.Gateway4), Equal(vmIP.Gateway6))) + } + }, config.GetIntervals("default", "wait-virtual-machine-vmip")...).Should(Succeed()) + + By("Verifying VM has the expected CD-ROM device") + vmMoid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vmName) + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoid} + cdrom := verifyCdromConnectionState(ctx, vmMoRef, propCollector, true, true) + Expect(cdrom.Backing).NotTo(BeNil()) + + By("Verifying VM's CD-ROM has expected backing file from the specified ISO-type image's content library item") + //nolint:staticcheck // VirtualMachineYaml (v1alpha3) still exposes deprecated Cdrom for this spec path. + clItemName := strings.Replace(vmParameters.Cdrom[0].ImageName, "vmi", "clitem", 1) + clItem := imgregv1a1.ContentLibraryItem{} + Expect(svClusterClient.Get(ctx, ctrlclient.ObjectKey{Name: clItemName, Namespace: input.WCPNamespaceName}, &clItem)).To(Succeed()) + Expect(clItem.Status.FileInfo).NotTo(BeEmpty()) + cdromFileName := cdrom.Backing.(*types.VirtualCdromIsoBackingInfo).FileName + Expect(cdromFileName).To(Equal(clItem.Status.FileInfo[0].StorageURI)) + + By("Verifying CD-ROM can be disconnected from the VM") + //nolint:staticcheck // VirtualMachineYaml (v1alpha3) still exposes deprecated Cdrom for this spec path. + vmParameters.Cdrom[0].Connected = false + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply updated VM YAML with CD-ROM disconnected %s", string(vmYaml)) + verifyCdromConnectionState(ctx, vmMoRef, propCollector, false, true) + + By("Verifying CD-ROM can be reconnected to the VM with allowGuestControl disabled") + //nolint:staticcheck // VirtualMachineYaml (v1alpha3) still exposes deprecated Cdrom for this spec path. + vmParameters.Cdrom[0].Connected = true + //nolint:staticcheck + vmParameters.Cdrom[0].AllowGuestControl = false + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply updated VM YAML with CD-ROM reconnected %s", string(vmYaml)) + verifyCdromConnectionState(ctx, vmMoRef, propCollector, true, false) + }) + + Context("IaaS Policies", func() { + + /* + This test validates the IaaS Compute Policies feature for VMs. + + Setup: + 1. Create 4 host and VM tags and assign the host tags to the host running the test VM. + 2. Create 4 compute policies, each linked to one tag pair. + 3. Create 4 infra policies from those compute policies: + - Mandatory (match all): Always enforced on all VMs of the namespace where this policy is assigned. + - Mandatory (match by label): Applied if the VM has the label matched. + - Optional (matchGuestID): Applied if the VM explicitly specifies and match the guest ID. + - Optional (matchWorkloadLabel): Applied if the VM explicitly specifies and match the labels. + 4. Create a Supervisor namespace with all 4 infra policies assigned. + + Test Scenarios: + 1. Existing VM (created before policy assignment to namespace): + -> Receives only the mandatory policy (match all) and its tag. + + 2. New VM with explicitly specified optional policy (matchGuestID): + -> Receives both the mandatory policy (match all) and the specified policy (matchGuestID), along with their corresponding tags. + + 3. Update VM labels and policies to use match-by-label: + -> Mandatory policy: retained + -> Mandatory policy (match by label): added + -> Guest ID-matched policy: removed + -> Label-matched policy: added + -> Tags updated accordingly + + 4. Update VM labels to not match the policies and remove the explicit optional policy: + -> Mandatory policy: retained + -> Mandatory policy (match by label): removed + -> Guest ID-matched policy: removed + -> Label-matched policy: removed + -> Tags updated accordingly + */ + + var ( + tagManager *tags.Manager + tmpNamespaceName string + mandatoryPolicyNameMatchAll string + mandatoryPolicyNameMatchByLabel string + opPolicyNameMatchByGuestID string + opPolicyNameMatchByLabel string + matchLabel map[string]string + policyNameToVMTagID map[string]string + ) + + BeforeEach(func() { + skipper.SkipUnlessSupervisorCapabilityEnabled(ctx, clusterProxy, consts.IaaSComputePoliciesCapabilityName) + + By("Creating a tag manager to verify actual tag assignment") + restClient, err := vcenter.NewRestClient(ctx, vCenterClient, testbed.AdminUsername, testbed.AdminPassword) + Expect(err).NotTo(HaveOccurred(), "failed to create rest client") + tagManager = tags.NewManager(restClient) + + By("Creating a new Supervisor namespace") + tmpNamespaceName = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(6)) + vmserviceCLID := vmservice.GetContentLibraryUUIDByName(consts.VMServiceCLName, wcpClient) + clIDs := []string{vmserviceCLID} + vmClassNames := []string{clusterResources.VMClassName} + vmsvcSpecs := wcp.NewVMServiceSpecDetails(vmClassNames, clIDs) + tmpNamespaceCtx, err = clusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, clusterResources.StorageClassName, clusterResources.WorkerStorageClassName, tmpNamespaceName, input.ArtifactFolder) + Expect(err).NotTo(HaveOccurred(), "failed to create wcp namespace") + wcp.WaitForNamespaceReady(wcpClient, tmpNamespaceName) + + By("Deploying a VM without any explicit policies") + vmiName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VMI name in namespace %q", tmpNamespaceName) + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(vmParameters) + e2eframework.Logf("VM YAML without spec.policies:\n%s", string(vmYaml)) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + + By("Creating a new tag category") + tagCategoryName := fmt.Sprintf("tag-category-%s", capiutil.RandomString(4)) + tagCategoryID, err := input.WCPClient.CreateTagCategory(tagCategoryName, "test-tag-category") + Expect(err).NotTo(HaveOccurred(), "failed to create tag category") + Expect(tagCategoryID).NotTo(BeEmpty(), "tag category ID should be returned") + + By("Creating 4 new tag pairs (host tag and VM tag) under the tag category") + newTagIDPairs := make([][2]string, 4) + for i := range newTagIDPairs { + newHostTagID, err := input.WCPClient.CreateTag(fmt.Sprintf("host-tag-%s", capiutil.RandomString(4)), "test-tag", tagCategoryID) + Expect(err).NotTo(HaveOccurred(), "failed to create host tag") + Expect(newHostTagID).NotTo(BeEmpty(), "host tag ID should be returned") + newVMTagID, err := input.WCPClient.CreateTag(fmt.Sprintf("vm-tag-%s", capiutil.RandomString(4)), "test-tag", tagCategoryID) + Expect(err).NotTo(HaveOccurred(), "failed to create VM tag") + Expect(newVMTagID).NotTo(BeEmpty(), "VM tag ID should be returned") + newTagIDPairs[i] = [2]string{newHostTagID, newVMTagID} + } + + By("Getting the host ID of the deployed VM") + vmMoID := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, tmpNamespaceName, vmName) + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoID} + var vmMO mo.VirtualMachine + Expect(propCollector.RetrieveOne(ctx, vmMoRef, []string{"runtime.host"}, &vmMO)).To(Succeed()) + hostID := vmMO.Runtime.Host.Value + Expect(hostID).NotTo(BeEmpty(), "host ID should be present") + + By("Assigning the previously created host tag IDs to the host") + hostTagIDs := make([]string, len(newTagIDPairs)) + for i, tagIDPair := range newTagIDPairs { + hostTagIDs[i] = tagIDPair[0] + } + Expect(input.WCPClient.AssignTagsToHost(hostTagIDs, hostID)).To(Succeed(), "failed to assign tags to host") + + By("Creating 4 compute policies with different tag ID pairs") + vmTagIDToCPID := make(map[string]string, len(newTagIDPairs)) + for i, tagIDPair := range newTagIDPairs { + // Cannot use tagID in policy name because it's too long. + policyName := fmt.Sprintf("%s-%d", capiutil.RandomString(4), i) + cpSpec := wcp.ComputePolicySpec{ + Name: policyName, + Description: policyName, + HostTagID: tagIDPair[0], + VMTagID: tagIDPair[1], + Capability: wcp.ComputePolicyCapabilityVMHostAffinity, + } + cpID, err := input.WCPClient.CreateComputePolicy(cpSpec) + Expect(err).NotTo(HaveOccurred(), "failed to create compute policy") + Expect(cpID).NotTo(BeEmpty(), "compute policy ID should be returned") + vmTagIDToCPID[tagIDPair[1]] = cpID + } + + policyNameToVMTagID = make(map[string]string, len(newTagIDPairs)) + + By("Creating a mandatory infra policy from the compute policy with the 1st tag ID") + tagIDIndex := 0 + mandatoryInfraPolicySpec := wcp.InfraPolicySpec{ + Name: fmt.Sprintf("mandatory-%s-%d", capiutil.RandomString(4), tagIDIndex), + Description: fmt.Sprintf("mandatory infra policy for tag %s", newTagIDPairs[tagIDIndex]), + ComputePolicyID: vmTagIDToCPID[newTagIDPairs[tagIDIndex][1]], + EnforcementMode: wcp.InfraPolicyEnforcementModeMandatory, + } + Expect(input.WCPClient.CreateInfraPolicy(mandatoryInfraPolicySpec)).To(Succeed(), "failed to create mandatory infra policy") + mandatoryPolicyNameMatchAll = mandatoryInfraPolicySpec.Name + policyNameToVMTagID[mandatoryPolicyNameMatchAll] = newTagIDPairs[tagIDIndex][1] + tagIDIndex++ + + By("Creating an optional infra policy with match guest ID value from the compute policy with the 2nd tag ID") + optionalInfraPolicyMatchByGuestIDSpec := wcp.InfraPolicySpec{ + Name: fmt.Sprintf("optional-match-by-guest-id-%s-%d", capiutil.RandomString(4), tagIDIndex), + Description: fmt.Sprintf("optional infra policy for tag %s", newTagIDPairs[tagIDIndex]), + ComputePolicyID: vmTagIDToCPID[newTagIDPairs[tagIDIndex][1]], + EnforcementMode: wcp.InfraPolicyEnforcementModeOptional, + MatchGuestIDValue: linuxImageGuestID, + } + Expect(input.WCPClient.CreateInfraPolicy(optionalInfraPolicyMatchByGuestIDSpec)).To(Succeed(), "failed to create optional infra policy match by guest ID") + opPolicyNameMatchByGuestID = optionalInfraPolicyMatchByGuestIDSpec.Name + policyNameToVMTagID[opPolicyNameMatchByGuestID] = newTagIDPairs[tagIDIndex][1] + tagIDIndex++ + + By("Creating an optional infra policy with match workload label from the compute policy with the 3rd tag ID") + matchLabel = map[string]string{"test-label": "test-value"} + optionalInfraPolicyMatchWorkloadLabelSpec := wcp.InfraPolicySpec{ + Name: fmt.Sprintf("optional-match-by-label-%s-%d", capiutil.RandomString(4), tagIDIndex), + Description: fmt.Sprintf("optional infra policy for tag %s", newTagIDPairs[tagIDIndex]), + ComputePolicyID: vmTagIDToCPID[newTagIDPairs[tagIDIndex][1]], + EnforcementMode: wcp.InfraPolicyEnforcementModeOptional, + MatchWorkloadLabel: matchLabel, + } + Expect(input.WCPClient.CreateInfraPolicy(optionalInfraPolicyMatchWorkloadLabelSpec)).To(Succeed(), "failed to create optional infra policy match by label") + opPolicyNameMatchByLabel = optionalInfraPolicyMatchWorkloadLabelSpec.Name + policyNameToVMTagID[opPolicyNameMatchByLabel] = newTagIDPairs[tagIDIndex][1] + tagIDIndex++ + + By("Creating a mandatory infra policy with match by label from the compute policy with the 4th tag ID") + mandatoryInfraPolicyMatchByLabelSpec := wcp.InfraPolicySpec{ + Name: fmt.Sprintf("mandatory-match-by-label-%s-%d", capiutil.RandomString(4), tagIDIndex), + Description: fmt.Sprintf("mandatory infra policy for tag %s", newTagIDPairs[tagIDIndex]), + ComputePolicyID: vmTagIDToCPID[newTagIDPairs[tagIDIndex][1]], + EnforcementMode: wcp.InfraPolicyEnforcementModeMandatory, + MatchWorkloadLabel: matchLabel, + } + Expect(input.WCPClient.CreateInfraPolicy(mandatoryInfraPolicyMatchByLabelSpec)).To(Succeed(), "failed to create mandatory infra policy match by label") + mandatoryPolicyNameMatchByLabel = mandatoryInfraPolicyMatchByLabelSpec.Name + policyNameToVMTagID[mandatoryPolicyNameMatchByLabel] = newTagIDPairs[tagIDIndex][1] + tagIDIndex++ + + Expect(tagIDIndex).To(BeEquivalentTo(len(newTagIDPairs)), "expected to create %d policies", len(newTagIDPairs)) + + By("Assigning all 4 policies to the namespace") + Expect(input.WCPClient.UpdateNamespaceWithInfraPolicies(tmpNamespaceName, mandatoryPolicyNameMatchAll, opPolicyNameMatchByGuestID, opPolicyNameMatchByLabel, mandatoryPolicyNameMatchByLabel)).To(Succeed(), "failed to assign policies to namespace") + }) + + It("Should apply policies and tags correctly during VM creation and update", func() { + By("Verifying the existing VM has only the mandatory policy (match all) and its tag applied") + expectedPolicyNames := []string{mandatoryPolicyNameMatchAll} + vmservice.VerifyVMTagsAndPolicyAssignment(ctx, config, svClusterClient, tagManager, tmpNamespaceName, vmName, policyNameToVMTagID, expectedPolicyNames) + + By("Creating a new VM with explicitly specified the optional policy match by guest ID") + vmName = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + GuestID: linuxImageGuestID, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + Policies: []vmopv1a5.PolicySpec{ + { + APIVersion: "vsphere.policy.vmware.com/v1alpha1", + Kind: "ComputePolicy", + Name: opPolicyNameMatchByGuestID, + }, + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(vmParameters) + e2eframework.Logf("Creating VM with explicit spec.policies:\n%s", string(vmYaml)) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create VM with explicitly specified optional policy") + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + + By("Verifying the new VM has both mandatory (match all) and optional policies (match by guest ID) and their tags applied") + expectedPolicyNames = []string{mandatoryPolicyNameMatchAll, opPolicyNameMatchByGuestID} + vmservice.VerifyVMTagsAndPolicyAssignment(ctx, config, svClusterClient, tagManager, tmpNamespaceName, vmName, policyNameToVMTagID, expectedPolicyNames) + + By("Updating the VM's labels and policies to match by label") + vmParameters.Labels = matchLabel + vmParameters.Policies = []vmopv1a5.PolicySpec{ + { + APIVersion: "vsphere.policy.vmware.com/v1alpha1", + Kind: "ComputePolicy", + Name: opPolicyNameMatchByLabel, + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(vmParameters) + e2eframework.Logf("Updating VM's labels and spec.policies:\n%s", string(vmYaml)) + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply updated VM YAML") + + By("Verifying the VM has both mandatory policies (match all and match by label) and optional policies (match by label) and their tags applied") + expectedPolicyNames = []string{mandatoryPolicyNameMatchAll, mandatoryPolicyNameMatchByLabel, opPolicyNameMatchByLabel} + vmservice.VerifyVMTagsAndPolicyAssignment(ctx, config, svClusterClient, tagManager, tmpNamespaceName, vmName, policyNameToVMTagID, expectedPolicyNames) + + By("Updating the VM's label to not match the policies and remove the explicit optional policy") + vmObj, err := utils.GetVirtualMachineA5(ctx, svClusterClient, tmpNamespaceName, vmName) + Expect(err).NotTo(HaveOccurred(), "failed to get existing VM") + vmLabels := vmObj.Labels + for key := range matchLabel { + Expect(vmLabels).To(HaveKey(key)) + delete(vmLabels, key) + } + vmParameters.Labels = vmLabels + vmParameters.Policies = []vmopv1a5.PolicySpec{} + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(vmParameters) + e2eframework.Logf("Updating VM's labels:\n%s", string(vmYaml)) + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply updated VM YAML") + + By("Verifying the VM has only the mandatory policy (match all) and its tag applied") + expectedPolicyNames = []string{mandatoryPolicyNameMatchAll} + vmservice.VerifyVMTagsAndPolicyAssignment(ctx, config, svClusterClient, tagManager, tmpNamespaceName, vmName, policyNameToVMTagID, expectedPolicyNames) + }) + }) +} + +func verifyCdromConnectionState( + ctx context.Context, + vmMoRef types.ManagedObjectReference, + propCollector *property.Collector, + connected, allowGuestControl bool) *types.VirtualDevice { + + var ( + moVM mo.VirtualMachine + cdrom *types.VirtualDevice + ) + + Eventually(func(g Gomega) { + g.Expect(propCollector.RetrieveOne(ctx, vmMoRef, []string{"config.hardware.device"}, &moVM)).To(Succeed()) + virtualDevices := object.VirtualDeviceList(moVM.Config.Hardware.Device) + curCdroms := virtualDevices.SelectByType((*types.VirtualCdrom)(nil)) + g.Expect(len(curCdroms)).To(Equal(1)) + cdrom = curCdroms[0].GetVirtualDevice() + g.Expect(cdrom.Connectable.Connected).To(Equal(connected)) + g.Expect(cdrom.Connectable.AllowGuestControl).To(Equal(allowGuestControl)) + }, 1*time.Minute, 5*time.Second).Should(Succeed(), "VM CD-ROM did not have expected connection state") + + return cdrom +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_encryption.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_encryption.go new file mode 100644 index 000000000..835101198 --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_encryption.go @@ -0,0 +1,804 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + crypto "github.com/vmware/govmomi/crypto" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1a3 "github.com/vmware-tanzu/vm-operator/api/v1alpha3" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/testutils" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type VMEncryptionInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *e2eConfig.E2EConfig + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string +} + +func VMEncryptionSpec(ctx context.Context, inputGetter func() VMEncryptionInput) { + const ( + specName = "vm-encryption" + + // Key Providers setup by hack/kms.sh + standardKeyProviderID = "gce2e-standard" + nativeKeyProviderID = "gce2e-native" + ) + + var ( + input VMEncryptionInput + wcpClient wcp.WorkloadManagementAPI + config *e2eConfig.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterConfig *e2eConfig.ManagementClusterConfig + svClusterClient ctrlclient.Client + vCenterClient *vim25.Client + cryptoManager *crypto.ManagerKmip + clusterResources *e2eConfig.Resources + tmpNamespaceCtx wcpframework.NamespaceContext + tmpNamespaceName string + vmYaml []byte + vmName string + vmiName string + createSpecE2eTestBestEffortSmallVTPM wcp.VMClassSpec + byokFSSEnabled bool + defaultKeyProviderID string + linuxImageDisplayName string + ) + + BeforeEach(func() { + var err error + + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", specName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + wcpClient = input.WCPClient + config = input.Config + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterConfig = config.InfraConfig.ManagementClusterConfig + clusterResources = svClusterConfig.Resources + svClusterClient = clusterProxy.GetClient() + vCenterClient = vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + cryptoManager = crypto.NewManagerKmip(vCenterClient) + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, input.ClusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, specName)) + DeferCleanup(cancelPodWatches) + + byokFSSEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSBYOK")) + + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + + vmYaml = nil + vmName = fmt.Sprintf("%s-vm-%s", specName, capiutil.RandomString(4)) + + By("Create a new namespace") + + vmserviceCLID := vmservice.GetContentLibraryUUIDByName(consts.VMServiceCLName, wcpClient) + clIDs := []string{vmserviceCLID} + vmClassNames := []string{clusterResources.VMClassName} + vmsvcSpecs := wcp.NewVMServiceSpecDetails(vmClassNames, clIDs) + + tmpNamespaceName = fmt.Sprintf("%s-ns-%s", specName, capiutil.RandomString(4)) + tmpNamespaceCtx, err = clusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, clusterResources.StorageClassName, clusterResources.WorkerStorageClassName, tmpNamespaceName, input.ArtifactFolder) + Expect(err).NotTo(HaveOccurred(), "failed to create wcp namespace %s", tmpNamespaceName) + + wcp.WaitForNamespaceReady(wcpClient, tmpNamespaceName) + + vmiName, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VM Image name in namespace %q", tmpNamespaceName) + + By(utils.E2EEncryptionStorageProfileName + " should exist") + Expect(utils.EnsureE2EEncryptionStorageInNamespace(ctx, vCenterClient, + wcpClient, clusterProxy.GetClientSet(), svClusterClient, *config, + tmpNamespaceName, clusterResources.StorageClassName)). + To(Succeed(), "failed to ensure encryption storage in namespace %s", + tmpNamespaceName) + + defaultKeyProviderID, err = cryptoManager.GetDefaultKmsClusterID(ctx, nil, true) + Expect(err).NotTo(HaveOccurred(), "failed to get default Key Provider ID") + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), tmpNamespaceName, vmName, "vm") + } + // Delete the virtual machine if it was created. + if len(vmYaml) > 0 { + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).To(Succeed(), "failed to delete virtualmachine") + // Verify that virtual machine does not exist. + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, tmpNamespaceName, vmName) + } + + // Delete vTPM class if it was created + if createSpecE2eTestBestEffortSmallVTPM.ID != "" { + vmservice.VerifyVMClassDeletion(wcpClient, createSpecE2eTestBestEffortSmallVTPM.ID) + createSpecE2eTestBestEffortSmallVTPM.ID = "" + } + // Delete the temporary namespace if it was created. + if tmpNamespaceCtx.GetNamespace() != nil { + clusterProxy.DeleteWCPNamespace(tmpNamespaceCtx) + wcp.WaitForNamespaceDeleted(wcpClient, tmpNamespaceCtx.GetNamespace().Name) + } + + _ = cryptoManager.SetDefaultKmsClusterId(ctx, defaultKeyProviderID, nil) + + vcenter.LogoutVimClient(vCenterClient) + }) + + It("Create an Encrypted VirtualMachine using encryption storage policy", Label("smoke"), func() { + useKeyProvider(ctx, cryptoManager, nativeKeyProviderID) + + By("Create VM using encryption storage policy") + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: "best-effort-small", + StorageClassName: utils.E2EEncryptionStorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + + if byokFSSEnabled { + waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "") + } + }) + + It("Create an Encrypted VirtualMachine using VM Class configured with vTPM", func() { + useKeyProvider(ctx, cryptoManager, nativeKeyProviderID) + + By("Create VM Class with vTPM and ensure namespace has access") + + createSpecE2eTestBestEffortSmallVTPM = vmservice.CreateSpecE2eVMClassVTPMConfigSpec() + vmservice.VerifyVMClassCreate(wcpClient, createSpecE2eTestBestEffortSmallVTPM, createSpecE2eTestBestEffortSmallVTPM) + vmservice.EnsureVMClassAccess(wcpClient, createSpecE2eTestBestEffortSmallVTPM.ID, tmpNamespaceName) + + By("Create VM using vTPM class") + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: createSpecE2eTestBestEffortSmallVTPM.ID, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + Annotations: map[string]string{"vmoperator.vmware.com/firmware": "efi"}, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + + if byokFSSEnabled { + waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "") + } + }) + + It("Create an Encrypted VirtualMachine using vTPM and encryption class", func() { + if !byokFSSEnabled { + Skip("BYOK FSS is not enabled") + } + + // See VCFCON-2837 + adminClusterProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + Expect(err).ToNot(HaveOccurred()) + + defer adminClusterProxy.Dispose(ctx) + + By("Create Encryption Class") + + class := manifestbuilders.EncryptionClass{ + Namespace: tmpNamespaceName, + Name: nativeKeyProviderID, + KeyProvider: nativeKeyProviderID, + } + ecYaml := manifestbuilders.GetEncryptionClassYaml(class) + Expect(adminClusterProxy.CreateWithArgs(ctx, ecYaml)).Should(Succeed(), "failed to create EncryptionClass:\n %s", string(ecYaml)) + useKeyProvider(ctx, cryptoManager, class.KeyProvider) // TODO: should not need to have a default provider set + + By("Create VM Class with vTPM and ensure namespace has access") + + createSpecE2eTestBestEffortSmallVTPM = vmservice.CreateSpecE2eVMClassVTPMConfigSpec() + vmservice.VerifyVMClassCreate(wcpClient, createSpecE2eTestBestEffortSmallVTPM, createSpecE2eTestBestEffortSmallVTPM) + vmservice.EnsureVMClassAccess(wcpClient, createSpecE2eTestBestEffortSmallVTPM.ID, tmpNamespaceName) + + By("Create VM using vTPM class") + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: createSpecE2eTestBestEffortSmallVTPM.ID, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + Annotations: map[string]string{"vmoperator.vmware.com/firmware": "efi"}, + Crypto: &manifestbuilders.Crypto{ + EncryptionClassName: class.Name, + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachineConditionCreated(ctx, config, svClusterClient, tmpNamespaceName, vmName) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, tmpNamespaceName, vmName, "PoweredOn") + waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "") + }) + + It("Create an Encrypted VirtualMachine using encryption class", func() { + if !byokFSSEnabled { + Skip("BYOK FSS is not enabled") + } + + // See VCFCON-2837 + adminClusterProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + Expect(err).ToNot(HaveOccurred()) + + defer adminClusterProxy.Dispose(ctx) + + By("Create Encryption Class") + + class := manifestbuilders.EncryptionClass{ + Namespace: tmpNamespaceName, + Name: standardKeyProviderID, + KeyProvider: standardKeyProviderID, + } + ecYaml := manifestbuilders.GetEncryptionClassYaml(class) + Expect(adminClusterProxy.CreateWithArgs(ctx, ecYaml)).Should(Succeed(), "failed to create EncryptionClass:\n %s", string(ecYaml)) + useKeyProvider(ctx, cryptoManager, class.KeyProvider) // Required when using encryption storage policy + + By("Create VM using invalid encryption class name") + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: "best-effort-small", + StorageClassName: utils.E2EEncryptionStorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: string(vmopv1a3.VirtualMachinePowerStateOn), + Crypto: &manifestbuilders.Crypto{ + EncryptionClassName: "invalid", + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, tmpNamespaceName, vmName) + waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "EncryptionClassNotFound") + + By("Update VM using valid encryption class name") + + vmParameters.Crypto.EncryptionClassName = class.Name + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to update virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + + cryptoStatus := waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "") + Expect(cryptoStatus.ProviderID).To(Equal(class.KeyProvider)) + }) + + It("Create an Encrypted VirtualMachine using an encryption key id", FlakeAttempts(3), func() { + if !byokFSSEnabled { + Skip("BYOK FSS is not enabled") + } + + // See VCFCON-2837 + adminClusterProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + Expect(err).ToNot(HaveOccurred()) + + defer adminClusterProxy.Dispose(ctx) + + By("Create Encryption Class with invalid key") + + keyID, err := cryptoManager.GenerateKey(ctx, standardKeyProviderID) + Expect(err).To(BeNil()) + + class := manifestbuilders.EncryptionClass{ + Namespace: tmpNamespaceName, + Name: standardKeyProviderID, + KeyProvider: standardKeyProviderID, + KeyID: keyID + "-invalid", + } + ecYaml := manifestbuilders.GetEncryptionClassYaml(class) + Expect(adminClusterProxy.CreateWithArgs(ctx, ecYaml)).Should(Succeed(), "failed to create EncryptionClass:\n %s", string(ecYaml)) + useKeyProvider(ctx, cryptoManager, class.KeyProvider) + + By("Create PVC using encryption class") + + clientSet := clusterProxy.GetClientSet() + pvcName := vmName + "-pvc" + testutils.AssertCreatePVC(clientSet, pvcName, tmpNamespaceName, utils.E2EEncryptionStorageClassName) + volumeClaims := clientSet.CoreV1().PersistentVolumeClaims(tmpNamespaceName) + pvc, err := volumeClaims.Get(ctx, pvcName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + pvc.Annotations["csi.vsphere.encryption-class"] = class.Name + _, err = volumeClaims.Update(ctx, pvc, metav1.UpdateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + By("Create VM using encryption class with invalid key") + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: "best-effort-small", + StorageClassName: utils.E2EEncryptionStorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: string(vmopv1a3.VirtualMachinePowerStateOn), + PVCNames: []string{pvcName}, + Crypto: &manifestbuilders.Crypto{ + EncryptionClassName: class.Name, + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "EncryptionClassInvalid") + + By("Update encryption class using valid key") + + class.KeyID = keyID + ecYaml = manifestbuilders.GetEncryptionClassYaml(class) + Expect(adminClusterProxy.ApplyWithArgs(ctx, ecYaml)).Should(Succeed(), "failed to update EncryptionClass:\n %s", string(ecYaml)) + + By("Expect VM to be created using updated encryption class key") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, tmpNamespaceName, vmName, "PoweredOn") + cryptoStatus := waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "") + Expect(cryptoStatus.ProviderID).To(Equal(class.KeyProvider)) + Expect(cryptoStatus.KeyID).To(Equal(class.KeyID)) + }) + + It("Encrypt VM config and classic disks using encryption storage policy", func() { + if !byokFSSEnabled { + Skip("BYOK FSS is not enabled") + } + + useKeyProvider(ctx, cryptoManager, nativeKeyProviderID) + + adminClusterProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + Expect(err).ToNot(HaveOccurred()) + + defer adminClusterProxy.Dispose(ctx) + + By("Create Encryption Class for the VM") + + class := manifestbuilders.EncryptionClass{ + Namespace: tmpNamespaceName, + Name: nativeKeyProviderID, + KeyProvider: nativeKeyProviderID, + } + ecYaml := manifestbuilders.GetEncryptionClassYaml(class) + Expect(adminClusterProxy.CreateWithArgs(ctx, ecYaml)).Should(Succeed(), "failed to create EncryptionClass:\n %s", string(ecYaml)) + + By("Create VM using encryption storage policy and encryption class") + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: "best-effort-small", + StorageClassName: utils.E2EEncryptionStorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: string(vmopv1a3.VirtualMachinePowerStateOn), + Crypto: &manifestbuilders.Crypto{ + EncryptionClassName: class.Name, + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + cryptoStatus := waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "") + + By("Verify VM config and disks are encrypted") + Expect(cryptoStatus.ProviderID).To(Equal(class.KeyProvider)) + Expect(cryptoStatus.Encrypted).To(ContainElement(vmopv1a3.VirtualMachineEncryptionTypeConfig)) + Expect(cryptoStatus.Encrypted).To(ContainElement(vmopv1a3.VirtualMachineEncryptionTypeDisks)) + }) + + It("Encrypt PVC using encryption class annotation on the PVC", func() { + if !byokFSSEnabled { + Skip("BYOK FSS is not enabled") + } + + adminClusterProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + Expect(err).ToNot(HaveOccurred()) + + defer adminClusterProxy.Dispose(ctx) + + By("Create Encryption Class for the PVC") + + class := manifestbuilders.EncryptionClass{ + Namespace: tmpNamespaceName, + Name: standardKeyProviderID, + KeyProvider: standardKeyProviderID, + } + ecYaml := manifestbuilders.GetEncryptionClassYaml(class) + Expect(adminClusterProxy.CreateWithArgs(ctx, ecYaml)).Should(Succeed(), "failed to create EncryptionClass:\n %s", string(ecYaml)) + useKeyProvider(ctx, cryptoManager, class.KeyProvider) + + By("Create PVC with encryption class annotation") + + clientSet := clusterProxy.GetClientSet() + pvcName := vmName + "-pvc" + testutils.AssertCreatePVC(clientSet, pvcName, tmpNamespaceName, utils.E2EEncryptionStorageClassName) + volumeClaims := clientSet.CoreV1().PersistentVolumeClaims(tmpNamespaceName) + pvc, err := volumeClaims.Get(ctx, pvcName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + pvc.Annotations["csi.vsphere.encryption-class"] = class.Name + _, err = volumeClaims.Update(ctx, pvc, metav1.UpdateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + By("Create VM with attached PVC") + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: "best-effort-small", + StorageClassName: utils.E2EEncryptionStorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: string(vmopv1a3.VirtualMachinePowerStateOn), + PVCNames: []string{pvcName}, + Crypto: &manifestbuilders.Crypto{ + EncryptionClassName: class.Name, + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, tmpNamespaceName, vmName, "PoweredOn") + cryptoStatus := waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "") + + By("Verify VM is encrypted with the encryption class provider") + Expect(cryptoStatus.ProviderID).To(Equal(class.KeyProvider)) + Expect(cryptoStatus.KeyID).NotTo(BeEmpty()) + + By("Verify crypto status of volumes reflects volume is encrypted using encryption class from annotation") + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA3(ctx, svClusterClient, tmpNamespaceName, vmName) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(vm.Status.Volumes).To(HaveLen(2)) + + for _, volume := range vm.Status.Volumes { + volumeStatusCrypto := volume.Crypto + g.Expect(volumeStatusCrypto).NotTo(BeNil()) + g.Expect(volumeStatusCrypto.ProviderID).To(Equal(class.KeyProvider)) + g.Expect(volumeStatusCrypto.KeyID).NotTo(BeEmpty()) + } + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out waiting for PVC volume encryption: %s", vmName) + }) + + It("Encrypt PVC using default key provider when no encryption class annotation", func() { + if !byokFSSEnabled { + Skip("BYOK FSS is not enabled") + } + + adminClusterProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + Expect(err).ToNot(HaveOccurred()) + + defer adminClusterProxy.Dispose(ctx) + + useKeyProvider(ctx, cryptoManager, standardKeyProviderID) + + By("Create Encryption Class for the VM") + + class := manifestbuilders.EncryptionClass{ + Namespace: tmpNamespaceName, + Name: standardKeyProviderID, + KeyProvider: standardKeyProviderID, + } + ecYaml := manifestbuilders.GetEncryptionClassYaml(class) + Expect(adminClusterProxy.CreateWithArgs(ctx, ecYaml)).Should(Succeed(), "failed to create EncryptionClass:\n %s", string(ecYaml)) + + By("Create PVC without encryption class annotation") + + clientSet := clusterProxy.GetClientSet() + pvcName := vmName + "-pvc" + testutils.AssertCreatePVC(clientSet, pvcName, tmpNamespaceName, utils.E2EEncryptionStorageClassName) + + By("Create VM with attached PVC and encryption class") + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: "best-effort-small", + StorageClassName: utils.E2EEncryptionStorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: string(vmopv1a3.VirtualMachinePowerStateOn), + PVCNames: []string{pvcName}, + Crypto: &manifestbuilders.Crypto{ + EncryptionClassName: class.Name, + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, tmpNamespaceName, vmName, "PoweredOn") + cryptoStatus := waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "") + + By("Verify VM is encrypted") + Expect(cryptoStatus.ProviderID).To(Equal(class.KeyProvider)) + Expect(cryptoStatus.KeyID).NotTo(BeEmpty()) + + By("Verify crypto status of volumes reflects volume is encrypted using the default key provider") + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA3(ctx, svClusterClient, tmpNamespaceName, vmName) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(vm.Status.Volumes).To(HaveLen(2)) + + for _, volume := range vm.Status.Volumes { + volumeStatusCrypto := volume.Crypto + g.Expect(volumeStatusCrypto).NotTo(BeNil()) + g.Expect(volumeStatusCrypto.ProviderID).NotTo(BeEmpty()) + } + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out waiting for PVC volume encryption with default provider: %s", vmName) + }) + + It("Re-encrypt VM and PVC when encryption class is updated on both", func() { + if !byokFSSEnabled { + Skip("BYOK FSS is not enabled") + } + + adminClusterProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + Expect(err).ToNot(HaveOccurred()) + + defer adminClusterProxy.Dispose(ctx) + + By("Create initial Encryption Class") + + class := manifestbuilders.EncryptionClass{ + Namespace: tmpNamespaceName, + Name: standardKeyProviderID, + KeyProvider: standardKeyProviderID, + } + ecYaml := manifestbuilders.GetEncryptionClassYaml(class) + Expect(adminClusterProxy.CreateWithArgs(ctx, ecYaml)).Should(Succeed(), "failed to create EncryptionClass:\n %s", string(ecYaml)) + useKeyProvider(ctx, cryptoManager, class.KeyProvider) + + By("Create PVC with encryption class annotation") + + clientSet := clusterProxy.GetClientSet() + pvcName := vmName + "-pvc" + testutils.AssertCreatePVC(clientSet, pvcName, tmpNamespaceName, utils.E2EEncryptionStorageClassName) + volumeClaims := clientSet.CoreV1().PersistentVolumeClaims(tmpNamespaceName) + pvc, err := volumeClaims.Get(ctx, pvcName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + pvc.Annotations["csi.vsphere.encryption-class"] = class.Name + _, err = volumeClaims.Update(ctx, pvc, metav1.UpdateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + By("Create VM with attached PVC") + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: "best-effort-small", + StorageClassName: utils.E2EEncryptionStorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: string(vmopv1a3.VirtualMachinePowerStateOn), + PVCNames: []string{pvcName}, + Crypto: &manifestbuilders.Crypto{ + EncryptionClassName: class.Name, + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, tmpNamespaceName, vmName, "PoweredOn") + cryptoStatus := waitForCryptoCondition(ctx, config, svClusterClient, tmpNamespaceName, vmName, "") + Expect(cryptoStatus.ProviderID).To(Equal(class.KeyProvider)) + Expect(cryptoStatus.KeyID).NotTo(BeEmpty()) + initialKeyID := cryptoStatus.KeyID + + By("Update encryption class with a new key") + + newKeyID, err := cryptoManager.GenerateKey(ctx, standardKeyProviderID) + Expect(err).ToNot(HaveOccurred()) + + class.KeyID = newKeyID + ecYaml = manifestbuilders.GetEncryptionClassYaml(class) + Expect(adminClusterProxy.ApplyWithArgs(ctx, ecYaml)).Should(Succeed(), "failed to update EncryptionClass:\n %s", string(ecYaml)) + + By("Verify VM is re-encrypted with new key") + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA3(ctx, svClusterClient, tmpNamespaceName, vmName) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(vm.Status.Crypto).NotTo(BeNil()) + g.Expect(vm.Status.Crypto.ProviderID).To(Equal(class.Name)) + g.Expect(vm.Status.Crypto.KeyID).To(Equal(newKeyID)) + g.Expect(vm.Status.Crypto.KeyID).NotTo(Equal(initialKeyID)) + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out waiting for VM re-encryption: %s", vmName) + + By("Verify crypto status of volumes reflects the encryption class") + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA3(ctx, svClusterClient, tmpNamespaceName, vmName) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(vm.Status.Volumes).To(HaveLen(2)) + + for _, volume := range vm.Status.Volumes { + volumeStatusCrypto := volume.Crypto + g.Expect(volumeStatusCrypto).NotTo(BeNil()) + g.Expect(volumeStatusCrypto.ProviderID).To(Equal(class.Name)) + } + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out waiting for PVC volume re-encryption: %s", vmName) + }) + + It("Error when PVC encryption class uses different provider type than VM", func() { + if !byokFSSEnabled { + Skip("BYOK FSS is not enabled") + } + + adminClusterProxy, err := clusterProxy.NewAdminClusterProxy(ctx) + Expect(err).ToNot(HaveOccurred()) + + defer adminClusterProxy.Dispose(ctx) + + By("Create Encryption Class using native key provider for the VM") + + vmClass := manifestbuilders.EncryptionClass{ + Namespace: tmpNamespaceName, + Name: nativeKeyProviderID, + KeyProvider: nativeKeyProviderID, + } + vmECYaml := manifestbuilders.GetEncryptionClassYaml(vmClass) + Expect(adminClusterProxy.CreateWithArgs(ctx, vmECYaml)).Should(Succeed(), "failed to create EncryptionClass:\n %s", string(vmECYaml)) + useKeyProvider(ctx, cryptoManager, nativeKeyProviderID) + + By("Create Encryption Class using standard key provider for the PVC") + + pvcClass := manifestbuilders.EncryptionClass{ + Namespace: tmpNamespaceName, + Name: standardKeyProviderID, + KeyProvider: standardKeyProviderID, + } + pvcECYaml := manifestbuilders.GetEncryptionClassYaml(pvcClass) + Expect(adminClusterProxy.CreateWithArgs(ctx, pvcECYaml)).Should(Succeed(), "failed to create EncryptionClass:\n %s", string(pvcECYaml)) + + By("Create PVC annotated with the standard key provider encryption class") + + clientSet := clusterProxy.GetClientSet() + pvcName := vmName + "-pvc" + testutils.AssertCreatePVC(clientSet, pvcName, tmpNamespaceName, utils.E2EEncryptionStorageClassName) + volumeClaims := clientSet.CoreV1().PersistentVolumeClaims(tmpNamespaceName) + pvc, err := volumeClaims.Get(ctx, pvcName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + pvc.Annotations["csi.vsphere.encryption-class"] = pvcClass.Name + _, err = volumeClaims.Update(ctx, pvc, metav1.UpdateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + By("Create VM with native provider encryption class and PVC using standard provider") + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + ImageName: vmiName, + VMClassName: "best-effort-small", + StorageClassName: utils.E2EEncryptionStorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: string(vmopv1a3.VirtualMachinePowerStateOn), + PVCNames: []string{pvcName}, + Crypto: &manifestbuilders.Crypto{ + EncryptionClassName: vmClass.Name, + }, + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA3(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).Should(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + By("Verify the encryption synced condition reports an error due to mixed provider types") + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA3(ctx, svClusterClient, tmpNamespaceName, vmName) + g.Expect(err).ToNot(HaveOccurred()) + + condition := vmoperator.GetVirtualMachineConditionA3(vm, vmopv1a3.VirtualMachineEncryptionSynced) + g.Expect(condition).ToNot(BeNil()) + g.Expect(condition.Status).To(Equal(metav1.ConditionFalse), + "expected EncryptionSynced to be False due to mixed provider types, got %s with reason %s: %s", + condition.Status, condition.Reason, condition.Message) + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), + "Timed out waiting for encryption error with mixed provider types: %s", vmName) + }) +} + +func useKeyProvider(ctx context.Context, cryptoManager *crypto.ManagerKmip, keyProviderID string) { + By(fmt.Sprintf("Use %s key provider and mark it as default", keyProviderID)) + status, err := cryptoManager.GetClusterStatus(ctx, keyProviderID) + Expect(err).NotTo(HaveOccurred(), "error fetching status of key provider %s", keyProviderID) + + Expect(status.OverallStatus).To(Equal(types.ManagedEntityStatusGreen)) + + Expect(cryptoManager.SetDefaultKmsClusterId(ctx, keyProviderID, nil)).To(Succeed()) +} + +func waitForCryptoCondition(ctx context.Context, _ *e2eConfig.E2EConfig, client ctrlclient.Client, ns string, vmName string, reason string) *vmopv1a3.VirtualMachineCryptoStatus { + expectedCondition := metav1.Condition{ + Type: vmopv1a3.VirtualMachineEncryptionSynced, + Status: metav1.ConditionTrue, + } + if reason != "" { + expectedCondition.Status = metav1.ConditionFalse + expectedCondition.Reason = reason + } + + By(fmt.Sprintf("Waiting for VirtualMachine.Status.Conditions[%s]", expectedCondition.Type)) + // Note: this is vmoperator.WaitOnVirtualMachineCondition with diff: + // - Using A3 + // - Check Reason before Status, gives more context on failure + // - Much shorter timeout + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA3(ctx, client, ns, vmName) + g.Expect(err).ToNot(HaveOccurred()) + + actualCondition := vmoperator.GetVirtualMachineConditionA3(vm, expectedCondition.Type) + g.Expect(actualCondition).ToNot(BeNil()) + + if actualCondition.Status == metav1.ConditionFalse { + g.Expect(actualCondition.Reason).Should(Equal(expectedCondition.Reason)) + } + + g.Expect(actualCondition.Status).Should(Equal(expectedCondition.Status)) + }, time.Minute*2, time.Second*5).Should(Succeed(), fmt.Sprintf("%s condition not %s", expectedCondition.Type, expectedCondition.Status)) + + By("Checking VirtualMachine.Status.Crypto") + + vm, err := utils.GetVirtualMachineA3(ctx, client, ns, vmName) + Expect(err).To(BeNil()) + + if expectedCondition.Status == metav1.ConditionTrue { + Expect(vm.Status.Crypto.KeyID).NotTo(BeEmpty()) + Expect(vm.Status.Crypto.Encrypted).NotTo(BeEmpty()) + Expect(vm.Status.Crypto.ProviderID).NotTo(BeEmpty()) + } else { + Expect(vm.Status.Crypto).To(BeNil()) + } + + return vm.Status.Crypto +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_group.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_group.go new file mode 100644 index 000000000..1eb37f1da --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_group.go @@ -0,0 +1,946 @@ +// © Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vapi/tags" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type VMGroupSpecInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *e2eConfig.E2EConfig + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + WCPNamespaceName string +} + +func VMGroupSpec(ctx context.Context, inputGetter func() VMGroupSpecInput) { + const ( + specName = "vm-group" + vmKind = "VirtualMachine" + vmgKind = "VirtualMachineGroup" + ) + + var ( + input VMGroupSpecInput + config *e2eConfig.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterClient ctrlclient.Client + vCenterClient *vim25.Client + clusterResources *e2eConfig.Resources + + vmgRootYaml []byte + vmgRootName string + vmgChildName string + vm1Name string + vm2Name string + vm3Name string + vm4Name string + vmMemberNames []string + + linuxImageDisplayName string + ) + + BeforeEach(func() { + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", specName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", specName) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + config = input.Config + clusterResources = config.InfraConfig.ManagementClusterConfig.Resources + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, clusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, specName)) + DeferCleanup(cancelPodWatches) + skipper.SkipUnlessSupervisorCapabilityEnabled(ctx, clusterProxy, consts.VMGroupsCapabilityName) + + svClusterClient = clusterProxy.GetClient() + vCenterClient = vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + + vmgRootYaml = nil + vmMemberNames = []string{} + vmgRootName = fmt.Sprintf("%s-%s-root", specName, capiutil.RandomString(4)) + vmgChildName = fmt.Sprintf("%s-child", vmgRootName) + vm1Name = fmt.Sprintf("%s-vm1", vmgRootName) + vm2Name = fmt.Sprintf("%s-vm2", vmgRootName) + vm3Name = fmt.Sprintf("%s-vm3", vmgRootName) + vm4Name = fmt.Sprintf("%s-vm4", vmgRootName) + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, vmgRootName, vmgKind) + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, vmgChildName, vmgKind) + + for _, vmName := range vmMemberNames { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, vmName, vmKind) + } + } + + // Delete the root VirtualMachineGroup if created. + if len(vmgRootYaml) > 0 { + By("Deleting the root VirtualMachineGroup") + Expect(clusterProxy.DeleteWithArgs(ctx, vmgRootYaml)).To(Succeed(), "failed to delete VirtualMachineGroup") + vmoperator.WaitForVirtualMachineGroupToBeDeleted(ctx, config, svClusterClient, input.WCPNamespaceName, vmgRootName) + + By("Waiting for all group members to be deleted automatically due to owner reference to the root group") + vmoperator.WaitForVirtualMachineGroupToBeDeleted(ctx, config, svClusterClient, input.WCPNamespaceName, vmgChildName) + + for _, vmName := range vmMemberNames { + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + } + } + + if vCenterClient != nil { + vcenter.LogoutVimClient(vCenterClient) + } + }) + + Context("Flat group", func() { + It("Should create and manage a VirtualMachineGroup with VM-kind members only", Label("smoke"), func() { + By("Creating a VirtualMachineGroup with 3 VMs and power on delays") + + vmGroupParameters := manifestbuilders.VirtualMachineGroupYaml{ + Namespace: input.WCPNamespaceName, + Name: vmgRootName, + BootOrder: []manifestbuilders.BootOrder{ + { + // No power on delay for the first boot order. + Members: []vmopv1a5.GroupMember{ + { + Kind: vmKind, + Name: vm1Name, + }, + }, + }, + { + PowerOnDelay: "30s", + Members: []vmopv1a5.GroupMember{ + { + Kind: vmKind, + Name: vm2Name, + }, + }, + }, + { + PowerOnDelay: "1m", + Members: []vmopv1a5.GroupMember{ + { + Kind: vmKind, + Name: vm3Name, + }, + }, + }, + }, + } + vmgRootYaml = manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmGroupParameters) + e2eframework.Logf("VirtualMachineGroup YAML:\n%s", string(vmgRootYaml)) + Expect(clusterProxy.CreateWithArgs(ctx, vmgRootYaml)).To(Succeed()) + + vmMemberNames = []string{vm1Name, vm2Name, vm3Name} + + By("Creating VMs with initially powered off state and spec.GroupName pointing to the VirtualMachineGroup") + + for _, vmName := range vmMemberNames { + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + GroupName: vmgRootName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PowerState: "PoweredOff", + } + vmYaml := manifestbuilders.GetVirtualMachineYamlA5(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create VM %q:\n %s", vmName, string(vmYaml)) + } + + By("Waiting for all VMs to exist") + + for _, vmName := range vmMemberNames { + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + } + + By("Waiting for all VMs to have group linked condition set to true") + + groupLinkedTrueCondition := metav1.Condition{ + Type: vmopv1a5.VirtualMachineGroupMemberConditionGroupLinked, + Status: metav1.ConditionTrue, + } + for _, vmName := range vmMemberNames { + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, groupLinkedTrueCondition) + } + + By("Verifying VirtualMachineGroup has Ready condition set to true") + + readyTrueCondition := metav1.Condition{ + Type: vmopv1a5.ReadyConditionType, + Status: metav1.ConditionTrue, + } + vmoperator.WaitOnVirtualMachineGroupCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmgRootName, readyTrueCondition) + + By("Setting VirtualMachineGroup spec.powerState to PoweredOn") + + vmGroupParameters.PowerState = "PoweredOn" + vmgRootYaml = manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmGroupParameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vmgRootYaml)).To(Succeed(), "failed to update VirtualMachineGroup power state:\n %s", string(vmgRootYaml)) + + By("Waiting for all VMs to be powered on") + + for _, vmName := range vmMemberNames { + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOn") + } + + By("Verifying VirtualMachineGroup has Ready condition set to true after power on") + vmoperator.WaitOnVirtualMachineGroupCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmgRootName, readyTrueCondition) + + By("Verifying VirtualMachineGroup has expected member statuses") + + for _, bootOrder := range vmGroupParameters.BootOrder { + for _, m := range bootOrder.Members { + ms, err := utils.GetVirtualMachineGroupMemberStatus(ctx, svClusterClient, input.WCPNamespaceName, vmgRootName, m.Name, m.Kind) + Expect(err).ToNot(HaveOccurred()) + Expect(ms.PowerState).ToNot(BeNil(), "member %s/%s power state status is nil", m.Name, m.Kind) + Expect(*ms.PowerState).To(BeEquivalentTo("PoweredOn")) + // Just check the placement status is not nil here. + // The exact placement info will be verified in affinity/anti-affinity tests. + Expect(ms.Placement).ToNot(BeNil(), "member %s/%s placement status is nil", m.Name, m.Kind) + + // Verify all expected member conditions are set to true. + expectedConditionTypes := []string{ + vmopv1a5.VirtualMachineGroupMemberConditionGroupLinked, + vmopv1a5.VirtualMachineGroupMemberConditionPowerStateSynced, + vmopv1a5.VirtualMachineGroupMemberConditionPlacementReady, + } + + Expect(ms.Conditions).To(HaveLen(3)) // GroupLinked, PowerStateSynced, PlacementReady + + for _, c := range ms.Conditions { + Expect(c.Type).To(BeElementOf(expectedConditionTypes)) + Expect(c.Status).To(Equal(metav1.ConditionTrue)) + } + } + } + + By("Verifying group member VMs were powered on with specified delays", func() { + vcClient := vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + defer vcenter.LogoutVimClient(vcClient) + + propCollector := property.DefaultCollector(vcClient) + + vm1Moid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vm1Name) + vm2Moid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vm2Name) + vm3Moid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vm3Name) + + vm1MoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vm1Moid} + vm2MoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vm2Moid} + vm3MoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vm3Moid} + + var vm1MO, vm2MO, vm3MO mo.VirtualMachine + Expect(propCollector.RetrieveOne(ctx, vm1MoRef, []string{"runtime.powerState", "runtime.bootTime"}, &vm1MO)).To(Succeed()) + Expect(propCollector.RetrieveOne(ctx, vm2MoRef, []string{"runtime.powerState", "runtime.bootTime"}, &vm2MO)).To(Succeed()) + Expect(propCollector.RetrieveOne(ctx, vm3MoRef, []string{"runtime.powerState", "runtime.bootTime"}, &vm3MO)).To(Succeed()) + + // Verify all VMs are powered on. + Expect(vm1MO.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + Expect(vm2MO.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + Expect(vm3MO.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + + // Verify boot times to be within the expected delays. + // VM1 should boot with no delay (VM1's boot order has no powerOnDelay). + // VM2 should boot ~30 seconds after VM1 is powered on (VM2's boot order has powerOnDelay set to 30s). + // VM3 should boot ~1 minute after VM2 is powered on (VM3's boot order has powerOnDelay set to 1m). + Expect(vm1MO.Runtime.BootTime).ToNot(BeNil()) + Expect(vm2MO.Runtime.BootTime).ToNot(BeNil()) + Expect(vm3MO.Runtime.BootTime).ToNot(BeNil()) + vm1BootTime := *vm1MO.Runtime.BootTime + vm2BootTime := *vm2MO.Runtime.BootTime + vm3BootTime := *vm3MO.Runtime.BootTime + vm2DelayFromVM1 := vm2BootTime.Sub(vm1BootTime) + vm3DelayFromVM2 := vm3BootTime.Sub(vm2BootTime) + By(fmt.Sprintf("VM boot timing: VM1: %v, VM2: %v (delay from VM1: %v), VM3: %v (delay from VM2: %v)", + vm1BootTime, vm2BootTime, vm2DelayFromVM1, vm3BootTime, vm3DelayFromVM2)) + + // Allow some tolerance (±10 seconds) for VM controller to reconcile and actually power on the VMs. + Expect(vm2DelayFromVM1).To(BeNumerically(">=", 20*time.Second), "VM2 boot delay should be at least 20s from VM1") + Expect(vm2DelayFromVM1).To(BeNumerically("<=", 40*time.Second), "VM2 boot delay should be at most 40s from VM1") + Expect(vm3DelayFromVM2).To(BeNumerically(">=", 50*time.Second), "VM3 boot delay should be at least 50s from VM2") + Expect(vm3DelayFromVM2).To(BeNumerically("<=", 70*time.Second), "VM3 boot delay should be at most 70s from VM2") + }) + + By("Creating a new standalone VM4 with spec.groupName unset and spec.powerState set to PoweredOn") + + vm4Parameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vm4Name, + PowerState: "PoweredOn", + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + } + vm4Yaml := manifestbuilders.GetVirtualMachineYamlA5(vm4Parameters) + Expect(clusterProxy.CreateWithArgs(ctx, vm4Yaml)).To(Succeed(), "failed to create VM %q:\n %s", vm4Name, string(vm4Yaml)) + + By("Waiting for VM4 to be created and powered on") + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, input.WCPNamespaceName, vm4Name) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vm4Name, "PoweredOn") + + By("Setting VM4 spec.groupName to the VirtualMachineGroup") + + vm4Parameters.GroupName = vmgRootName + vm4Yaml = manifestbuilders.GetVirtualMachineYamlA5(vm4Parameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vm4Yaml)).To(Succeed(), "failed to update VM %q group name:\n %s", vm4Name, string(vm4Yaml)) + + By("Updating VirtualMachineGroup to adopt the existing VM4") + + vmGroupParameters.BootOrder = append(vmGroupParameters.BootOrder, manifestbuilders.BootOrder{ + Members: []vmopv1a5.GroupMember{ + { + Kind: vmKind, + Name: vm4Name, + }, + }, + }) + vmgRootYaml = manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmGroupParameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vmgRootYaml)).To(Succeed(), "failed to update VirtualMachineGroup members:\n %s", string(vmgRootYaml)) + + vmMemberNames = append(vmMemberNames, vm4Name) + + By("Changing VirtualMachineGroup spec.powerState to PoweredOff and spec.powerOffMode to Hard") + + vmGroupParameters.PowerState = "PoweredOff" + vmGroupParameters.PowerOffMode = "Hard" + vmgRootYaml = manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmGroupParameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vmgRootYaml)).To(Succeed(), "failed to update VirtualMachineGroup power state:\n %s", string(vmgRootYaml)) + + By("Waiting for all VMs to be powered off") + + for _, vmName := range vmMemberNames { + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOff") + } + + By("Verifying VirtualMachineGroup has Ready condition set to true after power off") + vmoperator.WaitOnVirtualMachineGroupCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmgRootName, readyTrueCondition) + + By("Changing VM1 spec.powerState directly to PoweredOn") + + vm1Parameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vm1Name, + PowerState: "PoweredOn", + GroupName: vmgRootName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ImageName: linuxImageDisplayName, + } + vm1Yaml := manifestbuilders.GetVirtualMachineYamlA5(vm1Parameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vm1Yaml)).To(Succeed(), "failed to update VM %q power state:\n %s", vm1Name, string(vm1Yaml)) + + By("Waiting for VM1 to be powered on") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vm1Name, "PoweredOn") + + By("Verifying VirtualMachineGroup member status has PowerStateSynced condition set to false for VM1") + + powerStateSyncedFalseCondition := metav1.Condition{ + Type: vmopv1a5.VirtualMachineGroupMemberConditionPowerStateSynced, + Status: metav1.ConditionFalse, + Reason: "NotSynced", + } + vmoperator.WaitOnVirtualMachineGroupMemberCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmgRootName, vm1Name, vmKind, powerStateSyncedFalseCondition) + + By("Verifying VirtualMachineGroup member status has the actual power state recorded for VM1") + + vm1MemberStatus, err := utils.GetVirtualMachineGroupMemberStatus(ctx, svClusterClient, input.WCPNamespaceName, vmgRootName, vm1Name, vmKind) + Expect(err).ToNot(HaveOccurred()) + Expect(vm1MemberStatus.PowerState).ToNot(BeNil(), "VM1 member status power state is nil") + Expect(*vm1MemberStatus.PowerState).To(BeEquivalentTo("PoweredOn")) + + By("Setting VirtualMachineGroup nextForcePowerStateSyncTime to 'now'") + + vmGroupParameters.NextForcePowerStateSyncTime = "now" + vmgRootYaml = manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmGroupParameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vmgRootYaml)).To(Succeed(), "failed to update VirtualMachineGroup force sync time:\n %s", string(vmgRootYaml)) + + By("Verifying VirtualMachineGroup has Ready condition set to true after force power state sync") + vmoperator.WaitOnVirtualMachineGroupCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmgRootName, readyTrueCondition) + + By("Verifying power state sync forces VM1 back to PoweredOff to match group state") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vm1Name, "PoweredOff") + }) + }) + + Context("Nested group", func() { + It("Should create and manage a VirtualMachineGroup with both VMG-kind and VM-kind members", func() { + By("Creating a root VirtualMachineGroup with VM-1 and a child group with power on delays") + + vmgRootParameters := manifestbuilders.VirtualMachineGroupYaml{ + Namespace: input.WCPNamespaceName, + Name: vmgRootName, + BootOrder: []manifestbuilders.BootOrder{ + { + Members: []vmopv1a5.GroupMember{ + { + Kind: vmKind, + Name: vm1Name, + }, + }, + }, + { + PowerOnDelay: "30s", + Members: []vmopv1a5.GroupMember{ + { + Kind: vmgKind, + Name: vmgChildName, + }, + }, + }, + }, + } + vmgRootYaml = manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmgRootParameters) + e2eframework.Logf("Root VirtualMachineGroup YAML:\n%s", string(vmgRootYaml)) + Expect(clusterProxy.CreateWithArgs(ctx, vmgRootYaml)).To(Succeed()) + + vmMemberNames = []string{vm1Name} + + By("Creating a child VirtualMachineGroup with VM-2 and power on delay") + + vmgChildParameters := manifestbuilders.VirtualMachineGroupYaml{ + Namespace: input.WCPNamespaceName, + Name: vmgChildName, + GroupName: vmgRootName, + BootOrder: []manifestbuilders.BootOrder{ + { + Members: []vmopv1a5.GroupMember{ + { + Kind: vmKind, + Name: vm2Name, + }, + }, + PowerOnDelay: "1m", + }, + }, + } + vmgChildYaml := manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmgChildParameters) + e2eframework.Logf("Child VirtualMachineGroup YAML:\n%s", string(vmgChildYaml)) + Expect(clusterProxy.CreateWithArgs(ctx, vmgChildYaml)).To(Succeed()) + + vmMemberNames = append(vmMemberNames, vm2Name) + + By("Creating VM1 in the root group with powered off") + + vm1Parameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vm1Name, + GroupName: vmgRootName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PowerState: "PoweredOff", + } + vm1Yaml := manifestbuilders.GetVirtualMachineYamlA5(vm1Parameters) + Expect(clusterProxy.CreateWithArgs(ctx, vm1Yaml)).To(Succeed(), "failed to create vm1:\n %s", string(vm1Yaml)) + + By("Creating VM2 in the child group") + + vm2Parameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vm2Name, + GroupName: vmgChildName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PowerState: "PoweredOff", + } + vm2Yaml := manifestbuilders.GetVirtualMachineYamlA5(vm2Parameters) + Expect(clusterProxy.CreateWithArgs(ctx, vm2Yaml)).To(Succeed(), "failed to create vm2:\n %s", string(vm2Yaml)) + + By("Waiting for all VMs to exist") + + for _, vmName := range vmMemberNames { + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + } + + By("Waiting for all group members to have group linked condition set to true") + + groupLinkedTrueCondition := metav1.Condition{ + Type: vmopv1a5.VirtualMachineGroupMemberConditionGroupLinked, + Status: metav1.ConditionTrue, + } + for _, vmName := range vmMemberNames { + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, groupLinkedTrueCondition) + } + + vmoperator.WaitOnVirtualMachineGroupCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmgChildName, groupLinkedTrueCondition) + + By("Verifying both root and child VirtualMachineGroups are ready") + + readyTrueCondition := metav1.Condition{ + Type: vmopv1a5.ReadyConditionType, + Status: metav1.ConditionTrue, + } + vmoperator.WaitOnVirtualMachineGroupCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmgChildName, readyTrueCondition) + vmoperator.WaitOnVirtualMachineGroupCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmgRootName, readyTrueCondition) + + By("Setting root VirtualMachineGroup spec.powerState to PoweredOn") + + vmgRootParameters.PowerState = "PoweredOn" + vmgRootYaml = manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmgRootParameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vmgRootYaml)).To(Succeed(), "failed to update root VirtualMachineGroup power state:\n %s", string(vmgRootYaml)) + + By("Waiting for all VMs to be powered on") + + for _, vmName := range vmMemberNames { + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOn") + } + + By("Verifying group member VMs were powered on with specified delays", func() { + vcClient := vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + defer vcenter.LogoutVimClient(vcClient) + + propCollector := property.DefaultCollector(vcClient) + + vm1Moid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vm1Name) + vm2Moid := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vm2Name) + + vm1MoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vm1Moid} + vm2MoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vm2Moid} + + var vm1MO, vm2MO mo.VirtualMachine + Expect(propCollector.RetrieveOne(ctx, vm1MoRef, []string{"runtime.powerState", "runtime.bootTime"}, &vm1MO)).To(Succeed()) + Expect(propCollector.RetrieveOne(ctx, vm2MoRef, []string{"runtime.powerState", "runtime.bootTime"}, &vm2MO)).To(Succeed()) + + // Verify all VMs are powered on. + Expect(vm1MO.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + Expect(vm2MO.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + + // Verify boot times to be within the expected delays. + // VM1 should boot with no delay (VM1's boot order has no powerOnDelay). + // VM2 should boot ~90 seconds after VM1 is powered on (VM2's boot order has 1m powerOnDelay in child group, which also has 30s powerOnDelay in root group). + Expect(vm1MO.Runtime.BootTime).ToNot(BeNil()) + Expect(vm2MO.Runtime.BootTime).ToNot(BeNil()) + vm1BootTime := *vm1MO.Runtime.BootTime + vm2BootTime := *vm2MO.Runtime.BootTime + vm2DelayFromVM1 := vm2BootTime.Sub(vm1BootTime) + By(fmt.Sprintf("VM boot timing: VM1: %v, VM2: %v (delay from VM1: %v)", + vm1BootTime, vm2BootTime, vm2DelayFromVM1)) + + // Allow some tolerance (±30 seconds from 90 seconds) for VM controller to reconcile nested groups and actually power on the VMs. + Expect(vm2DelayFromVM1).To(BeNumerically(">=", 60*time.Second), "VM2 boot delay should be at least 60s from VM1") + Expect(vm2DelayFromVM1).To(BeNumerically("<=", 120*time.Second), "VM2 boot delay should be at most 120s from VM1") + }) + + By("Changing root VirtualMachineGroup spec.powerState to PoweredOff") + + vmgRootParameters.PowerState = "PoweredOff" + vmgRootParameters.PowerOffMode = "Hard" + vmgRootYaml = manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmgRootParameters) + Expect(clusterProxy.ApplyWithArgs(ctx, vmgRootYaml)).To(Succeed(), "failed to update root VirtualMachineGroup power state:\n %s", string(vmgRootYaml)) + + By("Waiting for all VMs to be powered off") + + for _, vmName := range vmMemberNames { + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOff") + } + + By("Verifying both VirtualMachineGroups are ready after power off") + vmoperator.WaitOnVirtualMachineGroupCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmgChildName, readyTrueCondition) + vmoperator.WaitOnVirtualMachineGroupCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmgRootName, readyTrueCondition) + }) + }) + + Context("Group placement with affinity and anti-affinity", func() { + var ( + tmpNamespaceName string + tmpNamespaceCtx wcpframework.NamespaceContext + + createVMWithAffinityAndAntiAffinityFunc = func(vmName, affinityTier string, antiAffinityTiers []string) { + GinkgoHelper() + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: tmpNamespaceName, + Name: vmName, + GroupName: vmgRootName, + Labels: map[string]string{"tier": affinityTier}, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + Affinity: &vmopv1a5.AffinitySpec{ + VMAffinity: &vmopv1a5.VMAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1a5.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "tier": affinityTier, + }, + }, + TopologyKey: "topology.kubernetes.io/zone", + }, + }, + }, + VMAntiAffinity: &vmopv1a5.VMAntiAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1a5.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "tier", + Operator: metav1.LabelSelectorOpIn, + Values: antiAffinityTiers, + }, + }, + }, + TopologyKey: "topology.kubernetes.io/zone", + }, + }, + }, + }, + } + vmYAML := manifestbuilders.GetVirtualMachineYamlA5(vmParameters) + e2eframework.Logf("VM YAML:\n%s", string(vmYAML)) + Expect(clusterProxy.ApplyWithArgs(ctx, vmYAML)).To(Succeed(), "failed to create vm %s:\n %s", vmName, string(vmYAML)) + } + + getVMZoneFunc = func(vmName string) string { + GinkgoHelper() + + vm, err := utils.GetVirtualMachine(ctx, svClusterClient, tmpNamespaceName, vmName) + Expect(err).ToNot(HaveOccurred()) + Expect(vm.Status.Zone).ToNot(BeEmpty()) + + return vm.Status.Zone + } + ) + + BeforeEach(func() { + skipper.SkipUnlessStretchSupervisorIsEnabled() + skipper.SkipUnlessSupervisorCapabilityEnabled(ctx, clusterProxy, consts.VMPlacementPoliciesCapabilityName) + + By("Verifying there are at least 3 zones bound with the Supervisor") + + supervisorID := vcenter.GetSupervisorIDFromKubeconfig(ctx, config.InfraConfig.KubeconfigPath) + Expect(supervisorID).ToNot(BeEmpty(), "Supervisor ID should not be empty") + zoneList, err := clusterProxy.GetZonesBoundWithSupervisor(supervisorID) + Expect(err).ToNot(HaveOccurred(), "failed to get zones bound with Supervisor") + Expect(len(zoneList.Zones)).To(BeNumerically(">=", 3)) + + By("Creating a temporary namespace") + + vmserviceCLID := vmservice.GetContentLibraryUUIDByName(consts.VMServiceCLName, input.WCPClient) + clIDs := []string{vmserviceCLID} + vmClassNames := []string{clusterResources.VMClassName} + vmsvcSpecs := wcp.NewVMServiceSpecDetails(vmClassNames, clIDs) + tmpNamespaceCtx, err = clusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, clusterResources.StorageClassName, clusterResources.WorkerStorageClassName, fmt.Sprintf("%s-%s", specName, capiutil.RandomString(6)), input.ArtifactFolder) + Expect(err).ToNot(HaveOccurred(), "failed to create wcp namespace") + Expect(tmpNamespaceCtx.GetNamespace()).ToNot(BeNil(), "namespace should not be nil") + tmpNamespaceName = tmpNamespaceCtx.GetNamespace().Name + wcp.WaitForNamespaceReady(input.WCPClient, tmpNamespaceName) + + By("Ensuring the Linux image is available in the temp namespace") + + _, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, tmpNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get VMI by display name %q in namespace %q", linuxImageDisplayName, tmpNamespaceName) + + By("Binding all zones to the temporary namespace") + + namespaceZones, err := utils.ListZonesByNamespace(ctx, input.ClusterProxy.GetClient(), tmpNamespaceName) + Expect(err).NotTo(HaveOccurred()) + + boundZones := make(map[string]struct{}, len(namespaceZones.Items)) + for _, zone := range namespaceZones.Items { + boundZones[zone.Name] = struct{}{} + } + + unboundZones := []string{} + + for _, zone := range zoneList.Zones { + if _, ok := boundZones[zone.Zone]; !ok { + unboundZones = append(unboundZones, zone.Zone) + } + } + + if len(unboundZones) > 0 { + _, err = clusterProxy.UpdateNamespaceWithZones(ctx, tmpNamespaceName, unboundZones, svClusterClient) + Expect(err).ToNot(HaveOccurred(), "failed to update namespace with Zones") + } + }) + + AfterEach(func() { + if tmpNamespaceName != "" { + clusterProxy.DeleteWCPNamespace(tmpNamespaceCtx) + + tmpNamespaceName = "" + } + }) + + It("Should create VMs with expected zonal affinity and anti-affinity placements in the temporary namespace", func() { + By("Creating a VirtualMachineGroup with 4 VM-kind members") + + vmgParameters := manifestbuilders.VirtualMachineGroupYaml{ + Namespace: tmpNamespaceName, + Name: vmgRootName, + BootOrder: []manifestbuilders.BootOrder{ + { + Members: []vmopv1a5.GroupMember{ + { + Kind: vmKind, + Name: vm1Name, + }, + { + Kind: vmKind, + Name: vm2Name, + }, + { + Kind: vmKind, + Name: vm3Name, + }, + { + Kind: vmKind, + Name: vm4Name, + }, + }, + }, + }, + } + vmgRootYaml = manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmgParameters) + e2eframework.Logf("VirtualMachineGroup YAML:\n%s", string(vmgRootYaml)) + Expect(clusterProxy.CreateWithArgs(ctx, vmgRootYaml)).To(Succeed()) + + By(fmt.Sprintf("Creating VM1 (%s) with tier=1 label, affinity to tier=1, and anti-affinity to tier=2 and tier=3", vm1Name)) + createVMWithAffinityAndAntiAffinityFunc(vm1Name, "1", []string{"2", "3"}) + By(fmt.Sprintf("Creating VM2 (%s) with tier=2 label, affinity to tier=2, and anti-affinity to tier=1 and tier=3", vm2Name)) + createVMWithAffinityAndAntiAffinityFunc(vm2Name, "2", []string{"1", "3"}) + By(fmt.Sprintf("Creating VM3 (%s) with tier=3 label, affinity to tier=3, and anti-affinity to tier=1 and tier=2", vm3Name)) + createVMWithAffinityAndAntiAffinityFunc(vm3Name, "3", []string{"1", "2"}) + By(fmt.Sprintf("Creating VM4 (%s) with tier=1 label, affinity to tier=1, and anti-affinity to tier=2, tier=3", vm4Name)) + createVMWithAffinityAndAntiAffinityFunc(vm4Name, "1", []string{"2", "3"}) + + By("Waiting for all VMs to be created") + + vmMemberNames = []string{vm1Name, vm2Name, vm3Name, vm4Name} + for _, vmName := range vmMemberNames { + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + } + + By("Getting VM placement zones") + + vm1Zone := getVMZoneFunc(vm1Name) + vm2Zone := getVMZoneFunc(vm2Name) + vm3Zone := getVMZoneFunc(vm3Name) + vm4Zone := getVMZoneFunc(vm4Name) + + By("Verifying VM1 and VM4 are in the same zone") + Expect(vm1Zone).To(Equal(vm4Zone)) + + By("Verifying VM1, VM2, and VM3 are in different zones respectively") + Expect(vm1Zone).ToNot(Equal(vm2Zone)) + Expect(vm1Zone).ToNot(Equal(vm3Zone)) + Expect(vm2Zone).ToNot(Equal(vm3Zone)) + }) + + When("VMs have both AF/AAF and IaaS Policies applied", func() { + var ( + tagManager *tags.Manager + tagIDs []string + policyNames []string + policyNameToTagID map[string]string + ) + + BeforeEach(func() { + skipper.SkipUnlessSupervisorCapabilityEnabled(ctx, clusterProxy, consts.IaaSComputePoliciesCapabilityName) + + By("Creating a tag manager to verify actual tag assignment") + + restClient, err := vcenter.NewRestClient(ctx, vCenterClient, testbed.AdminUsername, testbed.AdminPassword) + Expect(err).NotTo(HaveOccurred(), "failed to create rest client") + + tagManager = tags.NewManager(restClient) + + By("Creating a new tag category") + + tagCategoryName := fmt.Sprintf("tag-category-%s", capiutil.RandomString(4)) + tagCategoryID, err := input.WCPClient.CreateTagCategory(tagCategoryName, "test-tag-category") + Expect(err).NotTo(HaveOccurred(), "failed to create tag category") + Expect(tagCategoryID).NotTo(BeEmpty(), "tag category ID should be returned") + + By("Creating 3 new tags under the tag category") + + tagIDs = make([]string, 3) + for i := range tagIDs { + tagIDs[i], err = input.WCPClient.CreateTag(fmt.Sprintf("tag-%s-%d", capiutil.RandomString(4), i+1), "test-tag", tagCategoryID) + Expect(err).NotTo(HaveOccurred(), "failed to create tag") + Expect(tagIDs[i]).NotTo(BeEmpty(), "tag ID should be returned") + } + + By("Getting all 3 Supervisor hosts in SSV testbed") + + hostIDs, err := input.WCPClient.ListHostIDs() + Expect(err).NotTo(HaveOccurred(), "failed to list host IDs") + Expect(len(hostIDs)).To(BeNumerically(">=", 3)) + // SSV testbed has 3 Supervisor hosts and 1 infra host. + supervisorHostIDs := hostIDs[:3] + + By("Assigning one tag to each Supervisor host") + + for i, hostID := range supervisorHostIDs { + Expect(input.WCPClient.AssignTagsToHost(tagIDs[i:i+1], hostID)).To(Succeed(), "failed to assign tag %s to host %s", tagIDs[i], hostID) + } + + By("Creating 3 compute policies with different tag IDs") + + tagIDToCPID := make(map[string]string, len(tagIDs)) + for i, tagID := range tagIDs { + // Cannot use tagID in policy name because it's too long. + policyName := fmt.Sprintf("%s-%d", capiutil.RandomString(4), i+1) + cpSpec := wcp.ComputePolicySpec{ + Name: policyName, + Description: policyName, + HostTagID: tagID, + VMTagID: tagID, + Capability: wcp.ComputePolicyCapabilityVMHostAffinity, + } + cpID, err := input.WCPClient.CreateComputePolicy(cpSpec) + Expect(err).NotTo(HaveOccurred(), "failed to create compute policy") + Expect(cpID).NotTo(BeEmpty(), "compute policy ID should be returned") + tagIDToCPID[tagID] = cpID + } + + policyNames = make([]string, len(tagIDs)) + policyNameToTagID = make(map[string]string, len(tagIDs)) + + By("Creating 3 mandatory infra policies matched by label (tier=1, 2, 3) from the above compute policies") + + for i, tagID := range tagIDs { + policyName := fmt.Sprintf("%s-%d", capiutil.RandomString(4), i+1) + infraPolicySpec := wcp.InfraPolicySpec{ + Name: policyName, + Description: policyName, + ComputePolicyID: tagIDToCPID[tagID], + EnforcementMode: wcp.InfraPolicyEnforcementModeMandatory, + MatchWorkloadLabel: map[string]string{"tier": fmt.Sprintf("%d", i+1)}, + } + Expect(input.WCPClient.CreateInfraPolicy(infraPolicySpec)).To(Succeed(), "failed to create mandatory infra policy matched by label") + + policyNames[i] = policyName + policyNameToTagID[policyName] = tagID + } + + By("Assigning all 3 mandatory infra policies to the namespace") + Expect(input.WCPClient.UpdateNamespaceWithInfraPolicies(tmpNamespaceName, policyNames...)).To(Succeed(), "failed to assign policies to namespace") + }) + + It("Should create VMs with expected zonal AF/AAF and IaaS Policies & Tags applied", func() { + By("Creating a VirtualMachineGroup with 4 VM-kind members") + + vmgParameters := manifestbuilders.VirtualMachineGroupYaml{ + Namespace: tmpNamespaceName, + Name: vmgRootName, + BootOrder: []manifestbuilders.BootOrder{ + { + Members: []vmopv1a5.GroupMember{ + { + Kind: vmKind, + Name: vm1Name, + }, + { + Kind: vmKind, + Name: vm2Name, + }, + { + Kind: vmKind, + Name: vm3Name, + }, + { + Kind: vmKind, + Name: vm4Name, + }, + }, + }, + }, + } + vmgRootYaml = manifestbuilders.GetVirtualMachineGroupWithBootOrderYaml(vmgParameters) + e2eframework.Logf("VirtualMachineGroup YAML:\n%s", string(vmgRootYaml)) + Expect(clusterProxy.CreateWithArgs(ctx, vmgRootYaml)).To(Succeed()) + + By(fmt.Sprintf("Creating VM1 (%s) with tier=1 label, affinity to tier=1, and anti-affinity to tier=2 and tier=3", vm1Name)) + createVMWithAffinityAndAntiAffinityFunc(vm1Name, "1", []string{"2", "3"}) + By(fmt.Sprintf("Creating VM2 (%s) with tier=2 label, affinity to tier=2, and anti-affinity to tier=1 and tier=3", vm2Name)) + createVMWithAffinityAndAntiAffinityFunc(vm2Name, "2", []string{"1", "3"}) + By(fmt.Sprintf("Creating VM3 (%s) with tier=3 label, affinity to tier=3, and anti-affinity to tier=1 and tier=2", vm3Name)) + createVMWithAffinityAndAntiAffinityFunc(vm3Name, "3", []string{"1", "2"}) + By(fmt.Sprintf("Creating VM4 (%s) with tier=1 label, affinity to tier=1, and anti-affinity to tier=2, tier=3", vm4Name)) + createVMWithAffinityAndAntiAffinityFunc(vm4Name, "1", []string{"2", "3"}) + + By("Waiting for all VMs to be created") + + vmMemberNames = []string{vm1Name, vm2Name, vm3Name, vm4Name} + for _, vmName := range vmMemberNames { + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, tmpNamespaceName, vmName) + } + + By("Getting VM placement zones") + + vm1Zone := getVMZoneFunc(vm1Name) + vm2Zone := getVMZoneFunc(vm2Name) + vm3Zone := getVMZoneFunc(vm3Name) + vm4Zone := getVMZoneFunc(vm4Name) + + By("Verifying VM1 and VM4 are in the same zone") + Expect(vm1Zone).To(Equal(vm4Zone)) + + By("Verifying VM1, VM2, and VM3 are in different zones respectively") + Expect(vm1Zone).ToNot(Equal(vm2Zone)) + Expect(vm1Zone).ToNot(Equal(vm3Zone)) + Expect(vm2Zone).ToNot(Equal(vm3Zone)) + + By("Verifying the VMs have the expected tags and policies assigned") + // VM1 with tier=1 label should have the 1st mandatory policy applied. + vmservice.VerifyVMTagsAndPolicyAssignment(ctx, config, svClusterClient, tagManager, tmpNamespaceName, vm1Name, policyNameToTagID, policyNames[:1]) + // VM2 with tier=2 label should have the 2nd mandatory policy applied. + vmservice.VerifyVMTagsAndPolicyAssignment(ctx, config, svClusterClient, tagManager, tmpNamespaceName, vm2Name, policyNameToTagID, policyNames[1:2]) + // VM3 with tier=3 label should have the 3rd mandatory policy applied. + vmservice.VerifyVMTagsAndPolicyAssignment(ctx, config, svClusterClient, tagManager, tmpNamespaceName, vm3Name, policyNameToTagID, policyNames[2:3]) + // VM4 with tier=1 label should have the 1st mandatory policy applied. + vmservice.VerifyVMTagsAndPolicyAssignment(ctx, config, svClusterClient, tagManager, tmpNamespaceName, vm4Name, policyNameToTagID, policyNames[:1]) + }) + }) + }) +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_group_publishrequest.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_group_publishrequest.go new file mode 100644 index 000000000..0b16c18f6 --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_group_publishrequest.go @@ -0,0 +1,304 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "k8s.io/apimachinery/pkg/util/sets" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + imgregv1a2 "github.com/vmware-tanzu/vm-operator/external/image-registry-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/testutils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +const ( + vmGroupPubSpecName = "vm-group-publish" +) + +type VMGroupPublishRequestSpecInput struct { + Config *e2eConfig.E2EConfig + ClusterProxy wcpframework.WCPClusterProxyInterface + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + WCPNamespaceName string +} + +func VMGroupPublishRequestSpec(ctx context.Context, inputGetter func() VMGroupPublishRequestSpecInput) { + var ( + vmGroupPubInput VMGroupPublishRequestSpecInput + vmSvcClusterProxy *common.VMServiceClusterProxy + vmSvcE2EConfig *e2eConfig.E2EConfig + vmSvcClusterResources *e2eConfig.Resources + vmSvcNamespace string + skipCleanup bool + + vm0Name string + vm1Name string + vmChildGroupName string + vmGroupName string + vmGroupPubName string + targetCLID string + + inventoryFolder *object.Folder + inventoryCLName string + + user *vcenter.User + nonAdminClient ctrlclient.Client + ) + + createAndAttachWritableLocalCL := func() { + targetCLID = vmservice.CreateLocalContentLibrary(vmGroupPubName+"-content-library", vmGroupPubInput.WCPClient) + Expect(vmGroupPubInput.WCPClient.AssociateImageRegistryContentLibrariesToNamespace( + vmSvcNamespace, + wcp.ContentLibrarySpec{ContentLibrary: targetCLID, Writable: true}), + ).To(Succeed(), "failed to attach content library '%q' to namespace '%q'", targetCLID, vmSvcNamespace) + } + + createInventoryContentLibrary := func() { + kubeConfig := vmSvcClusterProxy.GetKubeconfigPath() + + vCenterHostname := vcenter.GetVCPNIDFromKubeconfig(ctx, kubeConfig) + vimClient, err := vcenter.NewVimClient( + vCenterHostname, + testbed.AdminUsername, + testbed.AdminPassword, + ) + Expect(err).NotTo(HaveOccurred()) + + sshCommandRunner, _, _ := testutils.GetHelpersFromKubeconfig(ctx, kubeConfig) + user, nonAdminClient = setupNonAdminUserForTests(ctx, vimClient, sshCommandRunner, vmSvcClusterProxy.GetClient(), vmSvcClusterProxy) + + inventoryFolderName := fmt.Sprintf("%s-%s-%s", vmPubSpecName, "folder", capiutil.RandomString(4)) + inventoryCLName = fmt.Sprintf("%s-%s-%s", vmPubSpecName, "content-library", capiutil.RandomString(4)) + + By("Creating library folder") + + finder := find.NewFinder(vimClient, false) + _, inventoryFolder = createLibraryFolder(ctx, finder, inventoryFolderName) + + By("Creating an inventory content library", func() { + inventoryCL := createInventoryContentLibraryCR(ctx, nonAdminClient, imgregv1a2.ResourceNamingStrategyPreferItemSourceID, vmSvcNamespace, inventoryCLName, inventoryFolder.Reference().Value, true, true) + validateContentLibraryV2(ctx, nonAdminClient, inventoryCL, inventoryFolder, inventoryCLName, vmSvcNamespace, "") + }) + } + + createVMGroups := func() { + vmservice.CreateVMGroup(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineGroupYaml{ + Namespace: vmSvcNamespace, + Name: vmChildGroupName, + GroupName: vmGroupName, + Members: []vmopv1a5.GroupMember{{Kind: "VirtualMachine", Name: vm1Name}}, + }) + vmservice.CreateVMGroup(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineGroupYaml{ + Namespace: vmSvcNamespace, + Name: vmGroupName, + Members: []vmopv1a5.GroupMember{ + {Kind: "VirtualMachine", Name: vm0Name}, + {Kind: "VirtualMachineGroup", Name: vmChildGroupName}, + }, + }) + } + + createVMs := func() { + vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcClusterResources, vmSvcNamespace, vm1Name, vmChildGroupName, nil) + vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcClusterResources, vmSvcNamespace, vm0Name, vmGroupName, nil) + vmoperator.VerifyVirtualMachineGroupLinked(ctx, vmSvcE2EConfig, vmSvcClusterProxy.GetClient(), vmSvcNamespace, vmGroupName, sets.New([]vmopv1a5.GroupMember{ + {Kind: "VirtualMachine", Name: vm0Name}, + {Kind: "VirtualMachineGroup", Name: vmChildGroupName}, + }...)) + } + + generateRequiredResources := func() { + vmGroupPubName = vmGroupPubSpecName + "-" + capiutil.RandomString(4) + vmGroupName = "vm-group-" + capiutil.RandomString(4) + vmChildGroupName = vmGroupName + "-child-group" + vm0Name = vmGroupName + "-vm-0" + vm1Name = vmGroupName + "-vm-1" + + createAndAttachWritableLocalCL() + createVMGroups() + createVMs() + } + + skipChecks := func() { + skipper.SkipUnlessInfraIs(vmGroupPubInput.Config.InfraConfig.InfraName, consts.WCP) + skipper.SkipUnlessVMImageRegistryFSSEnabled(ctx, vmSvcClusterProxy.GetClient(), vmSvcE2EConfig) + skipper.SkipUnlessSupervisorCapabilityEnabled(ctx, vmSvcClusterProxy, consts.VMGroupsCapabilityName) + skipper.SkipUnlessSupervisorCapabilityEnabled(ctx, vmSvcClusterProxy, consts.InventoryContentLibraryCapabilityName) + + skipCleanup = false + } + + BeforeEach(func() { + vmGroupPubInput = inputGetter() + Expect(vmGroupPubInput.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", vmGroupPubSpecName) + Expect(vmGroupPubInput.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", vmGroupPubSpecName) + Expect(vmGroupPubInput.Config.InfraConfig.ManagementClusterConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig.ManagementClusterConfig can't be nil when calling %s spec", vmGroupPubSpecName) + Expect(vmGroupPubInput.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", vmGroupPubSpecName) + Expect(vmGroupPubInput.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", vmGroupPubSpecName) + Expect(os.MkdirAll(vmGroupPubInput.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", vmGroupPubSpecName) + + vmSvcClusterProxy = vmGroupPubInput.ClusterProxy.(*common.VMServiceClusterProxy) + vmSvcE2EConfig = vmGroupPubInput.Config + vmSvcClusterResources = vmSvcE2EConfig.InfraConfig.ManagementClusterConfig.Resources + vmSvcNamespace = vmGroupPubInput.WCPNamespaceName + skipCleanup = true + + skipChecks() + + vmGroupCancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces( + ctx, + []string{vmSvcE2EConfig.GetVariable("VMOPNamespace")}, + vmSvcClusterProxy.GetClientSet(), + filepath.Join(vmGroupPubInput.ArtifactFolder, vmGroupPubSpecName)) + DeferCleanup(vmGroupCancelPodWatches) + + generateRequiredResources() + }) + + deleteVMGroups := func() { + vmservice.DeleteVMGroup(ctx, vmSvcClusterProxy, vmSvcE2EConfig, manifestbuilders.VirtualMachineGroupYaml{ + Namespace: vmSvcNamespace, + Name: vmChildGroupName, + }) + vmservice.DeleteVMGroup(ctx, vmSvcClusterProxy, vmSvcE2EConfig, manifestbuilders.VirtualMachineGroupYaml{ + Namespace: vmSvcNamespace, + Name: vmGroupName, + }) + } + + detachAndDeleteWritableLocalCL := func() { + Expect(vmGroupPubInput.WCPClient.DisassociateImageRegistryContentLibrariesFromNamespace(vmSvcNamespace, targetCLID)).To( + Succeed(), "failed to detach content library '%s' from namespace '%s'", targetCLID, vmSvcNamespace) + Expect(vmGroupPubInput.WCPClient.DeleteLocalContentLibrary(targetCLID)).To( + Succeed(), "failed to delete the publish content library, CL ID: %s", targetCLID) + } + + deleteInventoryFolder := func() { + if inventoryFolder != nil { + vcenter.DeleteFolder(ctx, inventoryFolder) + inventoryFolder = nil + inventoryCLName = "" + } + } + + cleanupRequiredResources := func() { + deleteVMGroups() + detachAndDeleteWritableLocalCL() + deleteInventoryFolder() + } + + AfterEach(func() { + if skipCleanup { + return + } + + cleanupRequiredResources() + }) + + vmGroupPublish := func(groupPubName, target string, vms []string) { + vmservice.CreateVMGroupPub(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineGroupPublishRequestYaml{ + Namespace: vmSvcNamespace, + Name: groupPubName, + Source: vmGroupName, + Target: target, + VirtualMachines: vms, + }, "") + vmoperator.VerifyVirtualMachineGroupPublishRequestCompleted( + ctx, + vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + groupPubName) + + By("set spec.TTLSecondsAfterFinished should delete the vm group publish request after TTL") + vmservice.UpdateVMGroupPub(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineGroupPublishRequestYaml{ + Namespace: vmSvcNamespace, + Name: groupPubName, + Source: vmGroupName, + Target: target, + TTLSecondsAfterFinished: int64(1), + }) + vmoperator.VerifyVirtualMachineGroupPublishRequestDeleted( + ctx, + vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + groupPubName) + } + + It("should succeed when vm group publish request with created with proper inputs", Label("smoke"), func() { + By("create a vm group publish without a content library should fail when there is no default content library") + vmservice.CreateVMGroupPub(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineGroupPublishRequestYaml{ + Namespace: vmSvcNamespace, + Name: vmGroupPubName, + Source: vmGroupName, + Target: "", + }, "cannot find a default content library with the \"imageregistry.vmware.com/default\" label") + + targetCLName, err := vmservice.GetK8sContentLibraryNameByUUID( + ctx, + vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + targetCLID) + Expect(err).NotTo(HaveOccurred(), "failed to get content library name from it's ID") + Expect(targetCLName).NotTo(BeEmpty(), "new publishing content library ID is empty") + + By("create a vm group publish without a source vm group should fail when there is no vm group with the same name as the request") + vmservice.CreateVMGroupPub(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineGroupPublishRequestYaml{ + Namespace: vmSvcNamespace, + Name: vmGroupPubName, + Source: "", + Target: targetCLName, + }, fmt.Sprintf("VirtualMachineGroup.vmoperator.vmware.com %q not found", vmGroupPubName)) + + By("create a vm group publish with vms that are not in the vm group should fail") + vmservice.CreateVMGroupPub(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineGroupPublishRequestYaml{ + Namespace: vmSvcNamespace, + Name: vmGroupPubName, + Source: vmGroupName, + Target: targetCLName, + VirtualMachines: []string{ + fmt.Sprintf("vm-%s", capiutil.RandomString(4)), + fmt.Sprintf("vm-%s", capiutil.RandomString(4))}, + }, "virtual machines must be a direct or indirect member of the source group") + + By("create a vm group publish with source vm group and content library defined per created resource should succeed") + // all VMs are published when the spec.virtualMachines is omitted + vmGroupPublish(vmGroupPubName, targetCLName, []string{}) + + By("create a vm group publish with a subset of VMs from the source group should succeed") + vmGroupPublish(vmGroupPubName+"-subset", targetCLName, []string{vm0Name}) + + By("create a vm group publish with target inventory library should succeed") + createInventoryContentLibrary() + vmGroupPublish(vmGroupPubName+"-inventory", inventoryCLName, []string{vm0Name}) + + By("Deleting non admin user") + vcenter.DeleteUserOrFail(user) + }) +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_guestcustomization.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_guestcustomization.go new file mode 100644 index 000000000..ba80502a3 --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_guestcustomization.go @@ -0,0 +1,485 @@ +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type VMGOSCSpecInput struct { + Config *e2eConfig.E2EConfig + ClusterProxy wcpframework.WCPClusterProxyInterface + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + WCPNamespaceName string + WindowsServerVMName string +} + +const ( + specName = "vm-guest-customization" + inlineCloudInit = "inlineCloudInit" + cloudInitTransport = "CloudInit" + ovfEnvTransport = "OvfEnv" + vAppConfigTransport = "vAppConfig" + ubuntuMarketplaceImage = "ubuntu-20.04-vmservice-v1alpha1.20210528" + centosMarketplaceImage = "centos-stream-8-vmservice-v1alpha1.20210528" +) + +var ( + input VMGOSCSpecInput + config *e2eConfig.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterConfig *e2eConfig.ManagementClusterConfig + svClusterClient ctrlclient.Client + clusterResources *e2eConfig.Resources + vmYaml []byte + configMapYaml []byte + secretYaml []byte + vmName string + configMapName string + secretName string + skipCleanup bool + vmServiceBackupRestoreEnabled bool + wcpClient wcp.WorkloadManagementAPI + vmParameters manifestbuilders.VirtualMachineYaml + v1a2vmParameters manifestbuilders.VirtualMachineYaml + v1a5vmParameters manifestbuilders.VirtualMachineYaml + linuxImageDisplayName string +) + +func createAndVerifyConfigMap(ctx context.Context, transport string) []byte { + // Create and apply ConfigMap yaml. + configMap := manifestbuilders.ConfigMap{ + Namespace: input.WCPNamespaceName, + Name: configMapName, + } + if transport == cloudInitTransport { + configMapYaml = manifestbuilders.GetConfigMapYamlGOSC(configMap) + } else if transport == ovfEnvTransport { + configMapYaml = manifestbuilders.GetConfigMapYamlOvfEnv(configMap) + } else if transport == vAppConfigTransport { + configMapYaml = manifestbuilders.GetConfigMapYamlVAppConfig(configMap) + } + + Expect(clusterProxy.CreateWithArgs(ctx, configMapYaml)).To(Succeed(), "failed to create configmap", string(configMapYaml)) + vmservice.VerifyConfigMapCreation(ctx, config, svClusterClient, input.WCPNamespaceName, configMapName) + + return configMapYaml +} + +func CreateAndVerifySecret(ctx context.Context, transport string) []byte { + // Create and apply Secret yaml. + secret := manifestbuilders.Secret{ + Namespace: input.WCPNamespaceName, + Name: secretName, + } + if transport == cloudInitTransport { + secretYaml = manifestbuilders.GetSecretYamlCloudConfig(secret) + } else if transport == ovfEnvTransport { + secretYaml = manifestbuilders.GetSecretYamlOvfEnv(secret) + } else if transport == vAppConfigTransport { + secretYaml = manifestbuilders.GetSecretYamlVAppConfig(secret) + } else if transport == inlineCloudInit { + secretYaml = manifestbuilders.GetSecretYamlInlineCloudInitData(secret) + } + + Expect(clusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create secret", string(secretYaml)) + vmservice.VerifySecretCreation(ctx, config, svClusterClient, input.WCPNamespaceName, secretName) + + return secretYaml +} + +// v1a2 also supports v1a3. +func CreateAndVerifyVM(ctx context.Context, vmParameters manifestbuilders.VirtualMachineYaml, v1a2 ...bool) { + if len(v1a2) == 1 && v1a2[0] { + // Create v1alpha2 VM deployment yaml + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + } else { + vmYaml = manifestbuilders.GetVirtualMachineYaml(vmParameters) + } + + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine", string(vmYaml)) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) +} + +func CreateAndVerifyVMA5(ctx context.Context, vmParameters manifestbuilders.VirtualMachineYaml) { + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(vmParameters) + + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine", string(vmYaml)) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) +} + +func verifyLoginAndRunCmds(ctx context.Context, vmIp string, cmds []string, expectedOutput []string) { + switch config.InfraConfig.NetworkingTopology { + case consts.NSX: + vmservice.WaitForPodReady(ctx, config, svClusterClient, input.WCPNamespaceName, consts.JumpboxPodVMName) + vmservice.VerifyLoginAndRunCmdsInNSXSetup(ctx, config, clusterProxy, input.WCPNamespaceName, consts.JumpboxPodVMName, vmIp, cmds, expectedOutput) + case consts.VDS: + vmservice.VerifyLoginAndRunCmdsInVDSSetup(config, vmIp, cmds, expectedOutput) + } +} + +func VMGOSCSpec(ctx context.Context, inputGetter func() VMGOSCSpecInput) { + BeforeEach(func() { + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", specName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", specName) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + config = input.Config + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterConfig = config.InfraConfig.ManagementClusterConfig + clusterResources = svClusterConfig.Resources + wcpClient = input.WCPClient + svClusterClient = clusterProxy.GetClient() + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, clusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, specName)) + DeferCleanup(cancelPodWatches) + + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + + vmName = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + configMapName = fmt.Sprintf("%s-%s", "configmap", capiutil.RandomString(4)) + secretName = fmt.Sprintf("%s-%s", "secret", capiutil.RandomString(4)) + + // Use default network + vmParameters = manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + v1a2vmParameters = manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + v1a5vmParameters = manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + skipCleanup = false + vmServiceBackupRestoreEnabled = utils.IsFssEnabled(ctx, svClusterClient, config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMServiceBackupRestore")) + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, vmName, "vm") + } + + if skipCleanup { + return + } + + if CurrentSpecReport().State.String() != "skipped" { + // Delete the virtual machine + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).NotTo(HaveOccurred(), "failed to delete virtualmachine") + // Verify that virtual machine does not exist + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + } + }) + + Context("CloudInit", func() { + var bootstrapYAML []byte + + BeforeEach(func() { + imageName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "Failed to get the VM Image name in namespace %q", input.WCPNamespaceName) + + vmParameters.ImageName = imageName + vmParameters.Transport = cloudInitTransport + v1a2vmParameters.ImageName = imageName + v1a2vmParameters.Bootstrap = manifestbuilders.Bootstrap{ + CloudInit: &manifestbuilders.CloudInit{}, + } + }) + + When("ConfigMap is used to provide raw cloud-init config", func() { + BeforeEach(func() { + bootstrapYAML = createAndVerifyConfigMap(ctx, cloudInitTransport) + vmParameters.ConfigMapName = configMapName + CreateAndVerifyVM(ctx, vmParameters) + }) + + It("should successfully apply customization and be able to register VM from backup", Label("smoke"), func() { + vmIp := vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vmName) + cmds := []string{"cat /helloworld"} + expectedOutput := []string{"Hello World"} + verifyLoginAndRunCmds(ctx, vmIp, cmds, expectedOutput) + + if vmServiceBackupRestoreEnabled { + vmservice.VerifyRegisterVMOnlyClassicDisk(ctx, vmParameters.Name, vmParameters.Namespace, bootstrapYAML, clusterProxy, config, svClusterClient, wcpClient) + } + }) + }) + + When("Secret is used to provide raw cloud-init config", func() { + BeforeEach(func() { + skipper.SkipUnlessV1a2FSSEnabled(ctx, svClusterClient, config) + + bootstrapYAML = CreateAndVerifySecret(ctx, cloudInitTransport) + v1a2vmParameters.Bootstrap.CloudInit.RawCloudConfig = &manifestbuilders.KeySelector{ + Key: "user-data", + Name: secretName, + } + CreateAndVerifyVM(ctx, v1a2vmParameters, true) + }) + + It("should successfully apply customization and be able to register VM from backup", func() { + vmIp := vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vmName) + cmds := []string{"cat /helloworld"} + expectedOutput := []string{"Hello World"} + verifyLoginAndRunCmds(ctx, vmIp, cmds, expectedOutput) + + if vmServiceBackupRestoreEnabled { + vmservice.VerifyRegisterVMOnlyClassicDisk(ctx, v1a2vmParameters.Name, v1a2vmParameters.Namespace, bootstrapYAML, clusterProxy, config, svClusterClient, wcpClient) + } + }) + }) + + When("InlineCloudConfig is used to provide cloud-init config", func() { + BeforeEach(func() { + bootstrapYAML = CreateAndVerifySecret(ctx, inlineCloudInit) + + inlinedCloudConfig := fmt.Sprintf(` + defaultUserEnabled: true + ssh_pwauth: true + users: + - name: vmware + lock_passwd: false + passwd: + name: %s + key: vmsvc-pwd + runcmd: + - [ "ls", "-a", "-l", "/" ] + - - echo + - "hello, world." + write_files: + - path: /etc/my-plaintext + permissions: '0644' + owner: root:root + content: + name: %s + key: hello`, secretName, secretName) + + v1a2vmParameters.Bootstrap.CloudInit.CloudConfig = &inlinedCloudConfig + }) + + It("should successfully apply customization and be able to register VM from backup", func() { + CreateAndVerifyVM(ctx, v1a2vmParameters, true) + vmIp := vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vmName) + cmds := []string{"cat /etc/my-plaintext"} + expectedOutput := []string{"Hello World"} + verifyLoginAndRunCmds(ctx, vmIp, cmds, expectedOutput) + + if vmServiceBackupRestoreEnabled { + vmservice.VerifyRegisterVMOnlyClassicDisk(ctx, v1a2vmParameters.Name, v1a2vmParameters.Namespace, bootstrapYAML, clusterProxy, config, svClusterClient, wcpClient) + } + }) + }) + }) + + Context("LinuxPrep", func() { + BeforeEach(func() { + skipper.SkipUnlessV1a2FSSEnabled(ctx, svClusterClient, config) + imageName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "Failed to get the VM Image name in namespace %q", input.WCPNamespaceName) + + v1a2vmParameters.ImageName = imageName + v1a2vmParameters.Bootstrap = manifestbuilders.Bootstrap{ + LinuxPrep: &manifestbuilders.LinuxPrep{ + HardwareClockIsUTC: true, + TimeZone: "US/Pacific", + }, + } + }) + + It("should successfully deploy VM and be able to register VM from backup", func() { + CreateAndVerifyVM(ctx, v1a2vmParameters, true) + + if vmServiceBackupRestoreEnabled { + vmservice.VerifyRegisterVMOnlyClassicDisk(ctx, v1a2vmParameters.Name, v1a2vmParameters.Namespace, nil, clusterProxy, config, svClusterClient, wcpClient) + } + }) + + Context("LinuxPrep with CustomizeAtNextPowerOn latch", func() { + BeforeEach(func() { + t := true + imageName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "Failed to get the VM Image name in namespace %q", input.WCPNamespaceName) + + v1a5vmParameters.ImageName = imageName + v1a5vmParameters.Bootstrap = manifestbuilders.Bootstrap{ + LinuxPrep: &manifestbuilders.LinuxPrep{ + HardwareClockIsUTC: true, + TimeZone: "US/Pacific", + CustomizeAtNextPowerOn: &t, + }, + } + }) + + It("should successfully deploy VM and set latch to false", func() { + CreateAndVerifyVMA5(ctx, v1a5vmParameters) + vmoperator.WaitForLinuxPrepCustomizeNextPowerOnFalse(ctx, config, svClusterClient, input.WCPNamespaceName, v1a5vmParameters.Name) + }) + }) + }) + + // TODO: Remove this as OvfEnv is deprecated. + Context("OvfEnv", func() { + // VMs with OvfEnv transport do not support restore due to the defer-cloud-init configuration. + // Once a VM is booted, the defer-cloud-init is not re-applied hence causing the race between + // vmtools and cloud-init to configure networking during the second boot from the restore. + // Therefore, VerifyRegisterVM is not called for these images using OvfEnv transport. + BeforeEach(func() { + vmParameters.Transport = ovfEnvTransport + + if os.Getenv("RUN_CANONICAL_TEST") == "true" { + Skip("These tests will be skipped for Canonical OVA testing.") + } + }) + + It("should successfully apply customization from a ConfigMap and get a valid IP assigned to ubuntu-20.04", func() { + // Create and apply ConfigMap yaml. + createAndVerifyConfigMap(ctx, ovfEnvTransport) + // This ubuntu 20.04 image is supported in marketplace + vmImageName := ubuntuMarketplaceImage + imageName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, vmImageName) + Expect(err).NotTo(HaveOccurred()) + + vmParameters.ImageName = imageName + vmParameters.ConfigMapName = configMapName + CreateAndVerifyVM(ctx, vmParameters) + vmIp := vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vmName) + cmds := []string{"cat /helloworld"} + expectedOutput := []string{"Hello World"} + verifyLoginAndRunCmds(ctx, vmIp, cmds, expectedOutput) + }) + + XIt("should successfully apply customization from a Secret and get a valid IP assigned to centos-stream-8", func() { + // Create and apply Secret yaml. + CreateAndVerifySecret(ctx, ovfEnvTransport) + // This centos stream 8 image is supported in marketplace + vmImageName := centosMarketplaceImage + imageName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, vmImageName) + Expect(err).NotTo(HaveOccurred()) + + vmParameters.ImageName = imageName + vmParameters.SecretName = secretName + CreateAndVerifyVM(ctx, vmParameters) + vmIp := vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vmName) + cmds := []string{"cat /helloworld"} + expectedOutput := []string{"Hello World"} + verifyLoginAndRunCmds(ctx, vmIp, cmds, expectedOutput) + }) + }) + + Context("vAppConfig", func() { + BeforeEach(func() { + skipper.SkipUnlessV1a2FSSEnabled(ctx, svClusterClient, config) + imageName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "Failed to get the VM Image name in namespace %q", input.WCPNamespaceName) + + v1a2vmParameters.ImageName = imageName + v1a2vmParameters.Bootstrap = manifestbuilders.Bootstrap{ + // LinuxPrep is needed here for VM to get a valid IP address. + LinuxPrep: &manifestbuilders.LinuxPrep{}, + VAppConfig: &manifestbuilders.VAppConfig{ + Properties: &[]manifestbuilders.KeyValueOrSecretKeySelectorPair{ + { + Key: "prop-1", + Value: manifestbuilders.ValueOrSecretKeySelector{ + Value: "my-val-1", + }, + }, + }, + }, + } + }) + + It("should successfully apply vAppConfig properties to VM and be able to register VM from backup", func() { + CreateAndVerifyVM(ctx, v1a2vmParameters, true) + + // TODO: Verify the vAppConfig properties are actually applied to the VM. + + if vmServiceBackupRestoreEnabled { + vmservice.VerifyRegisterVMOnlyClassicDisk(ctx, v1a2vmParameters.Name, v1a2vmParameters.Namespace, nil, clusterProxy, config, svClusterClient, wcpClient) + } + }) + }) + + Context("Sysprep", func() { + BeforeEach(func() { + // The Windows server VM was deployed during the suite setup and will be deleted in the suite teardown. + skipCleanup = true + + // Skip if WCP_Windows_Sysprep FSS not enabled + skipper.SkipUnlessWindowsFSSEnabled(ctx, svClusterClient, config) + }) + + When("raw Sysprep data is used", func() { + // TODO: VMSVC-2789: Re-enable this once we've fixed PR 3531430 + XIt("should successfully deploy VM and be able to register VM from backup", func() { + // The VM is already created during the suite setup. + vmName = input.WindowsServerVMName + Expect(vmName).ToNot(BeEmpty()) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + + if vmServiceBackupRestoreEnabled { + vmservice.VerifyRegisterVMOnlyClassicDisk(ctx, vmName, input.WCPNamespaceName, nil, clusterProxy, config, svClusterClient, wcpClient) + } + }) + }) + + When("Inline Sysprep is used", func() { + BeforeEach(func() { + skipper.SkipUnlessV1a2FSSEnabled(ctx, svClusterClient, config) + }) + // TODO: VMSVC-2789: Re-enable this once we've fixed PR 3531430 + XIt("should successfully deploy VM and be able to register VM from backup", func() { + // The VM is already created during the suite setup. + vmName = input.WindowsServerVMName + "-a2" + Expect(vmName).ToNot(BeEmpty()) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + + if vmServiceBackupRestoreEnabled { + vmservice.VerifyRegisterVMOnlyClassicDisk(ctx, vmName, input.WCPNamespaceName, nil, clusterProxy, config, svClusterClient, wcpClient) + } + }) + }) + }) +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_hardware.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_hardware.go new file mode 100644 index 000000000..df95d78ca --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_hardware.go @@ -0,0 +1,2401 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "slices" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vmware-tanzu/vm-operator/pkg/util/ptr" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + govc "github.com/vmware/govmomi/vapi/vcenter" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + vimtypes "github.com/vmware/govmomi/vim25/types" + "gopkg.in/yaml.v3" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + mopv1a2 "github.com/vmware-tanzu/vm-operator/external/mobility-operator/api/v1alpha2" + topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/csi" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +// VMHardwareSpecInput is the input for the VM Hardware test spec. +type VMHardwareSpecInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *e2eConfig.E2EConfig + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + WCPNamespaceName string + SkipCleanup bool +} + +func createPvcsFromSpec( + input VMHardwareSpecInput, + prefix string, + spec manifestbuilders.PVC, + count int, +) []manifestbuilders.PVC { + pvcs := []manifestbuilders.PVC{} + + for i := range count { + volumePrefix := fmt.Sprintf("%s-pvc-%s-%d", + strings.ToLower(prefix), capiutil.RandomString(4), i) + pvcs = append(pvcs, manifestbuilders.PVC{ + Namespace: input.WCPNamespaceName, + VolumeName: fmt.Sprintf("%s-volume", volumePrefix), + ClaimName: fmt.Sprintf("%s-claim", volumePrefix), + StorageClassName: spec.StorageClassName, + RequestSize: "1Mi", + ControllerType: spec.ControllerType, + ControllerBusNumber: spec.ControllerBusNumber, + SharingMode: spec.SharingMode, + AccessModes: spec.AccessModes, + VolumeMode: spec.VolumeMode, + ApplicationType: spec.ApplicationType, + UnitNumber: spec.UnitNumber, + }) + } + + return pvcs +} + +type testSpec struct { + pvcs []manifestbuilders.PVC + hardware vmopv1a5.VirtualMachineHardwareSpec +} + +func waitForVMAndBatchAttach( + ctx context.Context, + config *e2eConfig.E2EConfig, + svClusterClient ctrlclient.Client, + vmSvcNamespace, + vmPrefix string, + expectedVolumes []string, +) *vmopv1a5.VirtualMachine { + By("Waiting for the Virtual Machine to be created") + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, vmSvcNamespace, vmPrefix) + + By("Waiting for the batch attachment volumes to be attached") + csi.WaitForBatchAttachVolumesToBeAttached(ctx, config, svClusterClient, vmSvcNamespace, vmPrefix, expectedVolumes) + + By("Waiting for Virtual Machine to Power On") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, vmPrefix, "PoweredOn") + + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmPrefix) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + Expect(vm).ToNot(BeNil(), "VirtualMachine is nil") + + return vm +} + +// getBackfilledVolumes returns a list of backfilled volume names when the "all disks are PVCs" +// capability is enabled. If the capability is not enabled, it returns an empty list. +func getBackfilledVolumes( + ctx context.Context, + config *e2eConfig.E2EConfig, + svClusterClient ctrlclient.Client, + vmSvcNamespace, + vmName string, + allDisksArePVCapabilityEnabled bool, +) []string { + if !allDisksArePVCapabilityEnabled { + return []string{} + } + + // Wait for both conditions + conditions := []metav1.Condition{ + { + Type: "VirtualMachineUnmanagedVolumesBackfilled", + Status: metav1.ConditionTrue, + }, + { + Type: "VirtualMachineUnmanagedVolumesRegistered", + Status: metav1.ConditionTrue, + }, + } + for _, condition := range conditions { + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, vmSvcNamespace, vmName, condition) + } + + // Get the VM and extract backfilled volumes + var backfilledVolumes []string + + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + g.Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + + backfilledVolumes = []string{} + + for _, vol := range vm.Spec.Volumes { + // When VirtualMachineUnmanagedVolumesRegistered is true, all volumes should have PVC references + g.Expect(vol.PersistentVolumeClaim).ToNot(BeNil(), + "Volume %s should have PersistentVolumeClaim when VirtualMachineUnmanagedVolumesRegistered is true", vol.Name) + + // Backfilled volumes have removable: false + // Removable defaults to true, so we only include volumes where it's explicitly false + if vol.Removable != nil && !*vol.Removable { + backfilledVolumes = append(backfilledVolumes, vol.Name) + } + } + + // Verify all backfilled volumes are in vm.Status.Volumes with type Managed + for _, backfilledVolName := range backfilledVolumes { + found := false + + for _, volStatus := range vm.Status.Volumes { + if volStatus.Name == backfilledVolName { + g.Expect(volStatus.Type).To(Equal(vmopv1a5.VolumeTypeManaged), + "Expected backfilled volume %s to have type Managed", backfilledVolName) + + found = true + + break + } + } + + g.Expect(found).To(BeTrue(), "Expected backfilled volume %s to be found in vm.Status.Volumes", backfilledVolName) + } + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...). + Should(Succeed(), "Timed out waiting to get VirtualMachine %s and extract backfilled volumes", vmName) + + return backfilledVolumes +} + +func verifyCreatedControllersCount( + ctx context.Context, + config *e2eConfig.E2EConfig, + svClusterClient ctrlclient.Client, + vmSvcNamespace, + vmName string, + expectedControllersCount map[vmopv1a5.VirtualControllerType]int, +) { + Eventually(func(g Gomega) bool { + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + createdControllersCount := make(map[vmopv1a5.VirtualControllerType]int) + for _, controller := range vm.Status.Hardware.Controllers { + createdControllersCount[controller.Type]++ + } + + pass := true + + for controllerType, expectedCount := range expectedControllersCount { + actualCount, ok := createdControllersCount[controllerType] + if !ok { + actualCount = 0 + } + + if actualCount != expectedCount { + pass = false + + e2eframework.Logf("unexpected number of %s controllers: expected %d, got %d", + controllerType, expectedCount, actualCount) + } + } + + return pass + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...). + Should(BeTrue(), "Timed out waiting for VirtualMachines %s to be updated", vmName) +} + +func VMHardwareSpec(ctx context.Context, inputGetter func() VMHardwareSpecInput) { + const ( + specName = "vm-hardware" + eztStorageProfileName = "vmservice-ezt-storage-profile" + isoImageDisplayName = "ubuntu-24.04-live-server-amd64" + ) + + var ( + input VMHardwareSpecInput + vmSvcNamespace string + config *e2eConfig.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterClient ctrlclient.Client + clusterResources *e2eConfig.Resources + vCenterClient *vim25.Client + vmYamls [][]byte + vmName string + isoSupportFSSEnabled bool + allDisksArePVCapabilityEnabled bool + linuxImageDisplayName string + ) + + Context("VMs with attached hardware", Ordered, func() { + BeforeAll(func() { + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), + "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), + "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", specName) + Expect(input.Config.InfraConfig.ManagementClusterConfig).ToNot(BeNil(), + "Invalid argument. input.E2EConfig.InfraConfig.ManagementClusterConfig can't be nil when calling %s spec", + specName) + clusterResources = input.Config.InfraConfig.ManagementClusterConfig.Resources + + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), + "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.ClusterProxy).ToNot(BeNil(), + "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", specName) + + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), + "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", specName) + vmSvcNamespace = input.WCPNamespaceName + + config = input.Config + + wcpClient = input.WCPClient + kubeconfigPath := input.ClusterProxy.GetKubeconfigPath() + + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterClient = clusterProxy.GetClient() + svClientSet := clusterProxy.GetClientSet() + + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + + isoSupportFSSEnabled = utils.IsFssEnabled(ctx, + svClusterClient, + config.GetVariable("VMOPNamespace"), + config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), + config.GetVariable("EnvFSSIsoSupport"), + ) + + asyncSupervisorFSSEnabled, err := utils.CheckSupervisorCapabilitiesCRDSupport(ctx, svClusterClient) + Expect(err).NotTo(HaveOccurred()) + + allDisksArePVCapabilityEnabled = utils.IsSupervisorCapabilityEnabled( + ctx, + clusterProxy.GetClientSet(), + clusterProxy.GetDynamicClient(), + consts.AllDisksArePVCapabilityName, + asyncSupervisorFSSEnabled) + + vCenterClient = vcenter.NewVimClientFromKubeconfig(ctx, kubeconfigPath) + defer vcenter.LogoutVimClient(vCenterClient) + + By("Creating EZT storage policy for multi-writer PVCs") + // Get the base WCP storage class to extract its policy ID. + storageClass, err := svClientSet.StorageV1().StorageClasses(). + Get(ctx, clusterResources.StorageClassName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "Failed to get storage class %s", clusterResources.StorageClassName) + + basePolicyID := storageClass.Parameters["storagePolicyID"] + Expect(basePolicyID).NotTo(BeEmpty(), "Storage class %s does not have storagePolicyID parameter", + clusterResources.StorageClassName) + + // Create or get the EZT storage policy. + eztStoragePolicyID, err := vcenter.GetOrCreateEZTStoragePolicy(ctx, vCenterClient, eztStorageProfileName, + basePolicyID) + Expect(err).ShouldNot(HaveOccurred(), "Failed to create EZT storage policy") + Expect(eztStoragePolicyID).ShouldNot(BeEmpty(), "EZT storage policy ID is empty") + e2eframework.Logf("EZT storage policy created/found with ID: %s", eztStoragePolicyID) + + By("Configure namespace with EZT storage policy") + + details, err := wcpClient.GetNamespace(vmSvcNamespace) + Expect(err).NotTo(HaveOccurred(), "Failed to get namespace %s", vmSvcNamespace) + + // Check if EZT policy is already in the namespace + policyExists := slices.ContainsFunc(details.VMStorageSpec, func(spec wcp.StorageSpec) bool { + return spec.Policy == eztStoragePolicyID + }) + + if !policyExists { + details.VMStorageSpec = append(details.VMStorageSpec, wcp.StorageSpec{Policy: eztStoragePolicyID}) + Expect(wcpClient.SetNamespaceStorageSpecs(vmSvcNamespace, details.VMStorageSpec)). + Should(Succeed(), "Failed to set storage specs for namespace %s", vmSvcNamespace) + wcp.WaitForNamespaceReady(wcpClient, vmSvcNamespace) + e2eframework.Logf("EZT storage policy added to namespace %s", vmSvcNamespace) + } else { + e2eframework.Logf("EZT storage policy already exists in namespace %s", vmSvcNamespace) + } + + By("Ensure EZT storage class is available in namespace") + + podVMOnStretchedSupervisorEnabled := utils.IsFssEnabled(ctx, svClusterClient, + config.GetVariable("VMOPNamespace"), + config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), + config.GetVariable("EnvFSSPodVMOnStretchedSupervisor")) + utils.EnsureStorageClassInNamespace(ctx, svClusterClient, vmSvcNamespace, + eztStorageProfileName, podVMOnStretchedSupervisorEnabled, *config) + e2eframework.Logf( + "EZT storage class %s is available in namespace %s", + eztStorageProfileName, vmSvcNamespace, + ) + + skipper.SkipUnlessInfraIs(config.InfraConfig.InfraName, consts.WCP) + skipper.SkipUnlessSupervisorCapabilityEnabled(ctx, clusterProxy, consts.SharedDisksCapabilityName) + }) + + BeforeEach(func() { + vmYamls = [][]byte{} + vmName = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + By("Logging Virtual Machines and Batch Attachments after failure") + + for _, vmYaml := range vmYamls { + var virtualMachine manifestbuilders.VirtualMachineYaml + if err := yaml.Unmarshal(vmYaml, &virtualMachine); err != nil { + e2eframework.Logf("Failed to parse VM yaml: %s", err) + continue + } + + vmName := virtualMachine.Name + + stdout, err := framework.KubectlGet(ctx, + clusterProxy.GetKubeconfigPath(), + "virtualmachine", vmName, + "-n", virtualMachine.Namespace, + "-oyaml") + if err != nil { + e2eframework.Logf("Failed to get VirtualMachine %s: %s", vmName, err) + } else { + e2eframework.Logf("VirtualMachine %s:\n%s", vmName, stdout) + } + + stdout, err = framework.KubectlGet(ctx, + clusterProxy.GetKubeconfigPath(), + "batchattach", vmName, + "-n", virtualMachine.Namespace, + "-oyaml") + if err != nil { + e2eframework.Logf("Failed to get batch attachment: %s", err) + } else { + e2eframework.Logf("Batch Attachment %s:\n%s", vmName, stdout) + } + } + + stdout, err := framework.KubectlGet(ctx, clusterProxy.GetKubeconfigPath(), + "pvc", + "-n", vmSvcNamespace, + "-oyaml") + if err != nil { + e2eframework.Logf("Failed to get PVCs: %s", err) + } else { + e2eframework.Logf("PVCs:\n%s", stdout) + } + } + + for _, vmYaml := range vmYamls { + _ = clusterProxy.DeleteWithArgs(ctx, vmYaml) + } + }) + + DescribeTable("Virtual Machine with PVCs Power On", + func(specGetter func() testSpec) { + spec := specGetter() + + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PVCs: spec.pvcs, + Hardware: &spec.hardware, + }) + vmYamls = append(vmYamls, vmYaml) + + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine") + + By("Waiting for the VirtualMachine to be created") + + volumeNames := make([]string, len(spec.pvcs)) + for i, pvc := range spec.pvcs { + volumeNames[i] = pvc.VolumeName + } + // Get backfilled volumes and add them to volumeNames + backfilledVolumes := getBackfilledVolumes(ctx, config, svClusterClient, vmSvcNamespace, vmName, allDisksArePVCapabilityEnabled) + volumeNames = append(volumeNames, backfilledVolumes...) + vm := waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, volumeNames) + + By("Waiting on virtual machine conditions to become true") + + conditions := []metav1.Condition{ + { + Type: "VirtualMachineHardwareControllersVerified", + Status: metav1.ConditionTrue, + }, + { + Type: "VirtualMachineHardwareDeviceConfigVerified", + Status: metav1.ConditionTrue, + }, + { + Type: "VirtualMachineHardwareVolumesVerified", + Status: metav1.ConditionTrue, + }, + } + + for _, condition := range conditions { + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, vmSvcNamespace, vmName, condition) + } + + statusControllerDevices := make(map[vmopv1a5.VirtualControllerType]map[int32]int) + for _, controller := range vm.Status.Hardware.Controllers { + if _, ok := statusControllerDevices[controller.Type]; !ok { + statusControllerDevices[controller.Type] = make(map[int32]int) + } + + statusControllerDevices[controller.Type][controller.BusNumber] = len(controller.Devices) + } + + By("verifying 2 IDE controllers") + // The mutation webhook always creates 2 IDE controllers. + Expect(statusControllerDevices[vmopv1a5.VirtualControllerTypeIDE]).To(HaveLen(2), "Expected 2 IDE controllers") + + By("verifying explicit controllers") + + expectedScsiControllerDevicesCount := 0 + expectedDeviceCount := make(map[vmopv1a5.VirtualControllerType]map[int32]int) + + for _, pvc := range spec.pvcs { + if pvc.ControllerType == nil || pvc.ControllerBusNumber == nil { + // Unassigned PVCs are attached to SCSI controllers. + expectedScsiControllerDevicesCount++ + continue + } + + if *pvc.ControllerType == vmopv1a5.VirtualControllerTypeSCSI { + expectedScsiControllerDevicesCount++ + } + + if _, ok := expectedDeviceCount[*pvc.ControllerType]; !ok { + expectedDeviceCount[*pvc.ControllerType] = make(map[int32]int) + } + + expectedDeviceCount[*pvc.ControllerType][*pvc.ControllerBusNumber]++ + } + + for controllerType, busNumberToDeviceCount := range expectedDeviceCount { + for busNumber, deviceCount := range busNumberToDeviceCount { + devices, ok := statusControllerDevices[controllerType][busNumber] + Expect(ok).To(BeTrue(), "Expected controller %s:%d to be found in status", + controllerType, busNumber) + Expect(devices).To(BeNumerically(">=", deviceCount), + "Expected at least %d devices for controller %s:%d in status, got %d", + deviceCount, controllerType, busNumber, devices) + } + } + + totalStatusScsiControllerDevicesCount := 0 + + for _, controller := range vm.Status.Hardware.Controllers { + if controller.Type == vmopv1a5.VirtualControllerTypeSCSI { + totalStatusScsiControllerDevicesCount += len(controller.Devices) + } + } + + By("verifying implicit PVCs are attached to the SCSI controller") + // We check for greater than or equal because the image may have additional devices. + Expect(totalStatusScsiControllerDevicesCount).To(BeNumerically(">=", expectedScsiControllerDevicesCount), + "Expected at least %d devices for SCSI controller in status, got %d", + expectedScsiControllerDevicesCount, totalStatusScsiControllerDevicesCount) + + By("verifying the volume fields") + + expectedVolumes := make(map[string]manifestbuilders.PVC) + for _, pvc := range spec.pvcs { + expectedVolumes[pvc.VolumeName] = pvc + } + + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + g.Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + + bootDiskFound := false + + for _, pvc := range vm.Status.Volumes { + g.Expect(pvc.DiskUUID).To(Not(BeEmpty()), "Expected volume %s to have a disk UUID", pvc.Name) + g.Expect(pvc.Name).To(Not(BeEmpty()), "Expected volume name to be not empty") + g.Expect(pvc.Attached).To(BeTrue(), "Expected volume %s to be attached", pvc.Name) + g.Expect(pvc.ControllerType).To(Not(BeEmpty()), "Expected volume %s to have a controller type", pvc.Name) + g.Expect(pvc.ControllerBusNumber).To(Not(BeNil()), "Expected volume %s to have a controller bus number", pvc.Name) + g.Expect(pvc.UnitNumber).To(Not(BeNil()), "Expected volume %s to have a unit number", pvc.Name) + + g.Expect(pvc.Limit).To(Not(BeNil()), "Expected volume %s to have a limit", pvc.Name) + _, ok := pvc.Limit.AsInt64() + g.Expect(ok).To(BeTrue(), "Expected volume %s to have a limit", pvc.Name) + + g.Expect(pvc.Requested).To(Not(BeNil()), "Expected volume %s to have a requested", pvc.Name) + _, ok = pvc.Requested.AsInt64() + g.Expect(ok).To(BeTrue(), "Expected volume %s to have a requested", pvc.Name) + + g.Expect(pvc.Used).To(Not(BeNil()), "Expected volume %s to have a used", pvc.Name) + _, ok = pvc.Used.AsInt64() + g.Expect(ok).To(BeTrue(), "Expected volume %s to have a used", pvc.Name) + + expectedPvc, ok := expectedVolumes[pvc.Name] + + if *pvc.ControllerBusNumber == 0 && *pvc.UnitNumber == 0 { + bootDiskFound = true + + if allDisksArePVCapabilityEnabled { + g.Expect(pvc.Type).To(Equal(vmopv1a5.VolumeTypeManaged), + "Expected volume %s to have a type of %s", pvc.Name, vmopv1a5.VolumeTypeManaged) + } else { + // boot disk is not expect going to be in the manifest. + g.Expect(pvc.Type).To(Equal(vmopv1a5.VolumeTypeClassic), + "Expected volume %s to have a type of %s", pvc.Name, vmopv1a5.VolumeTypeClassic) + } + } else { + g.Expect(ok).To(BeTrue(), "Expected volume %s to be found in expected volumes", pvc.Name) + g.Expect(pvc.Type).To(Equal(vmopv1a5.VolumeTypeManaged), + "Expected volume %s to have a type of %s", pvc.Name, vmopv1a5.VolumeTypeManaged) + + if expectedPvc.ControllerType != nil { + g.Expect(pvc.ControllerType).To(Equal(*expectedPvc.ControllerType), + "Expected volume %s to have a controller type of %s", pvc.Name, *expectedPvc.ControllerType) + } + + if expectedPvc.ControllerBusNumber != nil { + g.Expect(pvc.ControllerBusNumber).To(Equal(expectedPvc.ControllerBusNumber), + "Expected volume %s to have a controller bus number of %d", pvc.Name, *expectedPvc.ControllerBusNumber) + } + + if expectedPvc.UnitNumber != nil { + g.Expect(pvc.UnitNumber).To(Equal(expectedPvc.UnitNumber), + "Expected volume %s to have a unit number of %d", pvc.Name, *expectedPvc.UnitNumber) + } + } + } + + g.Expect(bootDiskFound).To(BeTrue(), "Expected boot disk to be found") + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...). + Should(Succeed(), "Timed out waiting for the volume fields to be updated") + }, + Entry("create a virtual machine with a single PVC", Label("smoke"), func() testSpec { + return testSpec{ + pvcs: createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + }, 1), + } + }), + Entry("create a virtual machine with a combination of placements, controller types, and sharing modes", func() testSpec { + pvcs := []manifestbuilders.PVC{} + + // We are adding 5 PVCs without explicit assignment. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + }, 5)...) + + // We are adding a PVC with a persistent disk mode. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + DiskMode: ptr.To(string(vmopv1a5.VolumeDiskModePersistent)), + }, 1)...) + + // We are adding a PVC with a independent persistent disk mode. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + DiskMode: ptr.To(string(vmopv1a5.VolumeDiskModeIndependentPersistent)), + }, 1)...) + + // We are adding a PVC with a independent non persistent disk mode. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + DiskMode: ptr.To(string(vmopv1a5.VolumeDiskModeIndependentNonPersistent)), + }, 1)...) + + // We are adding a PVC with a non persistent disk mode. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + DiskMode: ptr.To(string(vmopv1a5.VolumeDiskModeNonPersistent)), + }, 1)...) + + // We are adding a PVC explicitly assigned to a SCSI:1:1. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + ControllerType: ptr.To(vmopv1a5.VirtualControllerTypeSCSI), + ControllerBusNumber: ptr.To(int32(1)), + UnitNumber: ptr.To(int32(1)), + }, 1)...) + + // We are adding a PVC explicitly assigned to a SCSI:2:2 with a multi-writer sharing mode. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: eztStorageProfileName, + ControllerType: ptr.To(vmopv1a5.VirtualControllerTypeSCSI), + ControllerBusNumber: ptr.To(int32(2)), + UnitNumber: ptr.To(int32(2)), + SharingMode: ptr.To(string(vmopv1a5.VolumeSharingModeMultiWriter)), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: ptr.To(corev1.PersistentVolumeBlock), + }, 1)...) + + // We are adding a PVC explicitly assigned to SCSI:1 and without a unit number. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + ControllerType: ptr.To(vmopv1a5.VirtualControllerTypeSCSI), + ControllerBusNumber: ptr.To(int32(1)), + }, 1)...) + + // We are adding a PVC explicitly assigned to SCSI:3:0 with a SCSI controller + // that has physical sharing mode defined in the VM below. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: eztStorageProfileName, + ControllerType: ptr.To(vmopv1a5.VirtualControllerTypeSCSI), + ControllerBusNumber: ptr.To(int32(3)), + UnitNumber: ptr.To(int32(0)), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: ptr.To(corev1.PersistentVolumeBlock), + }, 1)...) + + // We are adding a PVC with only application type set to OracleRAC. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: eztStorageProfileName, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: ptr.To(corev1.PersistentVolumeBlock), + ApplicationType: vmopv1a5.VolumeApplicationTypeOracleRAC, + }, 1)...) + + // We are adding a PVC with only application type set to MicrosoftWSFC. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: eztStorageProfileName, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: ptr.To(corev1.PersistentVolumeBlock), + ApplicationType: vmopv1a5.VolumeApplicationTypeMicrosoftWSFC, + ControllerType: ptr.To(vmopv1a5.VirtualControllerTypeSCSI), + ControllerBusNumber: ptr.To(int32(3)), + }, 1)...) + + // TODO(Faisal A): Enable this after fixing the issues with the NVME controller status. + // We are adding a PVC explicitly assigned to NVME:1 and without a unit number. + // pvcs = append(pvcs, createPvcsFromSpec(input, vmPrefix, manifestbuilders.PVC{ + // StorageClassName: clusterResources.StorageClassName, + // ControllerType: ptr.To(vmopv1a5.VirtualControllerTypeNVME), + // ControllerBusNumber: ptr.To(int32(1)), + // }, 1)...) + + // // We are adding a PVC explicitly assigned to SATA:1 and without a unit number. + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + ControllerType: ptr.To(vmopv1a5.VirtualControllerTypeSATA), + ControllerBusNumber: ptr.To(int32(1)), + }, 1)...) + + return testSpec{ + pvcs: pvcs, + hardware: vmopv1a5.VirtualMachineHardwareSpec{ + SCSIControllers: []vmopv1a5.SCSIControllerSpec{ + { + BusNumber: 1, + Type: vmopv1a5.SCSIControllerTypeParaVirtualSCSI, + }, + { + BusNumber: 2, + Type: vmopv1a5.SCSIControllerTypeParaVirtualSCSI, + }, + { + BusNumber: 3, + Type: vmopv1a5.SCSIControllerTypeParaVirtualSCSI, + SharingMode: vmopv1a5.VirtualControllerSharingModePhysical, + }, + }, + NVMEControllers: []vmopv1a5.NVMEControllerSpec{ + { + BusNumber: 1, + }, + }, + SATAControllers: []vmopv1a5.SATAControllerSpec{ + { + BusNumber: 1, + }, + }, + }, + } + }), + Entry("creating multiple implicit PVCs", func() testSpec { + pvcs := createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + }, 64+1) // Full bus number slots plus one. + + return testSpec{ + pvcs: pvcs, + } + }), + ) + + Describe("Controller lifecycle", func() { + DescribeTable("Adding/Deleting controllers", + func(vmPowerState vmopv1a5.VirtualMachinePowerState) { + hardware := vmopv1a5.VirtualMachineHardwareSpec{ + SCSIControllers: []vmopv1a5.SCSIControllerSpec{ + { + BusNumber: 0, + }, + { + BusNumber: 1, + Type: vmopv1a5.SCSIControllerTypeParaVirtualSCSI, + }, + { + BusNumber: 2, + Type: vmopv1a5.SCSIControllerTypeLsiLogic, + }, + }, + NVMEControllers: []vmopv1a5.NVMEControllerSpec{ + { + BusNumber: 1, + }, + { + BusNumber: 2, + SharingMode: vmopv1a5.VirtualControllerSharingModePhysical, + }, + }, + SATAControllers: []vmopv1a5.SATAControllerSpec{ + { + BusNumber: 1, + }, + }, + // We do not create IDE controllers because they are created by default. + } + + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PowerState: string(vmPowerState), + Hardware: &hardware, + }) + vmYamls = append(vmYamls, vmYaml) + + By("Create and wait for VM to power on") + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, vmName, string(vmPowerState)) + + By("Verifying the controllers are created") + verifyCreatedControllersCount(ctx, config, svClusterClient, vmSvcNamespace, vmName, + map[vmopv1a5.VirtualControllerType]int{ + vmopv1a5.VirtualControllerTypeSCSI: len(hardware.SCSIControllers), + vmopv1a5.VirtualControllerTypeSATA: len(hardware.SATAControllers), + vmopv1a5.VirtualControllerTypeNVME: len(hardware.NVMEControllers), + vmopv1a5.VirtualControllerTypeIDE: 2, + }, + ) + + By("Deleting controller from each type") + + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + Expect(vm).ToNot(BeNil(), "VirtualMachine is nil") + vmPatch := vm.DeepCopy() + vmPatch.Spec.Hardware.SCSIControllers = vmPatch.Spec.Hardware.SCSIControllers[:len(vmPatch.Spec.Hardware.SCSIControllers)-1] + vmPatch.Spec.Hardware.SATAControllers = vmPatch.Spec.Hardware.SATAControllers[:len(vmPatch.Spec.Hardware.SATAControllers)-1] + vmPatch.Spec.Hardware.NVMEControllers = vmPatch.Spec.Hardware.NVMEControllers[:len(vmPatch.Spec.Hardware.NVMEControllers)-1] + + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine") + verifyCreatedControllersCount(ctx, config, svClusterClient, vmSvcNamespace, vmName, + map[vmopv1a5.VirtualControllerType]int{ + vmopv1a5.VirtualControllerTypeSCSI: len(vmPatch.Spec.Hardware.SCSIControllers), + vmopv1a5.VirtualControllerTypeSATA: len(vmPatch.Spec.Hardware.SATAControllers), + vmopv1a5.VirtualControllerTypeNVME: len(vmPatch.Spec.Hardware.NVMEControllers), + vmopv1a5.VirtualControllerTypeIDE: 2, + }) + + By("Creating new controller for each type") + + vm, err = utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + Expect(vm).ToNot(BeNil(), "VirtualMachine is nil") + vmPatch = vm.DeepCopy() + vmPatch.Spec.Hardware.SCSIControllers = append(vmPatch.Spec.Hardware.SCSIControllers, hardware.SCSIControllers[len(hardware.SCSIControllers)-1]) + vmPatch.Spec.Hardware.SATAControllers = append(vmPatch.Spec.Hardware.SATAControllers, hardware.SATAControllers[len(hardware.SATAControllers)-1]) + vmPatch.Spec.Hardware.NVMEControllers = append(vmPatch.Spec.Hardware.NVMEControllers, hardware.NVMEControllers[len(hardware.NVMEControllers)-1]) + + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine") + verifyCreatedControllersCount(ctx, config, svClusterClient, vmSvcNamespace, vmName, + map[vmopv1a5.VirtualControllerType]int{ + vmopv1a5.VirtualControllerTypeSCSI: len(vmPatch.Spec.Hardware.SCSIControllers), + vmopv1a5.VirtualControllerTypeSATA: len(vmPatch.Spec.Hardware.SATAControllers), + vmopv1a5.VirtualControllerTypeNVME: len(vmPatch.Spec.Hardware.NVMEControllers), + vmopv1a5.VirtualControllerTypeIDE: 2, + }) + }, + Entry("while VM is powered on", vmopv1a5.VirtualMachinePowerStateOn), + Entry("while VM is powered off", vmopv1a5.VirtualMachinePowerStateOff), + ) + + It("Updating controller while VM is powered off should succeed", func() { + hardware := vmopv1a5.VirtualMachineHardwareSpec{ + SCSIControllers: []vmopv1a5.SCSIControllerSpec{ + { + BusNumber: 0, + }, + }, + NVMEControllers: []vmopv1a5.NVMEControllerSpec{ + { + BusNumber: 1, + }, + }, + } + + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PowerState: string(vmopv1a5.VirtualMachinePowerStateOff), + Hardware: &hardware, + }) + vmYamls = append(vmYamls, vmYaml) + + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, vmName, string(vmopv1a5.VirtualMachinePowerStateOff)) + + By("Verifying the controllers are created") + verifyCreatedControllersCount(ctx, config, svClusterClient, vmSvcNamespace, vmName, + map[vmopv1a5.VirtualControllerType]int{ + vmopv1a5.VirtualControllerTypeSCSI: len(hardware.SCSIControllers), + vmopv1a5.VirtualControllerTypeNVME: len(hardware.NVMEControllers), + }) + + By("Updating the controllers") + + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + Expect(vm).ToNot(BeNil(), "VirtualMachine is nil") + vmPatch := vm.DeepCopy() + vmPatch.Spec.Hardware.SCSIControllers[0].SharingMode = vmopv1a5.VirtualControllerSharingModePhysical + vmPatch.Spec.Hardware.SCSIControllers[0].Type = vmopv1a5.SCSIControllerTypeLsiLogic + + vmPatch.Spec.Hardware.NVMEControllers[0].SharingMode = vmopv1a5.VirtualControllerSharingModePhysical + + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine") + + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, vmSvcNamespace, vmName, + metav1.Condition{ + Type: "VirtualMachineHardwareControllersVerified", + Status: metav1.ConditionTrue, + }, + ) + }) + + It("Updating controller while VM is powered on should fail", func() { + hardware := vmopv1a5.VirtualMachineHardwareSpec{ + SCSIControllers: []vmopv1a5.SCSIControllerSpec{ + { + BusNumber: 0, + }, + }, + } + + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PowerState: string(vmopv1a5.VirtualMachinePowerStateOn), + Hardware: &hardware, + }) + vmYamls = append(vmYamls, vmYaml) + + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, vmName, string(vmopv1a5.VirtualMachinePowerStateOn)) + + By("Verifying the controllers are created") + verifyCreatedControllersCount(ctx, config, svClusterClient, vmSvcNamespace, vmName, + map[vmopv1a5.VirtualControllerType]int{ + vmopv1a5.VirtualControllerTypeSCSI: len(hardware.SCSIControllers), + }) + + By("Updating the controllers") + + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + Expect(vm).ToNot(BeNil(), "VirtualMachine is nil") + vmPatch := vm.DeepCopy() + vmPatch.Spec.Hardware.SCSIControllers[0].SharingMode = vmopv1a5.VirtualControllerSharingModePhysical + + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(MatchError(ContainSubstring("spec.hardware.scsiControllers[0].sharingMode: Forbidden")), + "expected error when updating controller while VM is powered on") + }) + }) + + Describe("Multiple VMs sharing MultiWriter PVCs", func() { + numberOfVMs := 3 + + AfterEach(func() { + // We delete the VMs one after the other instead of relying on + // deleting the entire yaml because the PVCs are shared between the VMs + // and we need to ensure that VMs are deleted before PVCs. + for i := range numberOfVMs { + vmName := fmt.Sprintf("%s-%d", vmName, i) + + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + if err != nil { + e2eframework.Logf("failed to get VirtualMachine %s: %v", vmName, err) + continue + } + + err = svClusterClient.Delete(ctx, vm) + if err != nil { + e2eframework.Logf("failed to delete VirtualMachine %s: %v", vmName, err) + } + } + }) + + It("VMs should power on successfully", func() { + pvcs := createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: eztStorageProfileName, + SharingMode: ptr.To(string(vmopv1a5.VolumeSharingModeMultiWriter)), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: ptr.To(corev1.PersistentVolumeBlock), + }, 3) + pvcs = append(pvcs, createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: eztStorageProfileName, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: ptr.To(corev1.PersistentVolumeBlock), + ApplicationType: vmopv1a5.VolumeApplicationTypeOracleRAC, + }, 1)...) + + var vms []string + for i := range numberOfVMs { + vms = append(vms, fmt.Sprintf("%s-%d", vmName, i)) + } + + for _, vmName := range vms { + By("Creating Virtual Machine: " + vmName) + vmYaml := manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PVCs: pvcs, + }) + vmYamls = append(vmYamls, vmYaml) + + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + } + + for _, vmName := range vms { + By("Waiting for the Virtual Machine to be created: " + vmName) + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, vmSvcNamespace, vmName) + + By("Waiting for the batch attachment volumes to be attached: " + vmName) + + volumeNames := make([]string, len(pvcs)) + for i, pvc := range pvcs { + volumeNames[i] = pvc.VolumeName + } + // Get backfilled volumes and add them to volumeNames + backfilledVolumes := getBackfilledVolumes(ctx, config, svClusterClient, vmSvcNamespace, vmName, allDisksArePVCapabilityEnabled) + volumeNames = append(volumeNames, backfilledVolumes...) + csi.WaitForBatchAttachVolumesToBeAttached(ctx, config, svClusterClient, vmSvcNamespace, vmName, + volumeNames) + + By("Waiting for Virtual Machine to Power On: " + vmName) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, vmName, + "PoweredOn") + } + }) + }) + + Describe("VM with volumes day two actions", func() { + It("Hot attach and detaching PVs should succeed", func() { + pvcs := createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + }, 3) + + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PVCs: pvcs, + }) + vmYamls = append(vmYamls, vmYaml) + + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + + volumeNames := make([]string, len(pvcs)) + for i, pvc := range pvcs { + volumeNames[i] = pvc.VolumeName + } + // Get backfilled volumes and add them to volumeNames + backfilledVolumes := getBackfilledVolumes(ctx, config, svClusterClient, vmSvcNamespace, vmName, allDisksArePVCapabilityEnabled) + volumeNames = append(volumeNames, backfilledVolumes...) + vm := waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, volumeNames) + + By("Detaching one of the PVCs from the Virtual Machine") + + volumeNameToDetach := pvcs[len(pvcs)-1].VolumeName + volumesAfterDetach := make(map[string]bool) + + for _, vol := range pvcs { + // Exclude the volume being detached + if vol.VolumeName != volumeNameToDetach { + volumesAfterDetach[vol.VolumeName] = true + } + } + // Include backfilled volumes as they should remain attached + for _, backfilledVol := range backfilledVolumes { + volumesAfterDetach[backfilledVol] = true + } + + e2eframework.Logf("Detaching volume %s from VM %s", + volumeNameToDetach, vmName) + + vmPatch := vm.DeepCopy() + vmPatch.Spec.Volumes = []vmopv1a5.VirtualMachineVolume{} + + var detachedVol vmopv1a5.VirtualMachineVolume + + for _, vol := range vm.Spec.Volumes { + if vol.Name == volumeNameToDetach { + detachedVol = vol + } else { + vmPatch.Spec.Volumes = append(vmPatch.Spec.Volumes, vol) + } + } + + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine") + + // Convert map to sorted slice + volumeNamesList := make([]string, 0, len(volumesAfterDetach)) + for volumeName := range volumesAfterDetach { + volumeNamesList = append(volumeNamesList, volumeName) + } + + vm = waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, + volumeNamesList) + + By("Attaching the PVC back to the Virtual Machine") + + vmPatch = vm.DeepCopy() + vmPatch.Spec.Volumes = append(vmPatch.Spec.Volumes, detachedVol) + + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine") + + By("Waiting for the batch attachment volumes to be attached") + waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, + volumeNames) + }) + + It("PVC resize should succeed", func() { + pvcs := createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + }, 1) + + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PVCs: pvcs, + }) + vmYamls = append(vmYamls, vmYaml) + + By("Creating the Virtual Machine") + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + + By("Waiting for the Virtual Machine to be created") + + volumeNames := make([]string, len(pvcs)) + for i, pvc := range pvcs { + volumeNames[i] = pvc.VolumeName + } + // Get backfilled volumes and add them to volumeNames + backfilledVolumes := getBackfilledVolumes(ctx, config, svClusterClient, vmSvcNamespace, vmName, allDisksArePVCapabilityEnabled) + volumeNames = append(volumeNames, backfilledVolumes...) + vm := waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, volumeNames) + + By("Verify persistent volume capacity is 1MB") + + found := false + + for _, volStatus := range vm.Status.Volumes { + if volStatus.Name == pvcs[0].VolumeName { + Expect(volStatus.Limit.String()).To(Equal("1Mi")) + + found = true + + break + } + } + + Expect(found).To(BeTrue(), "Expected volume %s to be found in the VirtualMachine status", + pvcs[0].VolumeName, vm.Status.Volumes) + + By("Resizing the PVC: " + pvcs[0].ClaimName) + + pvc := &corev1.PersistentVolumeClaim{} + key := types.NamespacedName{ + Namespace: vmSvcNamespace, + Name: pvcs[0].ClaimName, + } + err := svClusterClient.Get(ctx, key, pvc) + Expect(err).ToNot(HaveOccurred(), "failed to get PVC") + Expect(pvc).ToNot(BeNil(), "PVC is nil") + + pvcPatch := pvc.DeepCopy() + pvcPatch.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("5Mi") + + Expect(clusterProxy.GetClient().Patch(ctx, pvcPatch, ctrlclient.MergeFrom(pvc))). + To(Succeed(), "failed to update PVC") + + By("Waiting for the Virtual Machine to be resized") + Eventually(func(g Gomega) bool { + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + for _, volStatus := range vm.Status.Volumes { + if volStatus.Name == pvcs[0].VolumeName { + return g.Expect(volStatus.Limit.String()).To(Equal("5Mi")) + } + } + + return false + }, config.GetIntervals("default", "wait-virtual-machine-resize")...). + Should(BeTrue(), "Timed out waiting for VirtualMachines %s to be resized to 5MB", vmName) + }) + + It("Updating attached volume claim should succeed", func() { + pvcs := createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + }, 2) + // Updating the request sizes to test swapping the claims. + pvcs[0].RequestSize = "1Mi" + pvcs[1].RequestSize = "2Mi" + + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PVCs: pvcs, + }) + vmYamls = append(vmYamls, vmYaml) + + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + + By("Waiting for the Virtual Machine to be created") + + volumeNames := make([]string, len(pvcs)) + for i, pvc := range pvcs { + volumeNames[i] = pvc.VolumeName + } + // Get backfilled volumes and add them to volumeNames + backfilledVolumes := getBackfilledVolumes(ctx, config, svClusterClient, vmSvcNamespace, vmName, allDisksArePVCapabilityEnabled) + volumeNames = append(volumeNames, backfilledVolumes...) + vm := waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, volumeNames) + + By("Swapping the persistent volume claim order") + + vmPatch := vm.DeepCopy() + firstClaimName := pvcs[0].ClaimName + secondClaimName := pvcs[1].ClaimName + swapped := 0 + // We have to find and swap by name because order is not guaranteed in the spec. + for i, vol := range vmPatch.Spec.Volumes { + switch vol.Name { + case pvcs[0].VolumeName: + vmPatch.Spec.Volumes[i].PersistentVolumeClaim.ClaimName = secondClaimName + swapped++ + case pvcs[1].VolumeName: + vmPatch.Spec.Volumes[i].PersistentVolumeClaim.ClaimName = firstClaimName + swapped++ + } + } + + Expect(swapped).To(Equal(2), "Expected to swap 2 volumes, got %d", swapped) + + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine") + + By("Waiting for the batch attachment volumes to be updated") + waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, + volumeNames) + + By("Verify persistent volume claim order is swapped") + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + g.Expect(err).ToNot(HaveOccurred()) + } + + foundCount := 0 + + for _, vol := range vm.Status.Volumes { + switch vol.Name { + case pvcs[0].VolumeName: + g.Expect(vol.Requested.String()).To(Equal("2Mi")) + + foundCount++ + case pvcs[1].VolumeName: + foundCount++ + + g.Expect(vol.Requested.String()).To(Equal("1Mi")) + } + } + + g.Expect(foundCount). + To(Equal(2), "Expected to find 2 volumes in the VirtualMachine status, got %d", foundCount) + }, config.GetIntervals("default", "wait-virtual-machine-resize")...). + Should(Succeed(), + "Timed out waiting for VirtualMachines %s to have the persistent volume claim order swapped", + vmName) + }) + + It("Powering VM on/off and deleting VM should succeed", func() { + pvcs := createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + }, 1) + + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PVCs: pvcs, + }) + vmYamls = append(vmYamls, vmYaml) + + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + + By("Waiting for the Virtual Machine to be created and powered on") + + volumeNames := make([]string, len(pvcs)) + for i, pvc := range pvcs { + volumeNames[i] = pvc.VolumeName + } + // Get backfilled volumes and add them to volumeNames + backfilledVolumes := getBackfilledVolumes(ctx, config, svClusterClient, vmSvcNamespace, vmName, allDisksArePVCapabilityEnabled) + volumeNames = append(volumeNames, backfilledVolumes...) + vm := waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, volumeNames) + + By("Powering off the Virtual Machine") + + vmPatch := vm.DeepCopy() + vmPatch.Spec.PowerState = vmopv1a5.VirtualMachinePowerStateOff + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, vmName, "PoweredOff") + + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + Expect(vm).ToNot(BeNil(), "VirtualMachine is nil") + + By("Powering on the Virtual Machine") + + vmPatch = vm.DeepCopy() + vmPatch.Spec.PowerState = vmopv1a5.VirtualMachinePowerStateOn + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, vmName, "PoweredOn") + + By("Deleting the Virtual Machine") + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).To(Succeed(), "failed to delete virtualmachine") + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, vmSvcNamespace, vmName) + + By("Verifying the CnsNodeVMBatchAttachment is deleted") + csi.WaitForCnsNodeVMBatchAttachmentToBeDeleted(ctx, config, svClusterClient, vmSvcNamespace, vmName) + }) + + It("Detaching and reattaching all disks", func() { + pvcs := createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: clusterResources.StorageClassName, + }, 3) + + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PVCs: pvcs, + }) + vmYamls = append(vmYamls, vmYaml) + + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + + volumeNames := make([]string, len(pvcs)) + for i, pvc := range pvcs { + volumeNames[i] = pvc.VolumeName + } + // Get backfilled volumes and add them to volumeNames + backfilledVolumes := getBackfilledVolumes(ctx, config, svClusterClient, vmSvcNamespace, vmName, allDisksArePVCapabilityEnabled) + volumeNames = append(volumeNames, backfilledVolumes...) + waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, volumeNames) + + By("Waiting on virtual machine conditions to become true") + + conditions := []metav1.Condition{ + { + Type: "VirtualMachineHardwareVolumesVerified", + Status: metav1.ConditionTrue, + }, + } + for _, condition := range conditions { + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, vmSvcNamespace, vmName, condition) + } + + By("Storing the initial volume spec and status") + // Re-fetch to get the latest resource version before patching. + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + Expect(vm).ToNot(BeNil(), "VirtualMachine is nil") + + savedSpecVolumes := make([]vmopv1a5.VirtualMachineVolume, len(vm.Spec.Volumes)) + copy(savedSpecVolumes, vm.Spec.Volumes) + + savedStatusVolumes := make([]vmopv1a5.VirtualMachineVolumeStatus, len(vm.Status.Volumes)) + copy(savedStatusVolumes, vm.Status.Volumes) + vmopv1a5.SortVirtualMachineVolumeStatuses(savedStatusVolumes) + + // Build a lookup from DiskUUID -> saved status for field-by-field comparison. + savedStatusByDiskUUID := make(map[string]vmopv1a5.VirtualMachineVolumeStatus, len(savedStatusVolumes)) + for _, vol := range savedStatusVolumes { + savedStatusByDiskUUID[vol.DiskUUID] = vol + } + + By("Powering off the VM and setting all volumes to removable") + + vm, err = utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + + vmPatch := vm.DeepCopy() + + vmPatch.Spec.PowerState = vmopv1a5.VirtualMachinePowerStateOff + for i := range vmPatch.Spec.Volumes { + vmPatch.Spec.Volumes[i].Removable = ptr.To(true) + } + + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine removable fields") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, vmName, "PoweredOff") + + By("Detaching all volumes") + + vm, err = utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + + vmPatch = vm.DeepCopy() + vmPatch.Spec.Volumes = []vmopv1a5.VirtualMachineVolume{} + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine to detach all volumes") + + By("Waiting for status.volumes to be empty after detach") + Eventually(func(g Gomega) { + vm, err = utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + g.Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + g.Expect(vm.Status.Volumes).To(BeEmpty(), + "Expected status.volumes to be empty after detaching all volumes") + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...). + Should(Succeed(), "Timed out waiting for status.volumes to be empty after detach") + + By("Waiting on virtual machine conditions to become true after detach") + + for _, condition := range conditions { + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, vmSvcNamespace, vmName, condition) + } + + By("Reattaching all previously detached volumes") + + vm, err = utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + + vmPatch = vm.DeepCopy() + vmPatch.Spec.Volumes = savedSpecVolumes + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine to reattach volumes") + + By("Waiting for the batch attachment volumes to be attached") + csi.WaitForBatchAttachVolumesToBeAttached(ctx, config, svClusterClient, vmSvcNamespace, vmName, volumeNames) + + By("Waiting on virtual machine conditions to become true after reattach") + + for _, condition := range conditions { + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, vmSvcNamespace, vmName, condition) + } + + By("Powering on the Virtual Machine after reattaching volumes") + + vm, err = utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + + vmPatch = vm.DeepCopy() + vmPatch.Spec.PowerState = vmopv1a5.VirtualMachinePowerStateOn + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to patch virtualmachine power state to on") + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, vmName, "PoweredOn") + + By("Verifying status.volumes matches the originally recorded state") + Eventually(func(g Gomega) { + vm, err = utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + g.Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + + actualStatusVolumes := make([]vmopv1a5.VirtualMachineVolumeStatus, len(vm.Status.Volumes)) + copy(actualStatusVolumes, vm.Status.Volumes) + vmopv1a5.SortVirtualMachineVolumeStatuses(actualStatusVolumes) + + g.Expect(actualStatusVolumes).To(HaveLen(len(savedStatusVolumes)), + "Expected %d volumes in status, got %d", len(savedStatusVolumes), len(actualStatusVolumes)) + + for _, cur := range actualStatusVolumes { + saved, ok := savedStatusByDiskUUID[cur.DiskUUID] + g.Expect(ok).To(BeTrue(), "volume with DiskUUID %q not found in saved status", cur.DiskUUID) + + g.Expect(cur.Name).To(Equal(saved.Name), "volume %q: Name mismatch", cur.DiskUUID) + g.Expect(cur.Type).To(Equal(saved.Type), "volume %q: Type mismatch", cur.DiskUUID) + g.Expect(cur.Attached).To(Equal(saved.Attached), "volume %q: Attached mismatch", cur.DiskUUID) + g.Expect(cur.ControllerType).To(Equal(saved.ControllerType), "volume %q: ControllerType mismatch", cur.DiskUUID) + g.Expect(cur.ControllerBusNumber).To(Equal(saved.ControllerBusNumber), "volume %q: ControllerBusNumber mismatch", cur.DiskUUID) + g.Expect(&cur.UnitNumber).To(Equal(&saved.UnitNumber), "volume %q: UnitNumber mismatch", cur.DiskUUID) + g.Expect(cur.DiskMode).To(Equal(saved.DiskMode), "volume %q: DiskMode mismatch", cur.DiskUUID) + g.Expect(cur.SharingMode).To(Equal(saved.SharingMode), "volume %q: SharingMode mismatch", cur.DiskUUID) + g.Expect(&cur.Limit).To(Equal(&saved.Limit), "volume %q: Limit mismatch", cur.DiskUUID) + g.Expect(&cur.Requested).To(Equal(&saved.Requested), "volume %q: Requested mismatch", cur.DiskUUID) + g.Expect(&cur.Crypto).To(Equal(&saved.Crypto), "volume %q: Crypto mismatch", cur.DiskUUID) + g.Expect(&cur.Error).To(Equal(&saved.Error), "volume %q: Error mismatch", cur.DiskUUID) + + // Used represents bytes-on-disk and may differ between attach cycles; + // just verify it is still populated if it was originally. + if saved.Used != nil { + g.Expect(cur.Used).ToNot(BeNil(), "volume %q: Used should still be populated after reattach", cur.Name) + } + } + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...). + Should(Succeed(), "Timed out waiting for status.volumes to match the original state after reattach") + }) + }) + + DescribeTable("VM with CD-ROM", + func(cdrom vmopv1a5.VirtualMachineCdromSpec) { + if !isoSupportFSSEnabled { + Skip("ISO Support FSS is not enabled") + return + } + + By("Get the ISO-type image CR name") + + isoImageName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, vmSvcNamespace, isoImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VMI name in namespace %q", vmSvcNamespace) + + cdrom.Image = vmopv1a5.VirtualMachineImageRef{ + Kind: "VirtualMachineImage", + Name: isoImageName, + } + + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + GuestID: "ubuntu64Guest", + StorageClassName: clusterResources.StorageClassName, + Hardware: &vmopv1a5.VirtualMachineHardwareSpec{ + Cdrom: []vmopv1a5.VirtualMachineCdromSpec{ + cdrom, + }, + }, + }) + vmYamls = append(vmYamls, vmYaml) + + By("Creating the Virtual Machine") + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + + By("Waiting for the Virtual Machine to be created and power on") + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, vmSvcNamespace, vmName) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, vmName, "PoweredOn") + + By("Verifying the CD-ROMs are attached to the first IDE controller") + + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VirtualMachine") + Expect(vm).ToNot(BeNil(), "VirtualMachine is nil") + + found := false + + for _, controller := range vm.Status.Hardware.Controllers { + if controller.Type == vmopv1a5.VirtualControllerTypeIDE && controller.BusNumber == 0 { + Expect(controller.Devices).To(HaveLen(1), "Expected 1 device on the controller") + Expect(controller.Devices[0].Type).To(Equal(vmopv1a5.VirtualDeviceTypeCDROM), "Expected device type to be CDROM") + + if cdrom.UnitNumber != nil { + Expect(controller.Devices[0].UnitNumber).To(Equal(*cdrom.UnitNumber), "Expected device unit number to be %d", *cdrom.UnitNumber) + } else { + Expect(controller.Devices[0].UnitNumber).To(BeNumerically(">=", 0), "Expected device unit number to be >= 0") + } + + found = true + + break + } + } + + Expect(found).To(BeTrue(), "Expected to find the first IDE controller") + }, + Entry("VM with implicit CD-ROM placement", vmopv1a5.VirtualMachineCdromSpec{ + Name: "cdrom1", + Connected: ptr.To(true), + AllowGuestControl: ptr.To(true), + }), + Entry("VM with explicit CD-ROM placement", vmopv1a5.VirtualMachineCdromSpec{ + Name: "cdrom1", + Connected: ptr.To(true), + AllowGuestControl: ptr.To(true), + ControllerBusNumber: ptr.To(int32(0)), + ControllerType: vmopv1a5.VirtualControllerTypeIDE, + UnitNumber: ptr.To(int32(0)), + }), + ) + + Describe("VM with boot disk as PVC", func() { + BeforeEach(func() { + skipper.SkipUnlessSupervisorCapabilityEnabled(ctx, clusterProxy, consts.AllDisksArePVCapabilityName) + }) + + It("Boot disk PVC lifecycle operations should succeed", func() { + vmYaml = manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PVCs: []manifestbuilders.PVC{}, + }) + vmYamls = append(vmYamls, vmYaml) + + By("Creating the Virtual Machine") + e2eframework.Logf("Creating the Virtual Machine with yaml: %s", string(vmYaml)) + Expect(clusterProxy.ApplyWithArgs(ctx, vmYaml)).To(Succeed(), "failed to apply virtualmachine") + + waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, []string{}) + + By("Waiting on virtual machine conditions to become true") + + conditions := []metav1.Condition{ + { + Type: "VirtualMachineUnmanagedVolumesBackfilled", + Status: metav1.ConditionTrue, + }, + { + Type: "VirtualMachineUnmanagedVolumesRegistered", + Status: metav1.ConditionTrue, + }, + } + for _, condition := range conditions { + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, vmSvcNamespace, vmName, condition) + } + + volumeNames := make([]string, 0) + + By("Waiting for the boot disk to be promoted to a PVC") + + var bootDiskVolName string + + Eventually(func(g Gomega) bool { + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + for _, vol := range vm.Spec.Volumes { + volumeNames = append(volumeNames, vol.Name) + if vol.ControllerBusNumber != nil && *vol.ControllerBusNumber == 0 && + vol.UnitNumber != nil && *vol.UnitNumber == 0 { + g.Expect(vol.PersistentVolumeClaim).ToNot(BeNil(), + "Expected boot disk to have a PersistentVolumeClaim") + g.Expect(vol.PersistentVolumeClaim.ClaimName).ToNot(BeEmpty(), + "Expected boot disk PVC to have a claim name") + bootDiskVolName = vol.Name + + return true + } + } + + return false + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...). + Should(BeTrue(), "Timed out waiting for boot disk to be found in spec.volumes") + + By("Verify volumes in batch attachment CRD") + csi.WaitForBatchAttachVolumesToBeAttached(ctx, config, svClusterClient, vmSvcNamespace, vmName, volumeNames) + + By("Verifying the boot disk PVC exists") + + bootDiskPVC := &corev1.PersistentVolumeClaim{} + bootDiskPVCKey := types.NamespacedName{ + Namespace: vmSvcNamespace, + Name: bootDiskVolName, + } + + Eventually(func(g Gomega) { + err := svClusterClient.Get(ctx, bootDiskPVCKey, bootDiskPVC) + g.Expect(err).ToNot(HaveOccurred(), "failed to get boot disk PVC") + g.Expect(bootDiskPVC).ToNot(BeNil(), "boot disk PVC is nil") + g.Expect(bootDiskPVC.Status.Phase).To(Equal(corev1.ClaimBound), + "Expected boot disk PVC to be bound") + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...). + Should(Succeed(), "Timed out waiting for boot disk PVC to be found in status") + + By("Verifying the boot disk becomes managed PVC") + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + g.Expect(err).ToNot(HaveOccurred()) + + var bootDiskVolume *vmopv1a5.VirtualMachineVolumeStatus + + for i, volStatus := range vm.Status.Volumes { + if volStatus.Name == bootDiskVolName { + bootDiskVolume = &vm.Status.Volumes[i] + break + } + } + + g.Expect(bootDiskVolume).ToNot(BeNil(), "Expected to find boot disk volume in status") + g.Expect(bootDiskVolume.Type).To(Equal(vmopv1a5.VolumeTypeManaged), + "Expected boot disk to be of type Managed (PVC)") + g.Expect(bootDiskVolume.Name).ToNot(BeEmpty(), "Expected boot disk volume to have a name") + g.Expect(bootDiskVolume.Attached).To(BeTrue(), "Expected boot disk to be attached") + g.Expect(bootDiskVolume.DiskUUID).ToNot(BeEmpty(), "Expected boot disk to have a disk UUID") + + g.Expect(bootDiskVolume.Limit).ToNot(BeNil(), "Expected volume %s to have a limit", bootDiskVolume.Name) + _, ok := bootDiskVolume.Limit.AsInt64() + g.Expect(ok).To(BeTrue(), "Expected volume %s limit to be convertible to int64", bootDiskVolume.Name) + + g.Expect(bootDiskVolume.Requested).ToNot(BeNil(), "Expected volume %s to have a requested", bootDiskVolume.Name) + _, ok = bootDiskVolume.Requested.AsInt64() + g.Expect(ok).To(BeTrue(), "Expected volume %s requested to be convertible to int64", bootDiskVolume.Name) + + g.Expect(bootDiskVolume.Used).ToNot(BeNil(), "Expected volume %s to have a used", bootDiskVolume.Name) + _, ok = bootDiskVolume.Used.AsInt64() + g.Expect(ok).To(BeTrue(), "Expected volume %s used to be convertible to int64", bootDiskVolume.Name) + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...). + Should(Succeed(), "Timed out waiting for boot disk to become managed PVC") + + By("Creating a new PVC with a different storage class and larger size") + // Create a new PVC with a different storage class and larger size. + newBootDiskPVCName := fmt.Sprintf("%s-new-boot-disk", vmName) + newBootDiskPVCSize := bootDiskPVC.Spec.Resources.Requests[corev1.ResourceStorage] + newBootDiskPVCSize.Add(resource.MustParse("1Gi")) + newBootDiskPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: newBootDiskPVCName, + Namespace: vmSvcNamespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: newBootDiskPVCSize, + }, + }, + StorageClassName: ptr.To(eztStorageProfileName), + }, + } + Expect(svClusterClient.Create(ctx, newBootDiskPVC)). + To(Succeed(), "failed to create new boot disk PVC with different storage class") + + By("Updating the VM to use the new PVC with different storage class") + + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + Expect(err).ToNot(HaveOccurred(), "failed to get VM") + + vmPatch := vm.DeepCopy() + for i, vol := range vmPatch.Spec.Volumes { + if vol.PersistentVolumeClaim != nil && vol.PersistentVolumeClaim.ClaimName == bootDiskVolName { + vmPatch.Spec.Volumes[i].PersistentVolumeClaim.ClaimName = newBootDiskPVCName + break + } + } + + Expect(clusterProxy.GetClient().Patch(ctx, vmPatch, ctrlclient.MergeFrom(vm))). + To(Succeed(), "failed to update VM with new PVC") + + By("Waiting for the new PVC to be attached and reflected in VM status") + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, vmName) + g.Expect(err).ToNot(HaveOccurred()) + + verifiedInStatus := false + + for _, volStatus := range vm.Status.Volumes { + if volStatus.Name == bootDiskVolName { + verifiedInStatus = true + + g.Expect(volStatus.Attached).To(BeTrue()) + g.Expect(volStatus.Limit.Cmp(newBootDiskPVCSize) >= 0).To(BeTrue()) + } + } + + g.Expect(verifiedInStatus).To(BeTrue(), "Expected to find the new boot disk in status") + }, config.GetIntervals("default", "wait-virtual-machine-resize")...). + Should(Succeed(), "Timed out waiting for new PVC to be attached") + }) + }) + + Describe("Brownfield VM Import", func() { + var ( + vCenterAdminClient *vim25.Client + clusterMoID string + brownfieldVMName string + brownfieldVMMoID string + importOperation *mopv1a2.ImportOperation + ) + + BeforeEach(func() { + // Get vCenter client with admin credentials. + kubeconfigPath := clusterProxy.GetKubeconfigPath() + vCenterHostname := vcenter.GetVCPNIDFromKubeconfigFile(ctx, kubeconfigPath) + + var err error + + vCenterAdminClient, err = vcenter.NewVimClient(vCenterHostname, testbed.AdminUsername, testbed.AdminPassword) + Expect(err).ToNot(HaveOccurred(), "Failed to create vCenter client") + + // Get the WCP cluster MoID and find resources. + zones := &topologyv1.ZoneList{} + listOpts := &ctrlclient.ListOptions{Namespace: vmSvcNamespace} + err = svClusterClient.List(ctx, zones, listOpts) + Expect(err).ToNot(HaveOccurred(), "failed to list zones bound with namespace %s", vmSvcNamespace) + Expect(len(zones.Items)). + To(BeNumerically(">", 0), "Expected to have at least one zone bound with namespace %s", vmSvcNamespace) + azName := zones.Items[0].Spec.Zone.Name + az := &topologyv1.AvailabilityZone{} + err = svClusterClient.Get(ctx, ctrlclient.ObjectKey{Name: azName}, az) + Expect(err).ToNot(HaveOccurred()) + + clusterMoID = az.Spec.ClusterComputeResourceMoId + if clusterMoID == "" && len(az.Spec.ClusterComputeResourceMoIDs) > 0 { + clusterMoID = az.Spec.ClusterComputeResourceMoIDs[0] + } + + Expect(clusterMoID).ToNot(BeEmpty(), "Expected to have at least one cluster MoID in AvailabilityZone %s", azName) + e2eframework.Logf("WCP cluster MoID: %s", clusterMoID) + + brownfieldVMName = fmt.Sprintf("brownfield-vm-hardware-%s", capiutil.RandomString(4)) + }) + + AfterEach(func() { + if brownfieldVMMoID != "" { + By(fmt.Sprintf("Cleaning up brownfield VM %s (%s) in vCenter", brownfieldVMName, brownfieldVMMoID)) + vm := object.NewVirtualMachine(vCenterAdminClient, vimtypes.ManagedObjectReference{ + Type: "VirtualMachine", + Value: brownfieldVMMoID, + }) + + e2eframework.Logf("Found brownfield VM %s (MoID: %s), deleting it", brownfieldVMName, brownfieldVMMoID) + + powerState, err := vm.PowerState(ctx) + if err == nil && powerState == vimtypes.VirtualMachinePowerStatePoweredOn { + task, err := vm.PowerOff(ctx) + if err != nil { + e2eframework.Logf("Failed to power off VM %s: %v", brownfieldVMName, err) + } else { + err := task.Wait(ctx) + if err != nil { + e2eframework.Logf("Failed to wait for VM %s power off: %v", brownfieldVMName, err) + } + } + } + + destroyTask, err := vm.Destroy(ctx) + if err == nil { + err := destroyTask.Wait(ctx) + if err != nil { + e2eframework.Logf("Failed to wait for VM %s destruction: %v", brownfieldVMName, err) + } else { + e2eframework.Logf("Deleted brownfield VM %s", brownfieldVMName) + } + } else { + e2eframework.Logf("Failed to destroy VM %s: %v", brownfieldVMName, err) + } + } + + if importOperation != nil { + err := svClusterClient.Delete(ctx, importOperation) + if err != nil && !apierrors.IsNotFound(err) { + Expect(err).ToNot(HaveOccurred(), "Failed to delete ImportOperation") + } + } + + vcenter.LogoutVimClient(vCenterAdminClient) + }) + + It("Import brownfield VM with hardware should succeed", func() { + By("Creating a brownfield VM by deploying from content library template using govmomi") + // Create REST client for content library operations. + restClient, err := vcenter.NewRestClient(ctx, vCenterAdminClient, testbed.AdminUsername, testbed.AdminPassword) + Expect(err).ToNot(HaveOccurred(), "Failed to create REST client") + + // Find the photon image in the VirtualMachineImage CRs. + By("Finding photon template in content library") + + photonImageDisplayName := "photon-5.0" + photonImageName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, photonImageDisplayName) + Expect(err).ToNot(HaveOccurred(), "Failed to find photon image") + + photonImage := vmopv1a5.VirtualMachineImage{} + err = svClusterClient.Get(ctx, ctrlclient.ObjectKey{ + Name: photonImageName, + Namespace: input.WCPNamespaceName, + }, &photonImage) + Expect(err).ToNot(HaveOccurred(), "Failed to get photon image") + + libraryItemID := photonImage.Status.ProviderItemID + Expect(libraryItemID).ToNot(BeEmpty(), "Photon image has no ProviderItemID") + e2eframework.Logf("Found photon image %s with library item ID: %s", photonImageName, libraryItemID) + + // Setup finder to get cluster and resources. + finder := find.NewFinder(vCenterAdminClient, false) + ccr, err := finder.ClusterComputeResource(ctx, clusterMoID) + Expect(err).ToNot(HaveOccurred(), "Failed to get cluster compute resource") + datastores, err := ccr.Datastores(ctx) + Expect(err).ToNot(HaveOccurred(), "Failed to get datastores") + Expect(len(datastores)).To(BeNumerically(">", 0), "Expected to have at least one datastore") + + // Filter for shared datastores by checking the summary. + var datastore *object.Datastore + + for _, ds := range datastores { + var dsMO mo.Datastore + + err := ds.Properties(ctx, ds.Reference(), []string{"summary"}, &dsMO) + if err != nil { + continue + } + + // Check if datastore is shared (accessible by multiple hosts) + // MultipleHostAccess indicates a shared datastore + if dsMO.Summary.MultipleHostAccess != nil && *dsMO.Summary.MultipleHostAccess { + datastore = ds + + e2eframework.Logf("Found shared datastore: %s (type: %s)", dsMO.Summary.Name, dsMO.Summary.Type) + + break + } + } + + // Fallback to first datastore if no shared datastore found. + if datastore == nil { + datastore = datastores[0] + e2eframework.Logf("No shared datastore found, using first datastore: %s", datastore.Name()) + } + + // Get cluster resource pool. + resourcePool, err := ccr.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred(), "Failed to get cluster resource pool") + + // Deploy VM from content library using govmomi's vcenter library. + By(fmt.Sprintf("Deploying VM %s from content library item %s", brownfieldVMName, libraryItemID)) + + // Create vcenter manager for deployment. + vcenterManager := govc.NewManager(restClient) + + // Create deployment spec + deploySpec := govc.Deploy{ + DeploymentSpec: govc.DeploymentSpec{ + Name: brownfieldVMName, + DefaultDatastoreID: datastore.Reference().Value, + AcceptAllEULA: true, + }, + Target: govc.Target{ + ResourcePoolID: resourcePool.Reference().Value, + }, + } + + // Deploy the VM from the library item. + deployedVMRef, err := vcenterManager.DeployLibraryItem(ctx, libraryItemID, deploySpec) + Expect(err).ToNot(HaveOccurred(), "Failed to deploy VM from content library") + Expect(deployedVMRef).ToNot(BeNil(), "Deployed VM reference is nil") + + brownfieldVMMoID = deployedVMRef.Value + e2eframework.Logf("Deployed brownfield VM %s with MoID: %s", brownfieldVMName, brownfieldVMMoID) + + // Get the deployed VM object. + brownfieldVM := object.NewVirtualMachine(vCenterAdminClient, vimtypes.ManagedObjectReference{ + Type: "VirtualMachine", + Value: brownfieldVMMoID, + }) + + // Power on the VM. + By("Powering on the brownfield VM") + + powerOnTask, err := brownfieldVM.PowerOn(ctx) + Expect(err).ToNot(HaveOccurred(), "Failed to power on VM") + err = powerOnTask.Wait(ctx) + Expect(err).ToNot(HaveOccurred(), "Failed to wait for VM power on") + e2eframework.Logf("Brownfield VM powered on") + + By("Adding additional hardware to the brownfield VM using govmomi") + // We'll add various hardware components to test the import properly detects them: + // - Additional SCSI controller (ParaVirtual) + // - SATA controller + // - Additional disk on the new SCSI controller + + // Get current VM devices + var moVM mo.VirtualMachine + + err = brownfieldVM.Properties(ctx, brownfieldVM.Reference(), []string{"config.hardware.device", "datastore"}, &moVM) + Expect(err).ToNot(HaveOccurred(), "Failed to get VM properties") + + var deviceChanges []vimtypes.BaseVirtualDeviceConfigSpec + + // Add a ParaVirtual SCSI controller (bus 1). + // Use negative key so vCenter assigns it automatically. + pvscsiController := &vimtypes.ParaVirtualSCSIController{ + VirtualSCSIController: vimtypes.VirtualSCSIController{ + SharedBus: vimtypes.VirtualSCSISharingNoSharing, + VirtualController: vimtypes.VirtualController{ + BusNumber: 1, + VirtualDevice: vimtypes.VirtualDevice{ + Key: -1, + }, + }, + }, + } + + deviceChanges = append(deviceChanges, &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationAdd, + Device: pvscsiController, + }) + + e2eframework.Logf("Adding ParaVirtual SCSI controller (bus 1)") + + // Add a SATA controller (bus 0). + // Use negative key so vCenter assigns it automatically. + sataController := &vimtypes.VirtualAHCIController{ + VirtualSATAController: vimtypes.VirtualSATAController{ + VirtualController: vimtypes.VirtualController{ + BusNumber: 0, + VirtualDevice: vimtypes.VirtualDevice{ + Key: -2, + }, + }, + }, + } + + deviceChanges = append(deviceChanges, &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationAdd, + Device: sataController, + }) + + e2eframework.Logf("Adding SATA controller (bus 0)") + + // Add a small disk (5MB) attached to the new SCSI controller. + // Get the datastore for the disk + Expect(moVM.Datastore).ToNot(BeEmpty(), "VM has no datastores") + datastoreRef := moVM.Datastore[0] + + disk := &vimtypes.VirtualDisk{ + VirtualDevice: vimtypes.VirtualDevice{ + Backing: &vimtypes.VirtualDiskFlatVer2BackingInfo{ + DiskMode: string(vimtypes.VirtualDiskModePersistent), + ThinProvisioned: new(true), + VirtualDeviceFileBackingInfo: vimtypes.VirtualDeviceFileBackingInfo{ + Datastore: &datastoreRef, + }, + }, + ControllerKey: pvscsiController.Key, + UnitNumber: vimtypes.NewInt32(0), + }, + CapacityInBytes: 5 * 1024 * 1024, // 5MB + } + + deviceChanges = append(deviceChanges, &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationAdd, + FileOperation: vimtypes.VirtualDeviceConfigSpecFileOperationCreate, + Device: disk, + }) + + e2eframework.Logf("Adding 5MB disk on new SCSI controller") + + // Reconfigure the VM with the new devices. + configSpec := vimtypes.VirtualMachineConfigSpec{ + DeviceChange: deviceChanges, + ExtraConfig: []vimtypes.BaseOptionValue{ + &vimtypes.OptionValue{ + Key: "test.brownfield.import", + Value: "true", + }, + }, + } + + reconfigTask, err := brownfieldVM.Reconfigure(ctx, configSpec) + Expect(err).ToNot(HaveOccurred(), "Failed to reconfigure VM with new hardware") + err = reconfigTask.Wait(ctx) + Expect(err).ToNot(HaveOccurred(), "Failed to wait for VM reconfiguration") + e2eframework.Logf("Successfully added hardware to brownfield VM") + + // Now import the brownfield VM using ImportOperation. + By("Creating ImportOperation to import the brownfield VM") + + importOpName := fmt.Sprintf("import-%s", vmName) + importOperation = &mopv1a2.ImportOperation{ + ObjectMeta: metav1.ObjectMeta{ + Name: importOpName, + Namespace: vmSvcNamespace, + }, + Spec: mopv1a2.ImportOperationSpec{ + VirtualMachineID: brownfieldVMMoID, + StorageClass: clusterResources.StorageClassName, + }, + } + + Expect(svClusterClient.Create(ctx, importOperation)).To(Succeed(), "Failed to create ImportOperation") + e2eframework.Logf("Created ImportOperation: %s", importOpName) + + // Wait for ImportOperation to complete. + By("Waiting for ImportOperation to complete") + + var importedVMName string + + Eventually(func(g Gomega) { + err := svClusterClient.Get(ctx, ctrlclient.ObjectKey{ + Namespace: vmSvcNamespace, + Name: importOpName, + }, importOperation) + g.Expect(err).ToNot(HaveOccurred(), "Failed to get ImportOperation") + + // Check if operation completed. + for _, cond := range importOperation.Status.Conditions { + if cond.Type == "VirtualMachineCreated" && cond.Status == metav1.ConditionTrue { + importedVMName = importOperation.Status.VirtualMachineName + g.Expect(importedVMName).ToNot(BeEmpty(), "ImportOperation completed but VirtualMachineName is empty") + + return + } + + if cond.Type == "Failed" && cond.Status == metav1.ConditionTrue { + Fail(fmt.Sprintf("ImportOperation failed: %s", cond.Message)) + } + } + + g.Expect(false).To(BeTrue(), "ImportOperation not yet complete") + }, config.GetIntervals("default", "wait-virtual-machine-creation")...). + Should(Succeed(), "Timed out waiting for ImportOperation to complete") + + e2eframework.Logf("ImportOperation completed, imported VM name: %s", importedVMName) + + vmYamls = append(vmYamls, manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: importedVMName, + })) + + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, vmSvcNamespace, importedVMName, "PoweredOn") + + By("Verifying controllers in status") + verifyCreatedControllersCount(ctx, config, svClusterClient, vmSvcNamespace, importedVMName, map[vmopv1a5.VirtualControllerType]int{ + vmopv1a5.VirtualControllerTypeIDE: 2, + vmopv1a5.VirtualControllerTypeSCSI: 2, + vmopv1a5.VirtualControllerTypeSATA: 1, + }) + + By("Waiting on virtual machine conditions to become true") + + conditions := []metav1.Condition{ + { + Type: "VirtualMachineUnmanagedVolumesBackfilled", + Status: metav1.ConditionTrue, + }, + { + Type: "VirtualMachineUnmanagedVolumesRegistered", + Status: metav1.ConditionTrue, + }, + } + for _, condition := range conditions { + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, vmSvcNamespace, importedVMName, condition) + } + + By("Verifying volumes in VM status") + + volumeNames := make([]string, 0) + + Eventually(func(g Gomega) { + vm, err := utils.GetVirtualMachineA5(ctx, svClusterClient, vmSvcNamespace, importedVMName) + g.Expect(err).ToNot(HaveOccurred()) + + volumeNames = make([]string, 0) + + for i, vol := range vm.Status.Volumes { + volName := vol.Name + volumeNames = append(volumeNames, volName) + g.Expect(volName).ToNot(BeEmpty(), "Expected volume, volumes[%d] to have a name", i) + g.Expect(vol).ToNot(BeNil(), "Expected to find volume %s in status", volName) + g.Expect(vol.Type).To(Equal(vmopv1a5.VolumeTypeManaged), + "Expected volume to be of type Managed (PVC)") + g.Expect(vol.Attached).To(BeTrue(), "Expected volume %s to be attached", volName) + g.Expect(vol.DiskUUID).ToNot(BeEmpty(), "Expected volume %s to have a disk UUID", volName) + + g.Expect(vol.Limit).ToNot(BeNil(), "Expected volume %s to have a limit", volName) + _, ok := vol.Limit.AsInt64() + g.Expect(ok).To(BeTrue(), "Expected volume %s to have a limit", volName) + + g.Expect(vol.Requested).ToNot(BeNil(), "Expected volume %s to have a requested", volName) + _, ok = vol.Requested.AsInt64() + g.Expect(ok).To(BeTrue(), "Expected volume %s to have a requested", volName) + + g.Expect(vol.Used).ToNot(BeNil(), "Expected volume %s to have a used", volName) + _, ok = vol.Used.AsInt64() + g.Expect(ok).To(BeTrue(), "Expected volume %s to have a used", volName) + } + }, config.GetIntervals("default", "wait-virtual-machine-condition-update")...). + Should(Succeed(), "Timed out waiting for volumes to be found in status") + + if allDisksArePVCapabilityEnabled { + By("Verify volumes in batch attachment CRD") + csi.WaitForBatchAttachVolumesToBeAttached(ctx, config, svClusterClient, vmSvcNamespace, importedVMName, volumeNames) + + By("Verifying volumes have a PVC") + + for _, volName := range volumeNames { + pvc := &corev1.PersistentVolumeClaim{} + pvcKey := ctrlclient.ObjectKey{ + Namespace: vmSvcNamespace, + Name: volName, + } + err := svClusterClient.Get(ctx, pvcKey, pvc) + Expect(err).ToNot(HaveOccurred(), "Failed to get PVC %s", volName) + Expect(pvc.Status.Phase).To(Equal(corev1.ClaimBound), + "Expected PVC %s to be bound", volName) + } + } + }) + }) + + Describe("Multi-Writer and Encryption Validation Webhook", Label("extended-functional", "experimental"), func() { + BeforeAll(func() { + By("Ensuring encryption storage policy and class in test namespace (shared with VMEncryptionSpec)") + vCenterClient = vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + DeferCleanup(vcenter.LogoutVimClient, vCenterClient) + + Expect(utils.EnsureE2EEncryptionStorageInNamespace(ctx, vCenterClient, wcpClient, + clusterProxy.GetClientSet(), svClusterClient, *config, + vmSvcNamespace, clusterResources.StorageClassName)).To(Succeed(), + "failed to ensure encryption storage in namespace %s", vmSvcNamespace) + }) + + It("should reject a VM whose encrypted PVC uses MultiWriter sharing mode", func() { + pvcs := createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: utils.E2EEncryptionStorageClassName, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: ptr.To(corev1.PersistentVolumeBlock), + }, 1) + pvcSpec := pvcs[0] + + By("Creating the encrypted PVC before the VM") + Expect(clusterProxy.ApplyWithArgs(ctx, manifestbuilders.GetPersistentVolumeClaimYaml(pvcSpec))). + To(Succeed(), "failed to create encrypted PVC %s", pvcSpec.ClaimName) + DeferCleanup(func() { + _ = svClusterClient.Delete(ctx, &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcSpec.ClaimName, + Namespace: vmSvcNamespace, + }, + }) + }) + + By("Attempting to create VM with encrypted PVC in MultiWriter mode (should be webhook-rejected)") + vm := &vmopv1a5.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: vmName, + Namespace: vmSvcNamespace, + }, + Spec: vmopv1a5.VirtualMachineSpec{ + ClassName: clusterResources.VMClassName, + ImageName: linuxImageDisplayName, + StorageClass: clusterResources.StorageClassName, + Volumes: []vmopv1a5.VirtualMachineVolume{ + { + Name: pvcSpec.VolumeName, + VirtualMachineVolumeSource: vmopv1a5.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1a5.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcSpec.ClaimName, + }, + }, + }, + SharingMode: vmopv1a5.VolumeSharingModeMultiWriter, + }, + }, + }, + } + + err := svClusterClient.Create(ctx, vm) + Expect(err).To(HaveOccurred(), + "expected webhook to reject VM with encrypted PVC in MultiWriter sharing mode") + Expect(err.Error()).To(ContainSubstring("MultiWriter disk sharing is not supported for encrypted volumes"), + "expected webhook rejection message about encrypted MultiWriter disks") + }) + + It("should reject a VM whose encrypted PVC is on a physical-sharing SCSI controller", func() { + pvcs := createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: utils.E2EEncryptionStorageClassName, + ControllerType: ptr.To(vmopv1a5.VirtualControllerTypeSCSI), + ControllerBusNumber: ptr.To(int32(1)), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: ptr.To(corev1.PersistentVolumeBlock), + }, 1) + pvcSpec := pvcs[0] + + By("Creating the encrypted PVC before the VM") + Expect(clusterProxy.ApplyWithArgs(ctx, manifestbuilders.GetPersistentVolumeClaimYaml(pvcSpec))). + To(Succeed(), "failed to create encrypted PVC %s", pvcSpec.ClaimName) + DeferCleanup(func() { + _ = svClusterClient.Delete(ctx, &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcSpec.ClaimName, + Namespace: vmSvcNamespace, + }, + }) + }) + + vm := &vmopv1a5.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: vmName, + Namespace: vmSvcNamespace, + }, + Spec: vmopv1a5.VirtualMachineSpec{ + ClassName: clusterResources.VMClassName, + ImageName: linuxImageDisplayName, + StorageClass: clusterResources.StorageClassName, + Hardware: &vmopv1a5.VirtualMachineHardwareSpec{ + SCSIControllers: []vmopv1a5.SCSIControllerSpec{ + { + BusNumber: 1, + Type: vmopv1a5.SCSIControllerTypeParaVirtualSCSI, + SharingMode: vmopv1a5.VirtualControllerSharingModePhysical, + }, + }, + }, + Volumes: []vmopv1a5.VirtualMachineVolume{ + { + Name: pvcSpec.VolumeName, + VirtualMachineVolumeSource: vmopv1a5.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1a5.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcSpec.ClaimName, + }, + }, + }, + ControllerType: vmopv1a5.VirtualControllerTypeSCSI, + ControllerBusNumber: ptr.To(int32(1)), + }, + }, + }, + } + + By("Attempting to create VM with encrypted PVC on physical-sharing controller (should be webhook-rejected)") + err := svClusterClient.Create(ctx, vm) + Expect(err).To(HaveOccurred(), + "expected webhook to reject VM with encrypted PVC on physical-sharing SCSI controller") + Expect(err.Error()).To(ContainSubstring("not supported for encrypted volumes"), + "expected webhook rejection message about encrypted volumes and controller sharing") + }) + + It("should allow a non-encrypted VM with a physical-sharing SCSI controller", func() { + pvcs := createPvcsFromSpec(input, vmName, manifestbuilders.PVC{ + StorageClassName: eztStorageProfileName, + ControllerType: ptr.To(vmopv1a5.VirtualControllerTypeSCSI), + ControllerBusNumber: ptr.To(int32(1)), + UnitNumber: ptr.To(int32(0)), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: ptr.To(corev1.PersistentVolumeBlock), + }, 1) + + hardware := vmopv1a5.VirtualMachineHardwareSpec{ + SCSIControllers: []vmopv1a5.SCSIControllerSpec{ + { + BusNumber: 1, + Type: vmopv1a5.SCSIControllerTypeParaVirtualSCSI, + SharingMode: vmopv1a5.VirtualControllerSharingModePhysical, + }, + }, + } + + vmYaml := manifestbuilders.GetVirtualMachineYamlA5(manifestbuilders.VirtualMachineYaml{ + Namespace: vmSvcNamespace, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PVCs: pvcs, + Hardware: &hardware, + }) + vmYamls = append(vmYamls, vmYaml) + + By("Creating non-encrypted VM with physical-sharing SCSI controller") + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), + "expected webhook to accept VM with physical-sharing controller and non-encrypted storage") + + By("Waiting for the VM to power on with shared PVCs attached") + volumeNames := make([]string, len(pvcs)) + for i, pvc := range pvcs { + volumeNames[i] = pvc.VolumeName + } + + backfilledVolumes := getBackfilledVolumes(ctx, config, svClusterClient, vmSvcNamespace, vmName, allDisksArePVCapabilityEnabled) + volumeNames = append(volumeNames, backfilledVolumes...) + waitForVMAndBatchAttach(ctx, config, svClusterClient, vmSvcNamespace, vmName, volumeNames) + + By("Verifying the physical-sharing controller is reflected in VM status") + verifyCreatedControllersCount(ctx, config, svClusterClient, vmSvcNamespace, vmName, + map[vmopv1a5.VirtualControllerType]int{ + vmopv1a5.VirtualControllerTypeSCSI: 2, + vmopv1a5.VirtualControllerTypeIDE: 2, + }, + ) + }) + }) + }) +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_longevity.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_longevity.go new file mode 100644 index 000000000..26c06f995 --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_longevity.go @@ -0,0 +1,236 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1a2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type VMLongevityInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *e2eConfig.E2EConfig + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + SkipCleanup bool + WCPNamespaceName string +} + +func VMLongevitySpec(ctx context.Context, inputGetter func() VMLongevityInput) { + const ( + specName = "vm-longevity" + vmClassName = "longevity-vm-class" + secondNamespaceName = specName + "-second-namespace" + ) + + var ( + input VMLongevityInput + wcpClient wcp.WorkloadManagementAPI + config *e2eConfig.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterConfig *e2eConfig.ManagementClusterConfig + svClusterClient ctrlclient.Client + clusterResources *e2eConfig.Resources + secondNSContext wcpframework.NamespaceContext + vmYaml []byte + vmName string + classReady metav1.Condition + imageReady metav1.Condition + storageReady metav1.Condition + linuxImageDisplayName string + ) + + BeforeEach(func() { + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", specName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", specName) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + wcpClient = input.WCPClient + config = input.Config + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterConfig = config.InfraConfig.ManagementClusterConfig + clusterResources = svClusterConfig.Resources + svClusterClient = clusterProxy.GetClient() + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + + By("Create a second namespace without VMClass") + + if _, err := wcpClient.GetNamespace(secondNamespaceName); err != nil { + vmsvcSpecs := wcp.NewVMServiceSpecDetails([]string{}, []string{}) + secondNSContext, err = clusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, clusterResources.StorageClassName, clusterResources.WorkerStorageClassName, secondNamespaceName, input.ArtifactFolder) + Expect(err).ToNot(HaveOccurred(), "Failed to create a second test WCP namespace") + wcp.WaitForNamespaceReady(wcpClient, secondNamespaceName) + } + + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, input.ClusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, specName)) + DeferCleanup(cancelPodWatches) + + By("Create VMClass only one time") + + if _, err := wcpClient.GetVMClassInfo(vmClassName); err != nil { + createSpec, err := vmservice.GenerateVMClassSpecFunction("customize", vmClassName, 2, 1024, "description") + Expect(err).NotTo(HaveOccurred(), "failed to create vmclass %s", vmClassName) + Expect(wcpClient.CreateVMClass(createSpec.(wcp.VMClassSpec))).To(Succeed()) + } + + Expect(vmservice.EnsureNamespaceHasAccess(wcpClient, vmClassName, input.WCPNamespaceName)).To(Succeed()) + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, vmClassName, "vmclass") + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, vmName, "vm") + } + + // Delete and verify the virtual machine doesn't exist. + if len(vmYaml) > 0 { + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).To(Succeed(), "failed to delete virtualmachine") + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + } + }) + + When("VM Class is associated with default namespace, not associated with the second namespace", func() { + It("VMClass CR should exist in default namespace and not exist in the second namespace", Label("smoke"), func() { + // Skip testing if WCP_Namespaced_VM_Class FSS is not enabled. + skipper.SkipUnlessNamespacedVMClassFSSEnabled(ctx, svClusterClient, config) + + vmclass, err := vmoperator.GetVMClassInNamespace(ctx, svClusterClient, config, input.WCPNamespaceName, vmClassName) + Expect(vmclass).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred(), "vmclass should exist in namespace where it's associated with") + + vmclass, err = vmoperator.GetVMClassInNamespace(ctx, svClusterClient, config, secondNamespaceName, vmClassName) + Expect(vmclass).To(BeNil()) + Expect(err).To(HaveOccurred(), "vmclass shouldn't exist in namespace where it's not associated with") + }) + }) + + When("VM class is changed after VM successfully deployed", func() { + BeforeEach(func() { + // Associate VMClass to second namespace + Expect(vmservice.EnsureNamespaceHasAccess(wcpClient, vmClassName, secondNamespaceName)).To(Succeed()) + + By("Create VirtualMachine") + + vmName = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: vmClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine", string(vmYaml)) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + }) + + JustBeforeEach(func() { + classReady = metav1.Condition{ + Type: vmopv1a2.VirtualMachineConditionClassReady, + Status: metav1.ConditionTrue, + } + + imageReady = metav1.Condition{ + Type: vmopv1a2.VirtualMachineConditionImageReady, + Status: metav1.ConditionTrue, + } + + storageReady = metav1.Condition{ + Type: vmopv1a2.VirtualMachineConditionStorageReady, + Status: metav1.ConditionTrue, + } + + By("Verify that we have a single VirtualMachine with expected condition") + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, classReady) + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, imageReady) + vmoperator.WaitOnVirtualMachineCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, storageReady) + }) + + AfterEach(func() { + // Delete namespace + clusterProxy.DeleteWCPNamespace(secondNSContext) + }) + + It("VMClass CR should get changed but not impact VM", func() { + // Update VMClass + var ( + vmclass1, vmclass2 *vmopv1a2.VirtualMachineClass + err error + ) + + updatedCpu := 4 + updatedMemory := 2048 + updatedSpec, err := vmservice.GenerateVMClassSpecFunction("customize", vmClassName, updatedCpu, updatedMemory, "customize-description") + Expect(err).NotTo(HaveOccurred(), "failed to generate customized vmclass") + Expect(wcpClient.UpdateVMClass(updatedSpec.(wcp.VMClassSpec))).To(Succeed()) + + By("Updating VMClass in dcli will trigger VMClass CR get updated in both associated namespace") + Eventually(func() bool { + if vmclass1, err = vmoperator.GetVMClassInNamespace(ctx, svClusterClient, config, input.WCPNamespaceName, vmClassName); err != nil { + e2eframework.Logf("failed to get vmclass in namespace %s. err: %v", input.WCPNamespaceName, err) + return false + } + + if vmclass2, err = vmoperator.GetVMClassInNamespace(ctx, svClusterClient, config, secondNamespaceName, vmClassName); err != nil { + e2eframework.Logf("failed to get vmclass in namespace %s. err: %v", secondNamespaceName, err) + return false + } + + cpu_num1 := vmclass1.Spec.Hardware.Cpus + memory_num1 := vmoperator.MemoryQuantityToMb(vmclass1.Spec.Hardware.Memory) + cpu_num2 := vmclass2.Spec.Hardware.Cpus + memory_num2 := vmoperator.MemoryQuantityToMb(vmclass2.Spec.Hardware.Memory) + + return int(cpu_num1) == updatedCpu && int(cpu_num2) == updatedCpu && memory_num1 == updatedMemory && memory_num2 == updatedMemory + }, 30*time.Second, 3*time.Second).Should(BeTrue()) + + By("VirtualMachinePrereqReadyCondition on VMs should not change") + vmoperator.CheckVirtualMachinesConditionConsistent(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, classReady) + vmoperator.CheckVirtualMachinesConditionConsistent(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, imageReady) + vmoperator.CheckVirtualMachinesConditionConsistent(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, storageReady) + + By("Deleting VMClass in dcli will trigger VMClass CR get deleted in both associated namespace") + vmservice.VerifyVMClassDeletion(wcpClient, vmClassName) + + By("VirtualMachineClassReadyCondition must be set to false") + + classReadyFalseCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachineConditionClassReady, + Status: metav1.ConditionFalse, + Reason: "NotFound", + } + vmoperator.WaitOnVirtualMachineConditionUpdate(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, classReadyFalseCondition) + }) + }) +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_multipleclusters.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_multipleclusters.go new file mode 100644 index 000000000..32dd07212 --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_multipleclusters.go @@ -0,0 +1,150 @@ +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type VMMultipleClusterInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *e2eConfig.E2EConfig + ArtifactFolder string + SkipCleanup bool + WCPNamespaceName string +} + +func VMMultipleClusterSpec(ctx context.Context, inputGetter func() VMMultipleClusterInput) { + const ( + specName = "vm-multiple-cluster" + zoneName = "zone-2" + ) + + var ( + input VMMultipleClusterInput + config *e2eConfig.E2EConfig + svClusterConfig *e2eConfig.ManagementClusterConfig + svClusterClient ctrlclient.Client + clusterResources *e2eConfig.Resources + + vmYaml []byte + vmName string + + linuxImageDisplayName string + ) + + BeforeEach(func() { + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", specName) + // Only run this test in the stretched supervisor environment. + skipper.SkipUnlessStretchSupervisorIsEnabled() + + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", specName) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + config = input.Config + svClusterConfig = config.InfraConfig.ManagementClusterConfig + clusterResources = svClusterConfig.Resources + svClusterClient = input.ClusterProxy.GetClient() + vmName = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, input.ClusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, specName)) + DeferCleanup(cancelPodWatches) + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, input.ClusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, vmName, "vm") + } + + // Delete the virtual machine + vmoperator.DeleteVirtualMachine(ctx, svClusterClient, input.WCPNamespaceName, vmName) + // Verify that virtual machine does not exist + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + }) + + Context("When there are multiple clusters", func() { + When("Create a VM with a zone assigned", func() { + It("Should create successfully and be assigned to the correct zone", Label("smoke"), func() { + zoneList, err := utils.ListZonesByNamespace(ctx, input.ClusterProxy.GetClient(), input.WCPNamespaceName) + Expect(err).NotTo(HaveOccurred()) + + var bound bool + + for _, zone := range zoneList.Items { + if zone.Name == zoneName { + bound = true + break + } + } + + if !bound { + // Bind zone-2 to namespace. + zones := []string{zoneName} + _, err := input.ClusterProxy.UpdateNamespaceWithZones(ctx, input.WCPNamespaceName, zones, svClusterClient) + Expect(err).NotTo(HaveOccurred(), "failed to update namespace with Zones") + } + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + Labels: map[string]string{"topology.kubernetes.io/zone": zoneName}, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(input.ClusterProxy.Apply(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine", string(vmYaml)) + + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOn") + vmoperator.WaitForVirtualMachineZone(ctx, config, svClusterClient, input.WCPNamespaceName, zoneName, vmName) + + // Verify that this VM is created in the specific cluster. + vmservice.VerifyVMInZone(ctx, config, svClusterClient, input.WCPNamespaceName, zoneName, vmName) + }) + }) + + When("Create a VM without a zone assigned", func() { + It("Should create successfully and be assigned to a zone", func() { + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "poweredOn", + } + vmYaml = manifestbuilders.GetVirtualMachineYaml(vmParameters) + Expect(input.ClusterProxy.Apply(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine", string(vmYaml)) + + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOn") + vmoperator.WaitForVirtualMachineZone(ctx, config, svClusterClient, input.WCPNamespaceName, "", vmName) + }) + }) + }) +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_networking.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_networking.go new file mode 100644 index 000000000..bc78d90ae --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_networking.go @@ -0,0 +1,220 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "golang.org/x/crypto/ssh" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1a3 "github.com/vmware-tanzu/vm-operator/api/v1alpha3" + + "github.com/vmware-tanzu/vm-operator/test/e2e/appple2e/util" + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + e2essh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type VMNetworkSpecInput struct { + Config *e2eConfig.E2EConfig + ClusterProxy wcpframework.WCPClusterProxyInterface + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + WCPNamespaceName string +} + +func VMNetworkSpec(ctx context.Context, inputGetter func() VMNetworkSpecInput) { + const ( + specName = "vm-networking" + mutableNetworksCap = "supports_VM_service_mutable_networks" + ) + + var ( + input VMNetworkSpecInput + wcpClient wcp.WorkloadManagementAPI + config *e2eConfig.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterClient ctrlclient.Client + clusterResources *e2eConfig.Resources + tmpNamespaceCtx wcpframework.NamespaceContext + vmYaml []byte + vmName string + + isVMMutableNetworksCapEnabled bool + linuxImageDisplayName string + ) + + BeforeEach(func() { + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", specName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", specName) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + svClusterProxy := input.ClusterProxy + wcpClient = input.WCPClient + config = input.Config + clusterResources = config.InfraConfig.ManagementClusterConfig.Resources + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterClient = clusterProxy.GetClient() + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, clusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, specName)) + DeferCleanup(cancelPodWatches) + + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + vmYaml = nil + tmpNamespaceCtx = wcpframework.NamespaceContext{} + vmName = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + + sshCommandRunner, _ := e2essh.NewSSHCommandRunner( + vcenter.GetVCPNIDFromKubeconfigFile(ctx, svClusterProxy.GetKubeconfigPath()), + vcenter.VCSSHPort, testbed.RootUsername, []ssh.AuthMethod{ssh.Password(testbed.RootPassword)}) + isAsyncSvUpgradeEnabled, _ := util.IsFSSEnabled(sshCommandRunner, utils.SupervisorAsyncUpgradeFSS) + isVMMutableNetworksCapEnabled = utils.IsSupervisorCapabilityEnabled(ctx, + svClusterProxy.GetClientSet(), svClusterProxy.GetDynamicClient(), mutableNetworksCap, isAsyncSvUpgradeEnabled) + }) + + AfterEach(func() { + vmNamespaceName := input.WCPNamespaceName + if tmpNamespaceCtx.GetNamespace() != nil { + vmNamespaceName = tmpNamespaceCtx.GetNamespace().Name + } + + if CurrentGinkgoTestDescription().Failed { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), vmNamespaceName, vmName, "vm") + } + + // Delete the virtual machine if it was created. + if len(vmYaml) > 0 { + Expect(clusterProxy.DeleteWithArgs(ctx, vmYaml)).To(Succeed(), "failed to delete virtualmachine") + // Verify that virtual machine does not exist. + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, vmNamespaceName, vmName) + } + + // Delete the temporary namespace if it was created. + if tmpNamespaceCtx.GetNamespace() != nil { + clusterProxy.DeleteWCPNamespace(tmpNamespaceCtx) + wcp.WaitForNamespaceDeleted(wcpClient, tmpNamespaceCtx.GetNamespace().Name) + } + }) + + It("Should allow network interface to be added to VirtualMachine when mutability cap is enabled", Label("smoke"), func() { + if !isVMMutableNetworksCapEnabled { + Skip("VM Mutable Networks capability is not enabled") + } + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + Name: vmName, + ImageName: linuxImageDisplayName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + PowerState: "PoweredOff", + } + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(vmParameters) + Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine:\n %s", string(vmYaml)) + + vmoperator.WaitForVirtualMachineToExist(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmoperator.WaitForVirtualMachineMOID(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + vmMoID := vmoperator.GetVirtualMachineMOID(ctx, svClusterClient, input.WCPNamespaceName, vmName) + vmMoRef := types.ManagedObjectReference{Type: "VirtualMachine", Value: vmMoID} + + vCenterClient := vcenter.NewVimClientFromKubeconfig(ctx, clusterProxy.GetKubeconfigPath()) + propCollector := property.DefaultCollector(vCenterClient) + + var vmMO mo.VirtualMachine + Expect(propCollector.RetrieveOne(ctx, vmMoRef, []string{"config"}, &vmMO)).To(Succeed()) + ethCards := object.VirtualDeviceList(vmMO.Config.Hardware.Device).SelectByType((*types.VirtualEthernetCard)(nil)) + Expect(ethCards).To(HaveLen(1), "VM config should have one EthernetCard") + + By("Add second network interface to VM Spec") + + key := ctrlclient.ObjectKey{Name: vmName, Namespace: input.WCPNamespaceName} + Eventually(func() bool { + vm := &vmopv1a3.VirtualMachine{} + + err := svClusterClient.Get(ctx, key, vm) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + Expect(vm.Spec.Network).ToNot(BeNil()) + Expect(vm.Spec.Network.Interfaces).To(HaveLen(1)) + vm.Spec.Network.Interfaces = append(vm.Spec.Network.Interfaces, vm.Spec.Network.Interfaces[0]) + + vm.Spec.Network.Interfaces[1].Name = "eth1" + + err = svClusterClient.Update(ctx, vm) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + return true + }, config.GetIntervals("default", "wait-virtual-machine-resize")...).Should(BeTrue(), "Timed out updating VirtualMachine %s to add second network interface", vmName) + + By("Wait for VM to be reconfigured with second EthernetCard") + Eventually(func(g Gomega) { + g.Expect(propCollector.RetrieveOne(ctx, vmMoRef, []string{"config"}, &vmMO)).To(Succeed()) + ethCards := object.VirtualDeviceList(vmMO.Config.Hardware.Device).SelectByType((*types.VirtualEthernetCard)(nil)) + g.Expect(ethCards).To(HaveLen(2), "VM should have two EthernetCards configured") + }, config.GetIntervals("default", "wait-virtual-machine-resize")...).Should(Succeed(), "VM reconfigured with second EthernetCard") + + By("Power on VM") + Eventually(func() bool { + vm := &vmopv1a3.VirtualMachine{} + + err := svClusterClient.Get(ctx, key, vm) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + vm.Spec.PowerState = vmopv1a3.VirtualMachinePowerStateOn + + err = svClusterClient.Update(ctx, vm) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + return true + }, config.GetIntervals("default", "wait-virtual-machine-powerstate")...).Should(BeTrue(), "Timed out updating VirtualMachine %s PowerState to On", vmName) + vmoperator.WaitForVirtualMachinePowerState(ctx, config, svClusterClient, input.WCPNamespaceName, vmName, "PoweredOn") + vmoperator.WaitForVirtualMachineIP(ctx, config, svClusterClient, input.WCPNamespaceName, vmName) + + By("Powered On VM should still have two EthernetCards configured") + Expect(propCollector.RetrieveOne(ctx, vmMoRef, []string{"config"}, &vmMO)).To(Succeed()) + ethCards = object.VirtualDeviceList(vmMO.Config.Hardware.Device).SelectByType((*types.VirtualEthernetCard)(nil)) + Expect(ethCards).To(HaveLen(2), "VM config should have two EthernetCards") + }) +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_publishrequest.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_publishrequest.go new file mode 100644 index 000000000..9bf72474d --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_publishrequest.go @@ -0,0 +1,536 @@ +// Copyright (c) 2023-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1a2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + imgregv1a2 "github.com/vmware-tanzu/vm-operator/external/image-registry-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + libssh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/testutils" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +const ( + vmPubSpecName = "vmpub" + vmPubTargetItemName = "vm-publish-request-target-item-name" +) + +type VMPublishRequestSpecInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *e2eConfig.E2EConfig + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + SkipCleanup bool + LinuxVMName string + WCPNamespaceName string +} + +func VMPublishRequestSpec(ctx context.Context, inputGetter func() VMPublishRequestSpecInput) { + Context("VirtualMachinePublishRequest", Ordered, func() { + var ( + input VMPublishRequestSpecInput + wcpClient wcp.WorkloadManagementAPI + config *e2eConfig.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterConfig *e2eConfig.ManagementClusterConfig + svClusterClient ctrlclient.Client + clusterResources *e2eConfig.Resources + vimClient *vim25.Client + + targetContentLibraryName string + vmPublishRequestName string + ) + + BeforeAll(func() { + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", CurrentSpecReport) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", CurrentSpecReport) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", CurrentSpecReport) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", CurrentSpecReport) + Expect(input.LinuxVMName).ToNot(BeEmpty(), "Invalid argument. input.LinuxVMName can't be empty when calling %s spec", CurrentSpecReport) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", CurrentSpecReport) + + wcpClient = input.WCPClient + config = input.Config + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterConfig = config.InfraConfig.ManagementClusterConfig + clusterResources = svClusterConfig.Resources + + // Skip testing if WCP_VM_Image_Registry FSS is not enabled. + svClusterClient = clusterProxy.GetClient() + skipper.SkipUnlessVMImageRegistryFSSEnabled(ctx, svClusterClient, config) + + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, input.ClusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, vmPubSpecName)) + DeferCleanup(cancelPodWatches) + }) + + BeforeEach(func() { + targetContentLibraryName = fmt.Sprintf("%s-%s-%s", vmPubSpecName, "content-library", capiutil.RandomString(4)) + vmPublishRequestName = fmt.Sprintf("%s-%s", vmPubSpecName, capiutil.RandomString(4)) + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, input.LinuxVMName, "vm") + } + }) + + Context("CLS Content Library", Ordered, func() { + var ( + targetLocationCLID string + tarLocationCLIsAttached bool + keepTargetLocationCLAttached bool + ) + + BeforeEach(func() { + if targetLocationCLID == "" { + targetLocationCLID = vmservice.CreateLocalContentLibrary(targetContentLibraryName, wcpClient) + } + }) + + AfterEach(func() { + // Detach the content library from the namespace if attached and not keeping it attached. + if tarLocationCLIsAttached && !keepTargetLocationCLAttached { + Expect(wcpClient.DisassociateImageRegistryContentLibrariesFromNamespace(input.WCPNamespaceName, targetLocationCLID)).To(Succeed(), "failed to detach content library '%s' from namespace '%s'", targetLocationCLID, input.WCPNamespaceName) + + tarLocationCLIsAttached = false + } + + // Delete the content library if exists and not keeping it attached to the namespace. + if targetLocationCLID != "" && !keepTargetLocationCLAttached { + Expect(wcpClient.DeleteLocalContentLibrary(targetLocationCLID)).To(Succeed(), "failed to delete the publish content library, CL ID: %s", targetLocationCLID) + targetLocationCLID = "" + } + + vmoperator.DeleteVirtualMachinePublishRequest(ctx, svClusterClient, input.WCPNamespaceName, vmPublishRequestName) + vmoperator.WaitForVirtualMachinePublishRequestToBeDeleted(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName) + }) + + It("should set default values for source VM name and target item", func() { + vmPubReqBuilder := generateVMPublishRequestBuilder(input.WCPNamespaceName, vmPublishRequestName, "", "", "fake-cl") + createVMPublishRequest(ctx, *config, svClusterClient, *clusterProxy, vmPubReqBuilder) + + Expect(vmoperator.GetVirtualMachinePublishRequestSourceName(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName)).To(Equal(vmPublishRequestName)) + Expect(vmoperator.GetVirtualMachinePublishRequestTargetItemName(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName)).To(Equal(vmPublishRequestName + "-image")) + }) + + It("should have expected condition when the source VM doesn't exist", func() { + // This will attach the content library to the namespace without affecting the existing ones. + // And it will be detached from the namespace in AfterEach(). + Expect(wcpClient.AssociateImageRegistryContentLibrariesToNamespace(input.WCPNamespaceName, wcp.ContentLibrarySpec{ + ContentLibrary: targetLocationCLID, + Writable: true, + })).To(Succeed(), "failed to attach content library '%s' to namespace '%s'", targetLocationCLID, input.WCPNamespaceName) + + tarLocationCLIsAttached = true + + targetLocationK8sCLName, err := vmservice.GetK8sContentLibraryNameByUUID(ctx, config, svClusterClient, input.WCPNamespaceName, targetLocationCLID) + Expect(err).NotTo(HaveOccurred(), "failed to get the CL that is attached to the namespace") + + vmPubReqBuilder := generateVMPublishRequestBuilder(input.WCPNamespaceName, vmPublishRequestName, "fake-vm", vmPubTargetItemName, targetLocationK8sCLName) + createVMPublishRequest(ctx, *config, svClusterClient, *clusterProxy, vmPubReqBuilder) + + vmPubCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachinePublishRequestConditionSourceValid, + Status: metav1.ConditionFalse, + Reason: vmopv1a2.SourceVirtualMachineNotExistReason, + } + vmoperator.VerifyVirtualMachinePublishRequestCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName, vmPubCondition) + }) + + It("should have expected condition when target location content library does not exist", func() { + vmPubReqBuilder := generateVMPublishRequestBuilder(input.WCPNamespaceName, vmPublishRequestName, input.LinuxVMName, vmPubTargetItemName, "non-existing-content-library") + createVMPublishRequest(ctx, *config, svClusterClient, *clusterProxy, vmPubReqBuilder) + + vmPubCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachinePublishRequestConditionTargetValid, + Status: metav1.ConditionFalse, + Reason: vmopv1a2.TargetContentLibraryNotExistReason, + } + vmoperator.VerifyVirtualMachinePublishRequestCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName, vmPubCondition) + }) + + It("should have expected condition when target location content library exists but not writable", func() { + // This will attach the content library to the namespace without affecting the existing ones. + // And it will be detached from the namespace in AfterEach(). + Expect(wcpClient.AssociateImageRegistryContentLibrariesToNamespace(input.WCPNamespaceName, wcp.ContentLibrarySpec{ + ContentLibrary: targetLocationCLID, + Writable: false, + })).To(Succeed(), "failed to attach content library '%s' to namespace '%s'", targetLocationCLID, input.WCPNamespaceName) + + tarLocationCLIsAttached = true + + targetLocationK8sCLName, err := vmservice.GetK8sContentLibraryNameByUUID(ctx, config, svClusterClient, input.WCPNamespaceName, targetLocationCLID) + Expect(err).NotTo(HaveOccurred(), "failed to get the CL that is attached to the namespace") + + vmPubReqBuilder := generateVMPublishRequestBuilder(input.WCPNamespaceName, vmPublishRequestName, input.LinuxVMName, vmPubTargetItemName, targetLocationK8sCLName) + createVMPublishRequest(ctx, *config, svClusterClient, *clusterProxy, vmPubReqBuilder) + + vmPubCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachinePublishRequestConditionTargetValid, + Status: metav1.ConditionFalse, + Reason: vmopv1a2.TargetContentLibraryNotWritableReason, + } + vmoperator.VerifyVirtualMachinePublishRequestCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName, vmPubCondition) + }) + + It("should publish the VM when all conditions meet and successfully deploy from the published VM", Label("smoke"), func() { + Expect(wcpClient.AssociateImageRegistryContentLibrariesToNamespace(input.WCPNamespaceName, wcp.ContentLibrarySpec{ + ContentLibrary: targetLocationCLID, + Writable: true, + })).To(Succeed(), "failed to attach content library '%s' to namespace '%s'", targetLocationCLID, input.WCPNamespaceName) + + tarLocationCLIsAttached = true + + targetLocationK8sCLName, err := vmservice.GetK8sContentLibraryNameByUUID(ctx, config, svClusterClient, input.WCPNamespaceName, targetLocationCLID) + Expect(err).NotTo(HaveOccurred(), "failed to get the CL that is attached to the namespace") + + vmPubReqBuilder := generateVMPublishRequestBuilder(input.WCPNamespaceName, vmPublishRequestName, input.LinuxVMName, vmPubTargetItemName, targetLocationK8sCLName) + createVMPublishRequest(ctx, *config, svClusterClient, *clusterProxy, vmPubReqBuilder) + + vmPubCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachinePublishRequestConditionComplete, + Status: metav1.ConditionTrue, + } + vmoperator.VerifyVirtualMachinePublishRequestCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName, vmPubCondition) + + // Ensure the published image is available with expected display name under the namespace. + expectedPublishedImageCRName, err := vmoperator.GetVirtualMachinePublishRequestTargetItemName(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName) + Expect(err).NotTo(HaveOccurred(), "failed to get the published target item name in namespace %q", input.WCPNamespaceName) + publishedImageCRName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, expectedPublishedImageCRName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VMI name in namespace %q", input.WCPNamespaceName) + Expect(publishedImageCRName).NotTo(BeEmpty(), "published VM Image resource name is empty") + vmoperator.WaitForVirtualMachineImageStatusDisks(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, publishedImageCRName) + + // Keep the published image attached to the namespace for the next test case. + keepTargetLocationCLAttached = true + + // Use the published the vmi to deploy a new VM should succeed with VM powered on and IP assigned. + newVmName := fmt.Sprintf("%s-%s", vmPubSpecName+"-vm", capiutil.RandomString(4)) + newVMBuilder := generateVMBuilder(input.WCPNamespaceName, newVmName, publishedImageCRName, *clusterResources) + newVmYaml := manifestbuilders.GetVirtualMachineYamlA2(newVMBuilder) + Expect(clusterProxy.CreateWithArgs(ctx, newVmYaml)).NotTo(HaveOccurred(), "failed to create virtualmachine from the published image", string(newVmYaml)) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, newVmName) + vmoperator.DeleteVirtualMachine(ctx, svClusterClient, input.WCPNamespaceName, newVmName) + }) + + It("should have expected condition when the published target item already exists in the content library", func() { + Expect(tarLocationCLIsAttached).To(BeTrue(), "target location content library is not attached to the namespace") + + // Reset this before any error occurs below to ensure the CL will be deleted in AfterEach(). + keepTargetLocationCLAttached = false + + targetLocationK8sCLName, err := vmservice.GetK8sContentLibraryNameByUUID(ctx, config, svClusterClient, input.WCPNamespaceName, targetLocationCLID) + Expect(err).NotTo(HaveOccurred(), "failed to get the CL that is attached to the namespace") + + vmPubReqBuilder := generateVMPublishRequestBuilder(input.WCPNamespaceName, vmPublishRequestName, input.LinuxVMName, vmPubTargetItemName, targetLocationK8sCLName) + createVMPublishRequest(ctx, *config, svClusterClient, *clusterProxy, vmPubReqBuilder) + + vmPubCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachinePublishRequestConditionTargetValid, + Status: metav1.ConditionFalse, + Reason: vmopv1a2.TargetItemAlreadyExistsReason, + } + vmoperator.VerifyVirtualMachinePublishRequestCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName, vmPubCondition) + }) + }) + + Context("Inventory Content Library", Ordered, func() { + var ( + inventoryFolderName string + + inventoryFolder *object.Folder + inventoryCL *imgregv1a2.ContentLibrary + + deleteInventoryFolder bool + + user *vcenter.User + nonAdminClient ctrlclient.Client + ) + + BeforeAll(func() { + skipper.SkipUnlessSupervisorCapabilityEnabled(ctx, clusterProxy, consts.InventoryContentLibraryCapabilityName) + + kubeConfig := clusterProxy.GetKubeconfigPath() + svClusterClient = clusterProxy.GetClient() + + var err error + + vCenterHostname := vcenter.GetVCPNIDFromKubeconfig(ctx, kubeConfig) + vimClient, err = vcenter.NewVimClient( + vCenterHostname, + testbed.AdminUsername, + testbed.AdminPassword, + ) + Expect(err).NotTo(HaveOccurred()) + + sshCommandRunner, _, _ := testutils.GetHelpersFromKubeconfig(ctx, kubeConfig) + user, nonAdminClient = setupNonAdminUserForTests(ctx, vimClient, sshCommandRunner, svClusterClient, clusterProxy) + }) + + AfterAll(func() { + By("Deleting non admin user") + vcenter.DeleteUserOrFail(user) + }) + + BeforeEach(func() { + if inventoryFolder == nil { + inventoryFolderName = fmt.Sprintf("%s-%s-%s", vmPubSpecName, "folder", capiutil.RandomString(4)) + + By("Creating library folder") + + finder := find.NewFinder(vimClient, false) + _, inventoryFolder = createLibraryFolder(ctx, finder, inventoryFolderName) + } + + if inventoryCL == nil { + By("Creating an inventory content library", func() { + inventoryCL = createInventoryContentLibraryCR(ctx, nonAdminClient, imgregv1a2.ResourceNamingStrategyPreferItemSourceID, input.WCPNamespaceName, targetContentLibraryName, inventoryFolder.Reference().Value, true, true) + // Validate CL itself exists and reconciled. + validateContentLibraryV2(ctx, nonAdminClient, inventoryCL, inventoryFolder, targetContentLibraryName, input.WCPNamespaceName, "") + }) + } + }) + + AfterEach(func() { + if deleteInventoryFolder && inventoryFolder != nil { + vcenter.DeleteFolder(ctx, inventoryFolder) + + inventoryFolderName = "" + targetContentLibraryName = "" + + inventoryFolder = nil + inventoryCL = nil + } + + vmoperator.DeleteVirtualMachinePublishRequest(ctx, svClusterClient, input.WCPNamespaceName, vmPublishRequestName) + vmoperator.WaitForVirtualMachinePublishRequestToBeDeleted(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName) + }) + + It("should publish the VM to an inventory library when all conditions meet and successfully deploy from the published VM", Label("smoke"), func() { + vmPubReqBuilder := generateVMPublishRequestBuilder(input.WCPNamespaceName, vmPublishRequestName, input.LinuxVMName, vmPubTargetItemName, inventoryCL.Name) + createVMPublishRequest(ctx, *config, svClusterClient, *clusterProxy, vmPubReqBuilder) + + vmPubCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachinePublishRequestConditionComplete, + Status: metav1.ConditionTrue, + } + vmoperator.VerifyVirtualMachinePublishRequestCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName, vmPubCondition) + + // Ensure the published image is available with expected display name under the namespace. + expectedPublishedImageCRName, err := vmoperator.GetVirtualMachinePublishRequestTargetItemName(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName) + Expect(err).NotTo(HaveOccurred(), "failed to get the published target item name in namespace %q", input.WCPNamespaceName) + publishedImageCRName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, expectedPublishedImageCRName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VMI name in namespace %q", input.WCPNamespaceName) + Expect(publishedImageCRName).NotTo(BeEmpty(), "published VM Image resource name is empty") + vmoperator.WaitForVirtualMachineImageStatusDisks(ctx, &config.Config, svClusterClient, input.WCPNamespaceName, publishedImageCRName) + + // Keep the published image attached to the namespace for the next test case. + deleteInventoryFolder = false + + // Use the published the vmi to deploy a new VM should succeed with VM powered on and IP assigned. + newVmName := fmt.Sprintf("%s-%s", vmPubSpecName+"-vm", capiutil.RandomString(4)) + newVMBuilder := generateVMBuilder(input.WCPNamespaceName, newVmName, publishedImageCRName, *clusterResources) + newVmYaml := manifestbuilders.GetVirtualMachineYamlA2(newVMBuilder) + Expect(clusterProxy.CreateWithArgs(ctx, newVmYaml)).NotTo(HaveOccurred(), "failed to create virtualmachine from the published image", string(newVmYaml)) + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, newVmName) + vmoperator.DeleteVirtualMachine(ctx, svClusterClient, input.WCPNamespaceName, newVmName) + }) + + It("should have expected condition when the published target item already exists in inventory", func() { + deleteInventoryFolder = true + + vmPubReqBuilder := generateVMPublishRequestBuilder(input.WCPNamespaceName, vmPublishRequestName, input.LinuxVMName, vmPubTargetItemName, inventoryCL.Name) + createVMPublishRequest(ctx, *config, svClusterClient, *clusterProxy, vmPubReqBuilder) + + vmPubCondition := metav1.Condition{ + Type: vmopv1a2.VirtualMachinePublishRequestConditionTargetValid, + Status: metav1.ConditionFalse, + Reason: vmopv1a2.TargetItemAlreadyExistsReason, + } + vmoperator.VerifyVirtualMachinePublishRequestCondition(ctx, config, svClusterClient, input.WCPNamespaceName, vmPublishRequestName, vmPubCondition) + }) + }) + }) +} + +func generateVMBuilder( + namespace, name, imageName string, + clusterResources e2eConfig.Resources) manifestbuilders.VirtualMachineYaml { + return manifestbuilders.VirtualMachineYaml{ + Namespace: namespace, + Name: name, + ImageName: imageName, + VMClassName: clusterResources.VMClassName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + } +} + +func generateVMPublishRequestBuilder( + namespace, + vmpubName, + sourceName, + targetItemName, + targetLocationName string) manifestbuilders.VirtualMachinePublishRequestYaml { + return manifestbuilders.VirtualMachinePublishRequestYaml{ + Namespace: namespace, + Name: vmpubName, + Source: manifestbuilders.VirtualMachinePublishRequestSource{ + Name: sourceName, + }, + Target: manifestbuilders.VirtualMachinePublishRequestTarget{ + Item: manifestbuilders.VirtualMachinePublishRequestTargetItem{ + Name: targetItemName, // if empty, will be set to default vmPubReqName + "-image" + Description: "Test VM publish request target item description", + }, + Location: manifestbuilders.VirtualMachinePublishRequestTargetLocation{ + Name: targetLocationName, + }, + }, + } +} + +func createVMPublishRequest( + ctx context.Context, + config e2eConfig.E2EConfig, + client ctrlclient.Client, + clusterProxy common.VMServiceClusterProxy, + vmPubReqBuilder manifestbuilders.VirtualMachinePublishRequestYaml) { + vmPubReqYaml := manifestbuilders.GetVirtualMachinePublishRequestYaml(vmPubReqBuilder) + e2eframework.Logf("%v", string(vmPubReqYaml)) + Expect(clusterProxy.CreateWithArgs(ctx, vmPubReqYaml)).NotTo(HaveOccurred(), "failed to create VirtualMachinePublishRequest") + + namespace, name := vmPubReqBuilder.Namespace, vmPubReqBuilder.Name + Eventually(func() bool { + vmPub, err := utils.GetVirtualMachinePublishRequest(ctx, client, namespace, name) + if err != nil { + e2eframework.Logf("retry due to: %v", err) + return false + } + + return vmPub != nil + }, config.GetIntervals("default", "wait-virtual-machine-publish-request-creation")...).Should(Equal(true), "Timed out waiting for VirtualMachinePublishRequest %s to be created", name) +} + +func createLibraryFolder(ctx context.Context, finder *find.Finder, libFolder string) (*object.DatacenterFolders, *object.Folder) { + dc, err := finder.DatacenterList(ctx, "*") + Expect(err).NotTo(HaveOccurred()) + finder.SetDatacenter(dc[0]) + rootFolder, err := dc[0].Folders(ctx) + Expect(err).NotTo(HaveOccurred()) + folderObj, err := rootFolder.VmFolder.CreateFolder(ctx, libFolder) + Expect(err).NotTo(HaveOccurred()) + + return rootFolder, folderObj +} + +func createInventoryContentLibraryCR(ctx context.Context, c ctrlclient.Client, resourceNamingStrategy imgregv1a2.ResourceNamingStrategy, namespace, clName, clID string, allowPublish, allowDelete bool) *imgregv1a2.ContentLibrary { + cl := &imgregv1a2.ContentLibrary{ + ObjectMeta: metav1.ObjectMeta{ + Name: clName, + Namespace: namespace, + }, + Spec: imgregv1a2.ContentLibrarySpec{ + BaseContentLibrarySpec: imgregv1a2.BaseContentLibrarySpec{ + ID: clID, + Type: imgregv1a2.LibraryTypeInventory, + ResourceNamingStrategy: resourceNamingStrategy, + }, + AllowPublish: allowPublish, + AllowDelete: allowDelete, + }, + } + + err := c.Create(ctx, cl) + Expect(err).ToNot(HaveOccurred()) + + return cl +} + +func validateContentLibraryV2(ctx context.Context, svClusterClient ctrlclient.Client, cl *imgregv1a2.ContentLibrary, folder *object.Folder, clName, clNs, clDescription string) { + var foundCL imgregv1a2.ContentLibrary + + Eventually(func(g Gomega) { + g.Expect(svClusterClient.Get(ctx, ctrlclient.ObjectKey{ + Namespace: clNs, + Name: cl.Name, + }, &foundCL)).To(Succeed()) + + g.Expect(foundCL.Spec.ID).To(Equal(folder.Reference().Value)) + g.Expect(foundCL.Spec.AllowImport).To(BeFalse()) + + folderName, err := folder.ObjectName(ctx) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(foundCL.Status.Name).To(Equal(folderName)) + g.Expect(foundCL.ObjectMeta.Name).To(Equal(clName)) + + if clDescription != "" { + g.Expect(foundCL.Status.Description).To(Equal(clDescription)) + } + }).WithTimeout(5 * time.Minute).Should(Succeed()) +} + +func setupNonAdminUserForTests(ctx context.Context, vimClient *vim25.Client, sshCommandRunner libssh.SSHCommandRunner, _ ctrlclient.Client, svClusterProxy *common.VMServiceClusterProxy) (*vcenter.User, ctrlclient.Client) { + By("Creating non-admin user and assign it to SupervisorProviderAdministrators group") + + user, err := vcenter.CreateUserAndAssignToGrp(ctx, vimClient, sshCommandRunner, "gce2e-test-user", "Admin!23Admin", "SupervisorProviderAdministrators") + Expect(err).ToNot(HaveOccurred()) + + svClusterKubeConfig := svClusterProxy.GetKubeconfigPath() + _, _, supervisorClusterIP := testutils.GetHelpersFromKubeconfig(ctx, svClusterKubeConfig) + + By("Logging in as non-admin user in supervisor") + + kubectlPlugin := testutils.LoginWithUserWithRetry(user, supervisorClusterIP, "", "") + + By("Creating a k8s client from the non-admin user kubeconfig") + + restCfg, err := clientcmd.BuildConfigFromFlags("", kubectlPlugin.KubeconfigPath()) + Expect(err).NotTo(HaveOccurred()) + + nonAdminClient, err := ctrlclient.New(restCfg, ctrlclient.Options{Scheme: svClusterProxy.GetScheme()}) + Expect(err).NotTo(HaveOccurred()) + + By("Checking if non-admin user kubeconfig is able to do basic operations") + + pods := &corev1.PodList{} + err = nonAdminClient.List(ctx, pods) + Expect(err).NotTo(HaveOccurred()) + + return user, nonAdminClient +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_snapshot.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_snapshot.go new file mode 100644 index 000000000..ece315b88 --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_snapshot.go @@ -0,0 +1,451 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +const ( + vmSnapshotSpecName = "v1alpha5-vmsnapshot" +) + +type VMSnapshotSpecInput struct { + Config *e2eConfig.E2EConfig + ClusterProxy wcpframework.WCPClusterProxyInterface + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + WCPNamespaceName string +} + +func VMSnapshotSpec(ctx context.Context, inputGetter func() VMSnapshotSpecInput) { + var ( + vmSnapshotInput VMSnapshotSpecInput + vmSvcClusterProxy *common.VMServiceClusterProxy + vmSvcE2EConfig *e2eConfig.E2EConfig + svClusterClient ctrlclient.Client + vmSvcClusterResources *e2eConfig.Resources + vmSvcNamespace string + skipCleanup bool + + randomString string + vmName string + vmSnapshot1Name string + vmSnapshot2Name string + vmSnapshot3Name string + ) + + vmSnapshotReference := func(name ...string) *vmopv1a5.VirtualMachineSnapshotReference { + res := &vmopv1a5.VirtualMachineSnapshotReference{ + Type: vmopv1a5.VirtualMachineSnapshotReferenceTypeUnmanaged, + } + if len(name) > 0 { + res = &vmopv1a5.VirtualMachineSnapshotReference{ + Type: vmopv1a5.VirtualMachineSnapshotReferenceTypeManaged, + Name: name[0], + } + } + + return res + } + + skipChecks := func() { + skipper.SkipUnlessInfraIs(vmSnapshotInput.Config.InfraConfig.InfraName, consts.WCP) + skipper.SkipUnlessSupervisorCapabilityEnabled(ctx, vmSvcClusterProxy, consts.VirtualMachineSnapshotCapabilityName) + skipper.SkipUnlessSnapshotFSSEnabled(ctx, vmSvcClusterProxy, vmSvcE2EConfig) + + skipCleanup = false + } + + BeforeEach(func() { + vmSnapshotInput = inputGetter() + Expect(vmSnapshotInput.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", vmSnapshotSpecName) + Expect(vmSnapshotInput.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", vmSnapshotSpecName) + Expect(vmSnapshotInput.Config.InfraConfig.ManagementClusterConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig.ManagementClusterConfig can't be nil when calling %s spec", vmSnapshotSpecName) + Expect(vmSnapshotInput.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", vmSnapshotSpecName) + Expect(vmSnapshotInput.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", vmSnapshotSpecName) + Expect(os.MkdirAll(vmSnapshotInput.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", vmSnapshotSpecName) + + vmSvcClusterProxy = vmSnapshotInput.ClusterProxy.(*common.VMServiceClusterProxy) + vmSvcE2EConfig = vmSnapshotInput.Config + vmSvcClusterResources = vmSvcE2EConfig.InfraConfig.ManagementClusterConfig.Resources + vmSvcNamespace = vmSnapshotInput.WCPNamespaceName + svClusterClient = vmSvcClusterProxy.GetClient() + skipCleanup = true + + skipChecks() + + vmSnapshotCancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces( + ctx, + []string{vmSvcE2EConfig.GetVariable("VMOPNamespace")}, + vmSvcClusterProxy.GetClientSet(), + filepath.Join(vmSnapshotInput.ArtifactFolder, vmSnapshotSpecName)) + DeferCleanup(vmSnapshotCancelPodWatches) + + randomString = capiutil.RandomString(4) + vmName = "vm-" + randomString + vmSnapshot1Name = "sn-1-" + randomString + vmSnapshot2Name = "sn-2-" + randomString + vmSnapshot3Name = "sn-3-" + randomString + }) + + AfterEach(func() { + if skipCleanup { + return + } + + vmoperator.VerifyVMDeleted(ctx, svClusterClient, vmSvcE2EConfig, vmSvcNamespace, vmName) + vmoperator.EnsureVMSnapshotDeleted(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot1Name, + }) + vmoperator.EnsureVMSnapshotDeleted(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot2Name, + }) + vmoperator.EnsureVMSnapshotDeleted(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot3Name, + }) + }) + + When("VM doesn't have PVC", func() { + BeforeEach(func() { + vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcClusterResources, vmSvcNamespace, vmName, "", nil) + vmoperator.WaitForVirtualMachineConditionCreated(ctx, vmSvcE2EConfig, svClusterClient, vmSvcNamespace, vmName) + vmoperator.WaitForVirtualMachinePowerState(ctx, vmSvcE2EConfig, svClusterClient, vmSvcNamespace, vmName, "PoweredOn") + }) + + It("successfully create, update, revert to, and delete vm snapshots", Label("smoke"), func() { + By("create vm snapshot 1") + vmservice.CreateVMSnapshot(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot1Name, + VMName: vmName, + Memory: true, + Quiesce: "10m", + }) + + By("Verifying Snapshot's status, power should be on since snapshot's spec.memory is true") + vmoperator.VerifyVirtualMachineSnapshotCondition(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmSnapshot1Name, + vmopv1a5.VirtualMachinePowerStateOn, + true, + []vmopv1a5.VirtualMachineSnapshotReference{}) + + By("Verifying VM's Snapshot related status") + vmoperator.VerifySnapshotStatusOnVirtualMachine(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmName, + vmSnapshotReference(vmSnapshot1Name), + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot1Name)}, + vmopv1a5.VirtualMachinePowerStateOn) + + By("update vm snapshot 1") + vmservice.UpdateVMSnapshot(ctx, vmSvcClusterProxy, vmSvcE2EConfig, + manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot1Name, + VMName: vmName, + Memory: true, + Quiesce: "10m", + Description: "updated snapshot 1", + }) + + By("create a new vm snapshot 2") + vmservice.CreateVMSnapshot(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot2Name, + VMName: vmName, + }) + + By("Verifying Snapshot 2's status, power should be off since memory is false") + vmoperator.VerifyVirtualMachineSnapshotCondition(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmSnapshot2Name, + vmopv1a5.VirtualMachinePowerStateOff, + false, + []vmopv1a5.VirtualMachineSnapshotReference{}) + + By("Verifying Snapshot 1's status, it should have snapshot 2 as child") + vmoperator.VerifyVirtualMachineSnapshotCondition(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmSnapshot1Name, + vmopv1a5.VirtualMachinePowerStateOn, + true, + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot2Name)}) + + By("Verifying VM's Snapshot related status") + vmoperator.VerifySnapshotStatusOnVirtualMachine(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmName, + vmSnapshotReference(vmSnapshot2Name), + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot1Name)}, + vmopv1a5.VirtualMachinePowerStateOn) + + By("create a new vm snapshot 3") + vmservice.CreateVMSnapshot(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot3Name, + VMName: vmName, + }) + + By("Verifying Snapshot 3's status, power should be off since memory is false") + vmoperator.VerifyVirtualMachineSnapshotCondition(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmSnapshot3Name, + vmopv1a5.VirtualMachinePowerStateOff, + false, + []vmopv1a5.VirtualMachineSnapshotReference{}) + + By("Verifying Snapshot 2's status, it should have snapshot 3 as child") + vmoperator.VerifyVirtualMachineSnapshotCondition(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmSnapshot2Name, + vmopv1a5.VirtualMachinePowerStateOff, + false, + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot3Name)}) + + By("Verifying VM's Snapshot related status") + vmoperator.VerifySnapshotStatusOnVirtualMachine(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmName, + vmSnapshotReference(vmSnapshot3Name), + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot1Name)}, + vmopv1a5.VirtualMachinePowerStateOn) + + By("Revert to vm snapshot 2") + vmservice.RevertVMSnapshot(ctx, vmSvcClusterProxy, vmSvcE2EConfig, + vmName, vmSvcNamespace, vmSnapshotReference(vmSnapshot2Name)) + + By("Verifying VM's Snapshot related status, VM should be powered off") + vmoperator.VerifySnapshotStatusOnVirtualMachine(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmName, + vmSnapshotReference(vmSnapshot2Name), + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot1Name)}, + vmopv1a5.VirtualMachinePowerStateOff) + + By("Revert to vm snapshot 1") + vmservice.RevertVMSnapshot(ctx, + vmSvcClusterProxy, vmSvcE2EConfig, vmName, vmSvcNamespace, + vmSnapshotReference(vmSnapshot1Name)) + + By("Verifying VM's Snapshot related status, VM should be powered on") + vmoperator.VerifySnapshotStatusOnVirtualMachine(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmName, + vmSnapshotReference(vmSnapshot1Name), + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot1Name)}, + vmopv1a5.VirtualMachinePowerStateOn) + + By("Verifying Snapshot's quota usage") + vmoperator.VerifyVMSnapshotQuotaUsage(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, vmSvcNamespace, + vmSvcClusterResources.StorageClassName+"-vmsnapshot-usage", + vmSnapshot1Name, vmSnapshot2Name, vmSnapshot3Name) + + By("Revert to current snapshot: vm snapshot 1") + vmservice.RevertVMSnapshot(ctx, + vmSvcClusterProxy, vmSvcE2EConfig, vmName, vmSvcNamespace, + vmSnapshotReference(vmSnapshot1Name)) + + By("Verifying VM's Snapshot related status, VM should be powered on") + vmoperator.VerifySnapshotStatusOnVirtualMachine(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmName, + vmSnapshotReference(vmSnapshot1Name), + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot1Name)}, + vmopv1a5.VirtualMachinePowerStateOn) + + By("Delete vm snapshot 2") + vmoperator.EnsureVMSnapshotDeleted(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot2Name, + }) + + By("Verifying Snapshot 1's status, it should not have snapshot 2 as child") + vmoperator.VerifyVirtualMachineSnapshotCondition(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmSnapshot1Name, + vmopv1a5.VirtualMachinePowerStateOn, + true, + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot3Name)}) + + By("Delete vm") + vmoperator.VerifyVMDeleted(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, vmSvcNamespace, vmName) + + By("Verifying Snapshot 1 and Snapshot 3 are gone because of garbage collection") + vmoperator.VerifyVMSnapshotDeletion(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot1Name, + }) + vmoperator.VerifyVMSnapshotDeletion(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot3Name, + }) + + By("Verifying Snapshot's quota usage") + vmoperator.VerifyVMSnapshotQuotaUsage(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, vmSvcNamespace, vmSvcClusterResources.StorageClassName+"-vmsnapshot-usage") + }) + + It("successfully show unmanaged vm snapshot", func() { + By("create a snapshot in vsphere") + vmservice.CreateSnapshotInVC(ctx, vmSvcClusterProxy, vmSnapshot1Name, vmName, vmSvcNamespace) + + By("Verifying VM's Snapshot related status, it should show the unmanaged snapshot reference") + vmoperator.VerifySnapshotStatusOnVirtualMachine(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmName, + vmSnapshotReference(), + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference()}, + vmopv1a5.VirtualMachinePowerStateOn) + }) + + It("successfully revert to imported Snapshot", func() { + By("create imported vm snapshot 1 in vsphere") + vmservice.CreateSnapshotInVC(ctx, vmSvcClusterProxy, vmSnapshot1Name, vmName, vmSvcNamespace) + + By("Create the snapshot CR with same name as the imported snapshot, with imported snapshot annotation") + vmservice.CreateVMSnapshot(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot1Name, + VMName: vmName, + ImportedSnapshot: true, + }) + + By("Verifying VM's Snapshot related status") + vmoperator.VerifySnapshotStatusOnVirtualMachine(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmName, + vmSnapshotReference(vmSnapshot1Name), + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot1Name)}, + vmopv1a5.VirtualMachinePowerStateOn) + + By("Set restart mode to hard") + vmservice.UpdateVMRestartMode(ctx, vmSvcClusterProxy, vmSvcE2EConfig, + vmName, vmSvcNamespace, vmopv1a5.VirtualMachinePowerOpModeHard) + + By("Revert to imported snapshot") + vmservice.RevertVMSnapshot(ctx, vmSvcClusterProxy, vmSvcE2EConfig, + vmName, vmSvcNamespace, vmSnapshotReference(vmSnapshot1Name)) + + By("Verifying VM's restart mode is updated") + vmoperator.VerifyVirtualMachineRestartMode( + ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmName, + vmopv1a5.VirtualMachinePowerOpModeHard) + }) + }) + + When("VM has PVC", func() { + BeforeEach(func() { + vmservice.DeployVMWithCloudInit(ctx, vmSvcClusterProxy, vmSvcClusterResources, vmSvcNamespace, vmName, "", []manifestbuilders.PVC{ + { + VolumeName: "test-vol" + randomString, + ClaimName: "test-claim" + randomString, + StorageClassName: vmSvcClusterResources.StorageClassName, + RequestSize: "1Gi", + Namespace: vmSvcNamespace, + }, + }) + vmoperator.WaitForVirtualMachineConditionCreated(ctx, vmSvcE2EConfig, svClusterClient, vmSvcNamespace, vmName) + vmoperator.WaitForVirtualMachinePowerState(ctx, vmSvcE2EConfig, svClusterClient, vmSvcNamespace, vmName, "PoweredOn") + }) + + It("successfully create, and delete vm snapshots", func() { + By("create vm snapshot 1") + vmservice.CreateVMSnapshot(ctx, vmSvcClusterProxy, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot1Name, + VMName: vmName, + }) + + By("Verifying Snapshot's status") + vmoperator.VerifyVirtualMachineSnapshotCondition(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmSnapshot1Name, + vmopv1a5.VirtualMachinePowerStateOff, + false, + []vmopv1a5.VirtualMachineSnapshotReference{}) + + By("Verifying VM's Snapshot related status") + vmoperator.VerifySnapshotStatusOnVirtualMachine(ctx, vmSvcE2EConfig, + vmSvcClusterProxy.GetClient(), + vmSvcNamespace, + vmName, + vmSnapshotReference(vmSnapshot1Name), + []vmopv1a5.VirtualMachineSnapshotReference{*vmSnapshotReference(vmSnapshot1Name)}, + vmopv1a5.VirtualMachinePowerStateOn) + + By("Verifying Snapshot's quota usage") + vmoperator.VerifyVMSnapshotQuotaUsage(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, vmSvcNamespace, + vmSvcClusterResources.StorageClassName+"-vmsnapshot-usage", + vmSnapshot1Name) + + By("Delete vm snapshot 1") + vmoperator.EnsureVMSnapshotDeleted(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot1Name, + }) + + By("Verifying Snapshot 1 is deleted") + vmoperator.VerifyVMSnapshotDeletion(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, manifestbuilders.VirtualMachineSnapshotYaml{ + Namespace: vmSvcNamespace, + Name: vmSnapshot1Name, + }) + vmoperator.VerifyVMSnapshotQuotaUsage(ctx, vmSvcClusterProxy.GetClient(), + vmSvcE2EConfig, vmSvcNamespace, + vmSvcClusterResources.StorageClassName+"-vmsnapshot-usage") + }) + }) +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_vpcnetworking.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_vpcnetworking.go new file mode 100644 index 000000000..17ad8398c --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_vpcnetworking.go @@ -0,0 +1,479 @@ +// Copyright (c) 2024-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +type VMVPCSpecInput struct { + Config *e2eConfig.E2EConfig + ClusterProxy wcpframework.WCPClusterProxyInterface + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + WCPNamespaceName string +} + +const ( + vpcAPIVersion = "crd.nsx.vmware.com/v1alpha1" + subnetKind = "Subnet" + subnetSetKind = "SubnetSet" + subnetDHCPName = "vmsvc-subnet-dhcp" + subnetSetDHCPName = "vmsvc-subnetset-dhcp" + subnetCIDRName = "vmsvc-subnet-cidr" + subnetName = "vmsvc-subnet-test-communication" + subnetSetCIDRName = "vmsvc-subnetset-cidr" + dhcpConfig = "DHCP" + cidrConfig = "CIDR" + nic1Name = "subnet-dhcp-nic1" + nic2Name = "subnetset-cidr-nic2" +) + +func createSubnetOrSubnetSet(ctx context.Context, g Gomega, clusterProxy *common.VMServiceClusterProxy, kind, name, ns, networkConfigType string, private bool) { + GinkgoHelper() + + sYaml := utils.CreateSubnetOrSubnetSetYaml(kind, name, ns, networkConfigType, private) + g.Expect(clusterProxy.CreateWithArgs(ctx, sYaml)).To(Succeed(), "failed to create the %s Subnet or SubnetSet: %s", dhcpConfig, string(sYaml)) +} + +func createSecurityPolicy(ctx context.Context, g Gomega, clusterProxy *common.VMServiceClusterProxy, name, ns string) { + GinkgoHelper() + + sYaml := utils.CreateSecurityPolicyYaml(name, ns) + g.Expect(clusterProxy.CreateWithArgs(ctx, sYaml)).To(Succeed(), "failed to create SecurityPolicy: %s", string(sYaml)) +} + +func createSecret(ctx context.Context, g Gomega, clusterProxy *common.VMServiceClusterProxy, ns, secretName string) { + GinkgoHelper() + // Create and apply Secret yaml. + secret := manifestbuilders.Secret{ + Namespace: ns, + Name: secretName, + } + secretYaml := manifestbuilders.GetSecretYamlCloudConfig(secret) + g.Expect(clusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create secret: %s", string(secretYaml)) +} + +func createVM(ctx context.Context, g Gomega, clusterProxy *common.VMServiceClusterProxy, vmYaml []byte) { + GinkgoHelper() + g.Expect(clusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create virtualmachine: %s", string(vmYaml)) +} + +// Delete VM before deleting Subnet or SubnetSet. +func deleteVMAndSubnet(ctx context.Context, config *e2eConfig.E2EConfig, svClusterClient ctrlclient.Client, namespace, vmName, subnetName, subnetKind string) { + vmoperator.DeleteVirtualMachine(ctx, svClusterClient, namespace, vmName) + vmoperator.WaitForVirtualMachineToBeDeleted(ctx, config, svClusterClient, namespace, vmName) + vmoperator.DeleteSubnetOrSubnetSet(ctx, svClusterClient, namespace, subnetName, subnetKind) + vmoperator.WaitForSubnetOrSubnetSetToBeDeleted(ctx, config, svClusterClient, namespace, subnetName, subnetKind) +} + +func VMVPCSpec(ctx context.Context, inputGetter func() VMVPCSpecInput) { + const ( + specName = "vm-vpc-networking" + ) + + var ( + input VMVPCSpecInput + config *e2eConfig.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterConfig *e2eConfig.ManagementClusterConfig + svClusterClient ctrlclient.Client + wcpClient wcp.WorkloadManagementAPI + clusterResources *e2eConfig.Resources + v1a2vmParameters manifestbuilders.VirtualMachineYaml + vm1Name string + vm2Name string + vm1IP string + vm2IP string + secretName string + vmYaml []byte + linuxImageDisplayName string + ) + + BeforeEach(func() { + input = inputGetter() + config = input.Config + Expect(config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", specName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterClient = clusterProxy.GetClient() + wcpClient = input.WCPClient + // This test is specific for networking VPC + skipper.SkipUnlessNetworkingIsVPC(ctx, svClusterClient, config) + // Skip if WCP_VMService_v1alpha2 FSS not enabled + skipper.SkipUnlessV1a2FSSEnabled(ctx, svClusterClient, config) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + svClusterConfig = config.InfraConfig.ManagementClusterConfig + clusterResources = svClusterConfig.Resources + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, clusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, specName)) + DeferCleanup(cancelPodWatches) + + linuxImageDisplayName = vmservice.GetDefaultImageDisplayName(clusterResources) + vm1Name = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + vm2Name = fmt.Sprintf("%s-%s", specName, capiutil.RandomString(4)) + vm1IP = "" + vm2IP = "" + secretName = fmt.Sprintf("%s-%s", "secret", capiutil.RandomString(4)) + + Eventually(func(g Gomega) { + createSecret(ctx, g, clusterProxy, input.WCPNamespaceName, secretName) + }, config.GetIntervals("default", "wait-secret-creation")...).Should(Succeed(), "Timed out in creating the Secret") + vmservice.VerifySecretCreation(ctx, config, svClusterClient, input.WCPNamespaceName, secretName) + + v1a2vmParameters = manifestbuilders.VirtualMachineYaml{ + Namespace: input.WCPNamespaceName, + VMClassName: clusterResources.VMClassName, + ImageName: linuxImageDisplayName, + StorageClassName: clusterResources.StorageClassName, + ResourcePolicy: clusterResources.VMResourcePolicyName, + PowerState: "PoweredOn", + Bootstrap: manifestbuilders.Bootstrap{ + CloudInit: &manifestbuilders.CloudInit{ + RawCloudConfig: &manifestbuilders.KeySelector{ + Key: "user-data", + Name: secretName, + }, + }, + }, + } + }) + + // Describe the VMs if the test failed before they are deleted. + JustAfterEach(func() { + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, vm1Name, "vm") + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, vm2Name, "vm") + } + }) + + AfterEach(func() { + }) + + Context("VPC DHCP should successfully create VMs", func() { + AfterEach(func() { + if vm1IP != "" { + deleteVMAndSubnet(ctx, config, svClusterClient, input.WCPNamespaceName, vm1Name, subnetDHCPName, subnetKind) + } + + if vm2IP != "" { + deleteVMAndSubnet(ctx, config, svClusterClient, input.WCPNamespaceName, vm2Name, subnetSetDHCPName, subnetSetKind) + } + }) + + It("using customized DHCP Subnet/SubnetSet to assign valid ip addresses and ping each other", Label("smoke"), func() { + By("Creating VM1 using DHCP Private Subnet") + Eventually(func(g Gomega) { + createSubnetOrSubnetSet(ctx, g, clusterProxy, subnetKind, subnetDHCPName, input.WCPNamespaceName, dhcpConfig, true) + }, config.GetIntervals("default", "wait-subnet-creation")...).Should(Succeed(), "Timed out in creating Subnet") + vmservice.VerifySubnetOrSubnetSetCreation(ctx, config, svClusterClient, input.WCPNamespaceName, subnetDHCPName, subnetKind) + + v1a2vmParameters.Name = vm1Name + v1a2vmParameters.NetworkA2 = manifestbuilders.NetworkA2{ + Interfaces: []manifestbuilders.InterfaceSpec{ + { + Name: subnetDHCPName, + Kind: subnetKind, + APIVersion: vpcAPIVersion, + }, + }, + } + // Create v1alpha2 VM deployment yaml + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(v1a2vmParameters) + + Eventually(func(g Gomega) { + createVM(ctx, g, clusterProxy, vmYaml) + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out in creating the VirtualMachine") + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vm1Name) + vm1IP = vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vm1Name) + + By("Creating VM2 using DHCP Private SubnetSet") + Eventually(func(g Gomega) { + createSubnetOrSubnetSet(ctx, g, clusterProxy, subnetSetKind, subnetSetDHCPName, input.WCPNamespaceName, dhcpConfig, true) + }, config.GetIntervals("default", "wait-subnet-creation")...).Should(Succeed(), "Timed out in creating SubnetSet") + vmservice.VerifySubnetOrSubnetSetCreation(ctx, config, svClusterClient, input.WCPNamespaceName, subnetSetDHCPName, subnetSetKind) + + v1a2vmParameters.Name = vm2Name + v1a2vmParameters.NetworkA2 = manifestbuilders.NetworkA2{ + Interfaces: []manifestbuilders.InterfaceSpec{ + { + Name: subnetSetDHCPName, + Kind: subnetSetKind, + APIVersion: vpcAPIVersion, + }, + }, + } + // Create v1alpha2 VM deployment yaml + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(v1a2vmParameters) + + Eventually(func(g Gomega) { + createVM(ctx, g, clusterProxy, vmYaml) + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out in creating the VirtualMachine") + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vm2Name) + vm2IP = vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vm2Name) + + By("In the same ns, two VMs on independent Private Subnet/SubnetSets should be able to communicate with each other") + verifyLoginAndPingVM(ctx, config, clusterProxy, svClusterClient, input.WCPNamespaceName, vm1IP, vm2IP) + }) + }) + + Context("VPC CIDR should successfully create VM", func() { + AfterEach(func() { + if vm1IP != "" { + deleteVMAndSubnet(ctx, config, svClusterClient, input.WCPNamespaceName, vm1Name, subnetCIDRName, subnetKind) + } + + if vm2IP != "" { + deleteVMAndSubnet(ctx, config, svClusterClient, input.WCPNamespaceName, vm2Name, subnetSetCIDRName, subnetSetKind) + } + }) + + It("using customized CIDR Subnet to assign valid ip address", func() { + By("Creating VM1 using CIDR Private Subnet") + Eventually(func(g Gomega) { + createSubnetOrSubnetSet(ctx, g, clusterProxy, subnetKind, subnetCIDRName, input.WCPNamespaceName, cidrConfig, true) + }, config.GetIntervals("default", "wait-subnet-creation")...).Should(Succeed(), "Timed out in creating Subnet") + vmservice.VerifySubnetOrSubnetSetCreation(ctx, config, svClusterClient, input.WCPNamespaceName, subnetCIDRName, subnetKind) + + v1a2vmParameters.Name = vm1Name + v1a2vmParameters.NetworkA2 = manifestbuilders.NetworkA2{ + Interfaces: []manifestbuilders.InterfaceSpec{ + { + Name: subnetCIDRName, + Kind: subnetKind, + APIVersion: vpcAPIVersion, + }, + }, + } + // Create v1alpha2 VM deployment yaml + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(v1a2vmParameters) + + Eventually(func(g Gomega) { + createVM(ctx, g, clusterProxy, vmYaml) + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out in creating the VirtualMachine") + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vm1Name) + vm1IP = vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vm1Name) + + By("Creating VM2 using CIDR Public SubnetSet") + Eventually(func(g Gomega) { + createSubnetOrSubnetSet(ctx, g, clusterProxy, subnetSetKind, subnetSetCIDRName, input.WCPNamespaceName, cidrConfig, false) + }, config.GetIntervals("default", "wait-subnet-creation")...).Should(Succeed(), "Timed out in creating SubnetSet") + vmservice.VerifySubnetOrSubnetSetCreation(ctx, config, svClusterClient, input.WCPNamespaceName, subnetSetCIDRName, subnetSetKind) + + v1a2vmParameters.Name = vm2Name + v1a2vmParameters.NetworkA2 = manifestbuilders.NetworkA2{ + Interfaces: []manifestbuilders.InterfaceSpec{ + { + Name: subnetSetCIDRName, + Kind: subnetSetKind, + APIVersion: vpcAPIVersion, + }, + }, + } + // Create v1alpha2 VM deployment yaml + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(v1a2vmParameters) + + Eventually(func(g Gomega) { + createVM(ctx, g, clusterProxy, vmYaml) + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out in creating the VirtualMachine") + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vm2Name) + vm2IP = vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vm2Name) + + By("In the same ns, two VMs on Private and Public Subnet/SubnetSets should be able to communicate with each other") + verifyLoginAndPingVM(ctx, config, clusterProxy, svClusterClient, input.WCPNamespaceName, vm1IP, vm2IP) + }) + }) + + Context("VPC supports multiple NICs", func() { + AfterEach(func() { + if vm1IP != "" { + deleteVMAndSubnet(ctx, config, svClusterClient, input.WCPNamespaceName, vm1Name, nic1Name, subnetKind) + } + // Delete second SubnetSet + vmoperator.DeleteSubnetOrSubnetSet(ctx, svClusterClient, input.WCPNamespaceName, nic2Name, subnetSetKind) + vmoperator.WaitForSubnetOrSubnetSetToBeDeleted(ctx, config, svClusterClient, input.WCPNamespaceName, nic2Name, subnetSetKind) + }) + + It("VM deployment should succeed with 2 NICs", func() { + By("Creating VM with 2 NICs") + Eventually(func(g Gomega) { + createSubnetOrSubnetSet(ctx, g, clusterProxy, subnetKind, nic1Name, input.WCPNamespaceName, dhcpConfig, true) + }, config.GetIntervals("default", "wait-subnet-creation")...).Should(Succeed(), "Timed out in creating Subnet") + vmservice.VerifySubnetOrSubnetSetCreation(ctx, config, svClusterClient, input.WCPNamespaceName, nic1Name, subnetKind) + + Eventually(func(g Gomega) { + createSubnetOrSubnetSet(ctx, g, clusterProxy, subnetSetKind, nic2Name, input.WCPNamespaceName, cidrConfig, true) + }, config.GetIntervals("default", "wait-subnet-creation")...).Should(Succeed(), "Timed out in creating SubnetSet") + vmservice.VerifySubnetOrSubnetSetCreation(ctx, config, svClusterClient, input.WCPNamespaceName, nic2Name, subnetSetKind) + + v1a2vmParameters.Name = vm1Name + v1a2vmParameters.NetworkA2 = manifestbuilders.NetworkA2{ + Interfaces: []manifestbuilders.InterfaceSpec{ + { + Name: nic1Name, + Kind: subnetKind, + APIVersion: vpcAPIVersion, + }, + { + Name: nic2Name, + Kind: subnetSetKind, + APIVersion: vpcAPIVersion, + }, + }, + } + // Create v1alpha2 VM deployment yaml + vmYaml = manifestbuilders.GetVirtualMachineWithMultiNetworkYamlA2(v1a2vmParameters) + + Eventually(func(g Gomega) { + createVM(ctx, g, clusterProxy, vmYaml) + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out in creating the VirtualMachine") + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vm1Name) + vm1IP = vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vm1Name) + }) + }) + + Context("Across namespaces, VPC Public and Private accessMode", func() { + var ( + secondNamespaceName string + secondNamespaceCtx wcpframework.NamespaceContext + ) + + AfterEach(func() { + if vm1IP != "" { + deleteVMAndSubnet(ctx, config, svClusterClient, input.WCPNamespaceName, vm1Name, subnetName, subnetKind) + } + + if vm2IP != "" { + deleteVMAndSubnet(ctx, config, svClusterClient, secondNamespaceName, vm2Name, subnetName, subnetKind) + } + // Delete the second namespace if it was created. + if secondNamespaceCtx.GetNamespace() != nil { + clusterProxy.DeleteWCPNamespace(secondNamespaceCtx) + } + }) + + It("one VirtualMachine within Private Subnet can communicate to another VM in another ns within Public Subnet", func() { + By("Create VM1 using Private Subnet") + Eventually(func(g Gomega) { + createSubnetOrSubnetSet(ctx, g, clusterProxy, subnetKind, subnetName, input.WCPNamespaceName, cidrConfig, true) + }, config.GetIntervals("default", "wait-subnet-creation")...).Should(Succeed(), "Timed out in creating Subnet") + vmservice.VerifySubnetOrSubnetSetCreation(ctx, config, svClusterClient, input.WCPNamespaceName, subnetName, subnetKind) + + v1a2vmParameters.Name = vm1Name + v1a2vmParameters.NetworkA2 = manifestbuilders.NetworkA2{ + Interfaces: []manifestbuilders.InterfaceSpec{ + { + Name: subnetName, + Kind: subnetKind, + APIVersion: vpcAPIVersion, + }, + }, + } + // Create v1alpha2 VM deployment yaml + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(v1a2vmParameters) + + Eventually(func(g Gomega) { + createVM(ctx, g, clusterProxy, vmYaml) + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out in creating the VirtualMachine") + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, input.WCPNamespaceName, vm1Name) + vm1IP = vmoperator.GetVirtualMachineIP(ctx, svClusterClient, input.WCPNamespaceName, vm1Name) + + By("Create a second namespace") + + secondNamespaceName = fmt.Sprintf("%s-second", input.WCPNamespaceName) + clID := vmservice.GetContentLibraryUUIDByName(consts.VMServiceCLName, wcpClient) + vmsvcSpecs := wcp.NewVMServiceSpecDetails([]string{clusterResources.VMClassName}, []string{clID}) + + var err error + + secondNamespaceCtx, err = clusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, clusterResources.StorageClassName, clusterResources.WorkerStorageClassName, secondNamespaceName, input.ArtifactFolder) + Expect(err).ToNot(HaveOccurred(), "Failed to create a second test WCP namespace") + wcp.WaitForNamespaceReady(wcpClient, secondNamespaceName) + + By("Wait for Linux VM Image to be available in the second namespace") + + var vmImageName2 string + + vmImageName2, err = vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterClient, secondNamespaceName, linuxImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VM Image name in namespace %q", secondNamespaceName) + Expect(vmImageName2).NotTo(BeEmpty(), "VM Image CR name is empty for the second namespace") + + By("Create a secret in the second namespace with cloud-init config") + Eventually(func(g Gomega) { + createSecret(ctx, g, clusterProxy, secondNamespaceName, secretName) + }, config.GetIntervals("default", "wait-secret-creation")...).Should(Succeed(), "Timed out in creating the Secret") + vmservice.VerifySecretCreation(ctx, config, svClusterClient, secondNamespaceName, secretName) + + By("Create VM2 with a Public Subnet in the second ns") + // Create a Public subnet with the same name but in a different ns + Eventually(func(g Gomega) { + createSubnetOrSubnetSet(ctx, g, clusterProxy, subnetKind, subnetName, secondNamespaceName, cidrConfig, false) + }, config.GetIntervals("default", "wait-subnet-creation")...).Should(Succeed(), "Timed out in creating Subnet") + vmservice.VerifySubnetOrSubnetSetCreation(ctx, config, svClusterClient, secondNamespaceName, subnetName, subnetKind) + + v1a2vmParameters.Name = vm2Name + v1a2vmParameters.Namespace = secondNamespaceName + v1a2vmParameters.ImageName = vmImageName2 + // Create v1alpha2 VM deployment yaml + vmYaml = manifestbuilders.GetVirtualMachineYamlA2(v1a2vmParameters) + + Eventually(func(g Gomega) { + createVM(ctx, g, clusterProxy, vmYaml) + }, config.GetIntervals("default", "wait-virtual-machine-creation")...).Should(Succeed(), "Timed out in creating the VirtualMachine") + vmoperator.WaitForVirtualMachineCreation(ctx, config, svClusterClient, secondNamespaceName, vm2Name) + vm2IP = vmoperator.GetVirtualMachineIP(ctx, svClusterClient, secondNamespaceName, vm2Name) + + By("Label VM2 and apply Security Policy that allows ingress") + Expect(vmservice.LabelVM(ctx, config, clusterProxy, vm2Name, secondNamespaceName, "role", "allow-ingress")).To(Succeed()) + + securityPolicyName := "allow-all-ingress" + + Eventually(func(g Gomega) { + createSecurityPolicy(ctx, g, clusterProxy, securityPolicyName, secondNamespaceName) + }, config.GetIntervals("default", "wait-security-policy-creation")...).Should(Succeed(), "Timed out in creating SecurityPolicy") + vmservice.VerifySecurityPolicyCreation(ctx, config, svClusterClient, secondNamespaceName, securityPolicyName) + + By("VM1 on Private Subnet should now be able to ping VM2 on Public Subnet with Security Policy") + verifyLoginAndPingVM(ctx, config, clusterProxy, svClusterClient, input.WCPNamespaceName, vm1IP, vm2IP) + }) + }) +} + +// verifyLoginAndPingVM creates a jumpbox PodVM and exec inside the PodVM. +// From there, it SSH into vm1IP and use /dev/tcp to verify vm2IP is reachable. +func verifyLoginAndPingVM(ctx context.Context, config *e2eConfig.E2EConfig, clusterProxy *common.VMServiceClusterProxy, svClusterClient ctrlclient.Client, wcpNamespace, vm1IP, vm2IP string) { + vmservice.WaitForPodReady(ctx, config, svClusterClient, wcpNamespace, consts.JumpboxPodVMName) + + // Photon5 VM has ICMP (ping) traffic blocked by default. + // To verify VM communication, use the built-in /dev/tcp to open a TCP + // connection to the other VM's IP address at port 22. + cmds := []string{fmt.Sprintf("timeout 5 bash -c 'echo > /dev/tcp/%s/22' && echo 'VM communication successful'", vm2IP)} + // Expect successful output for VM communication testing + expectedOutput := []string{"VM communication successful"} + vmservice.VerifyLoginAndRunCmdsInNSXSetup(ctx, config, clusterProxy, wcpNamespace, consts.JumpboxPodVMName, vm1IP, cmds, expectedOutput) +} diff --git a/test/e2e/vmservice/vmservice/virtualmachine/vm_webconsolerequest.go b/test/e2e/vmservice/vmservice/virtualmachine/vm_webconsolerequest.go new file mode 100644 index 000000000..68e6c00ba --- /dev/null +++ b/test/e2e/vmservice/vmservice/virtualmachine/vm_webconsolerequest.go @@ -0,0 +1,110 @@ +// Copyright (c) 2023-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +const ( + vmWebConsoleSpecName = "vm-web-console" +) + +type VMWebConsoleRequestSpecInput struct { + ClusterProxy wcpframework.WCPClusterProxyInterface + Config *e2eConfig.E2EConfig + ArtifactFolder string + SkipCleanup bool + LinuxVMName string + WCPNamespaceName string +} + +func VMWebConsoleRequestSpec(ctx context.Context, inputGetter func() VMWebConsoleRequestSpecInput) { + var ( + input VMWebConsoleRequestSpecInput + config *e2eConfig.E2EConfig + clusterProxy *common.VMServiceClusterProxy + svClusterClient ctrlclient.Client + webconsoleName string + resourceName string + webconsoleParams manifestbuilders.VirtualMachineWebConsoleRequestYaml + ) + + BeforeEach(func() { + // Set up infrastructure related configs. + input = inputGetter() + Expect(input.Config).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", vmWebConsoleSpecName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", vmWebConsoleSpecName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.SVClusterProxy can't be nil when calling %s spec", vmWebConsoleSpecName) + Expect(input.WCPNamespaceName).ToNot(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", vmWebConsoleSpecName) + Expect(input.LinuxVMName).ToNot(BeEmpty(), "Invalid argument. input.LinuxVMName can't be empty when calling %s spec", vmWebConsoleSpecName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", vmWebConsoleSpecName) + + config = input.Config + clusterProxy = input.ClusterProxy.(*common.VMServiceClusterProxy) + svClusterClient = clusterProxy.GetClient() + + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, []string{config.GetVariable("VMOPNamespace")}, input.ClusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, vmWebConsoleSpecName)) + DeferCleanup(cancelPodWatches) + + webconsoleName = fmt.Sprintf("%s-%s", vmWebConsoleSpecName, capiutil.RandomString(4)) + webconsoleParams = manifestbuilders.VirtualMachineWebConsoleRequestYaml{ + Namespace: input.WCPNamespaceName, + Name: webconsoleName, + VMName: input.LinuxVMName, + } + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, svClusterClient, clusterProxy.GetKubeconfigPath(), input.WCPNamespaceName, webconsoleName, resourceName) + } + }) + + Context("Create web console request CR", func() { + It("should successfully create v1a1 webconsolerequests and populate status", Label("smoke"), func() { + resourceName = "webconsolerequests" + webconsoleYaml := manifestbuilders.GetV1A1WebConsoleRequestYaml(webconsoleParams) + e2eframework.Logf("%v", string(webconsoleYaml)) + Expect(clusterProxy.CreateWithArgs(ctx, webconsoleYaml)).NotTo(HaveOccurred(), "failed to create v1a1 webconsole request", string(webconsoleYaml)) + // Verify webconsole request creation. + vmservice.VerifyWebConsoleRequestCreation(ctx, config, svClusterClient, input.WCPNamespaceName, webconsoleName) + vmoperator.VerifyWebConsoleRequestStatus(ctx, config, svClusterClient, input.WCPNamespaceName, webconsoleName) + }) + + It("When WCP_VMService_v1alpha2 enabled, should successfully create v1a2 virtualmachinewebconsolerequests and populate status", func() { + // Skip if WCP_VMService_v1alpha2 FSS not enabled + skipper.SkipUnlessV1a2FSSEnabled(ctx, svClusterClient, config) + + resourceName = "virtualmachinewebconsolerequests" + vmWebconsoleYaml := manifestbuilders.GetVirtualMachineWebConsoleRequestYaml(webconsoleParams) + e2eframework.Logf("%v", string(vmWebconsoleYaml)) + Expect(clusterProxy.CreateWithArgs(ctx, vmWebconsoleYaml)).NotTo(HaveOccurred(), "failed to create v1a2 virtualmachinewebconsole request", string(vmWebconsoleYaml)) + // Verify virtualmachine webconsole request creation. + vmservice.VerifyVirtualMachineWebConsoleRequestCreation(ctx, config, svClusterClient, input.WCPNamespaceName, webconsoleName) + vmoperator.VerifyVirtualMachineWebConsoleRequestStatus(ctx, config, svClusterClient, input.WCPNamespaceName, webconsoleName) + }) + }) +} diff --git a/test/e2e/vmservice/vmservice_suite_test.go b/test/e2e/vmservice/vmservice_suite_test.go new file mode 100644 index 000000000..696bb4d08 --- /dev/null +++ b/test/e2e/vmservice/vmservice_suite_test.go @@ -0,0 +1,501 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmservicee2e + +import ( + "context" + "flag" + "fmt" + "os" + "regexp" + "strings" + "testing" + + // Configure testing. + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + // Configure logging. + logs "github.com/sirupsen/logrus" + klog "k8s.io/klog/v2" + _ "k8s.io/kubernetes/test/e2e/framework" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + conformancetestdata "k8s.io/kubernetes/test/conformance/testdata" + "k8s.io/kubernetes/test/e2e/framework/testfiles" + e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests" + testfixtures "k8s.io/kubernetes/test/fixtures" + capiutil "sigs.k8s.io/cluster-api/util" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/common" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +const ( + defaultBaseLogDirectoryPath = "test_logs" + runCanonicalTestEnvValue = "true" +) + +var ( + echoPodLogs bool + skipCleanup bool + workloadIsolationFSSEnabled bool + vksTesting bool + configFilePath string + artifactFolder string + skipTests string + repoRoot string + testSuiteName string + wcpNamespaceName string + windowsServerVMName string + linuxVMName string + config *e2eConfig.E2EConfig + svClusterProxy wcpframework.WCPClusterProxyInterface + wcpClient wcp.WorkloadManagementAPI + wcpNamespaceCtx wcpframework.NamespaceContext +) + +func init() { + flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") + flag.StringVar(&repoRoot, "e2e.repo-root", "../../", "Root directory of kubernetes repository, for finding test files.") + flag.StringVar(&artifactFolder, "e2e.artifactFolder", defaultBaseLogDirectoryPath, "folder where e2e test artifact should be stored") + flag.BoolVar(&echoPodLogs, "e2e.echo-pod-logs", false, "Emit pod logs and events to STDOUT") + flag.StringVar(&configFilePath, "e2e.e2e-config", "", "path to the e2e config file") + flag.StringVar(&skipTests, "e2e.test-skip", "", "if set, skip the tests matching the given regex") + flag.StringVar(&wcpNamespaceName, "e2e.wcp-namespace-name", "", "name of the WCP namespace that already exists") + + // Enable embedded FS file lookup as fallback + testfiles.AddFileSource(e2etestingmanifests.GetE2ETestingManifestsFS()) + testfiles.AddFileSource(testfixtures.GetTestFixturesFS()) + testfiles.AddFileSource(conformancetestdata.GetConformanceTestdataFS()) +} + +func TestVMService(t *testing.T) { + // TODO: Deprecating repo-root over time... instead just use gobindata_util.go , see #23987. + // Right now it is still needed, for example by + // test/e2e/framework/ingress/ingress_utils.go + // for providing the optional secret.yaml file and by + // test/e2e/framework/util.go for cluster/log-dump. + if repoRoot != "" { + testfiles.AddFileSource(testfiles.RootFileSource{Root: repoRoot}) + } + + klog.SetOutput(GinkgoWriter) + logf.SetLogger(klog.Background()) + logs.SetFormatter(&logs.JSONFormatter{}) + logs.SetOutput(GinkgoWriter) + + defer klog.Flush() + + RegisterFailHandler(Fail) + RunSpecs(t, "VM Service E2E suite") +} + +var _ = SynchronizedBeforeSuite(func() []byte { + Expect(configFilePath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.") + Expect(os.MkdirAll(artifactFolder, 0775)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder) + + testSuiteName = "vmsvc-e2e" + framework.Byf("Testsuite Name: %s", testSuiteName) + + By("Initializing a runtime.Scheme with all the GVK relevant for this test") + + scheme := common.InitScheme() + + framework.Byf("Loading the e2e test configuration from %q", configFilePath) + config = e2eConfig.LoadE2EConfig(configFilePath) + Expect(config).ToNot(BeNil(), "e2eConfig can't be nil when calling %s test suite", testSuiteName) + + By("Setting up supervisor cluster cluster proxy") + + kubeconfigPath := config.InfraConfig.KubeconfigPath + svClusterProxy = setupSupervisorClusterProxy(kubeconfigPath, scheme, config) + Expect(svClusterProxy).ToNot(BeNil(), "svClusterProxy can't be nil when calling %s test suite", testSuiteName) + + By("Setting up cluster role bindings for e2e tests") + + vmsvcClusterProxy := svClusterProxy.(*common.VMServiceClusterProxy) + err := vmservice.SetupClusterRoleBindings(vmsvcClusterProxy) + Expect(err).ToNot(HaveOccurred(), "failed to setup cluster role bindings") + + By("Creating a new WCP namespace associated with the default storage class, VM class, and vmservice content library") + + ctx := context.TODO() + + By("Creating the VM Service content library") + // Create the VM Service content library if it doesn't exist + subscriptionURL := config.InfraConfig.ManagementClusterConfig.Resources.ContentLibrarySubscriptionURL + Expect(subscriptionURL).ToNot(BeEmpty(), "contentLibrarySubscriptionURL is not set in %s", configFilePath) + vmservice.EnsureVMServiceContentLibrary(ctx, wcpClient, subscriptionURL) + + vmClassName := config.InfraConfig.ManagementClusterConfig.Resources.VMClassName + storageClassName := config.InfraConfig.ManagementClusterConfig.Resources.StorageClassName + workerStorageClassName := config.InfraConfig.ManagementClusterConfig.Resources.WorkerStorageClassName + + if wcpNamespaceName == "" { + wcpNamespaceName = config.GetVariable("E2ENamespace") + } + + if wcpNamespaceName == "" { + namespaceName := fmt.Sprintf("%s-%s", testSuiteName, capiutil.RandomString(6)) + wcpNamespaceCtx = createWCPNamespaceCtx(ctx, namespaceName, consts.VMServiceCLName, vmClassName, storageClassName, workerStorageClassName) + wcpNamespaceName = wcpNamespaceCtx.GetNamespace().Name + wcp.WaitForNamespaceReady(wcpClient, wcpNamespaceName) + } else { + _, err := wcpClient.GetNamespace(wcpNamespaceName) + if err != nil { + framework.Byf("Namespace %q does not exist, creating it", wcpNamespaceName) + wcpNamespaceCtx = createWCPNamespaceCtx(ctx, wcpNamespaceName, consts.VMServiceCLName, vmClassName, storageClassName, workerStorageClassName) + wcp.WaitForNamespaceReady(wcpClient, wcpNamespaceName) + } else { + framework.Byf("Namespace %q already exists, skipping creation", wcpNamespaceName) + } + } + + if workerStorageClassName != "" { + svClient := svClusterProxy.GetClientSet() + primarySC, err := svClient.StorageV1().StorageClasses().Get(ctx, storageClassName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "primary storage class %q must exist for worker StorageClass setup", storageClassName) + wcp.EnsureWorkerKubernetesStorageClassIfMissing(ctx, kubeconfigPath, svClient, primarySC, workerStorageClassName, config, wcpNamespaceName, wcpClient) + } + + By("Ensure the storage class is available in the WCP namespace") + + podVMOnStretchedSupervisorEnabled := utils.IsFssEnabled( + ctx, + svClusterProxy.GetClient(), + config.GetVariable("VMOPNamespace"), + config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), + config.GetVariable("EnvFSSPodVMOnStretchedSupervisor"), + ) + utils.EnsureStorageClassInNamespace(ctx, svClusterProxy.GetClient(), + wcpNamespaceName, storageClassName, podVMOnStretchedSupervisorEnabled, + *config) + + if workerStorageClassName != "" { + utils.EnsureStorageClassInNamespace(ctx, svClusterProxy.GetClient(), + wcpNamespaceName, workerStorageClassName, + podVMOnStretchedSupervisorEnabled, *config) + } + + By("Ensure the VM Class is available in the WCP namespace") + Expect(vmservice.EnsureVMClassPresent(wcpClient, vmClassName)).To(Succeed()) + Expect(vmservice.EnsureNamespaceHasAccess(wcpClient, vmClassName, wcpNamespaceName)).To(Succeed()) + + if os.Getenv("RUN_CANONICAL_TEST") == runCanonicalTestEnvValue { + // Deploy an Ubuntu VM for verifying Canonical image + By("Ensure the Canonical Ubuntu image is available in the WCP namespace") + + ubuntuImageDisplayName := config.InfraConfig.ManagementClusterConfig.Resources.UbuntuImageDisplayName + Expect(ubuntuImageDisplayName).ToNot(BeEmpty(), "ubuntuImageDisplayName is not set in %s", configFilePath) + + ubuntuVMIName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterProxy.GetClient(), wcpNamespaceName, ubuntuImageDisplayName) + Expect(err).ToNot(HaveOccurred(), "failed to get the Canonical Ubuntu VMI name in namespace %q", wcpNamespaceName) + + By("Deploying an Canonical Ubuntu VM under the common WCP namespace to make it available for all test specs") + // We only deploy the VM here and not block to wait for it to be ready. + linuxVMName = fmt.Sprintf("common-ubuntu-vm-%s", capiutil.RandomString(4)) + deployVMWithCloudInit(ctx, wcpNamespaceName, linuxVMName, ubuntuVMIName) + } else { + // Deploy a Photon VM + By("Ensure the Photon image is available in the WCP namespace") + + photonImageDisplayName := config.InfraConfig.ManagementClusterConfig.Resources.PhotonImageDisplayName + photonVMIName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterProxy.GetClient(), wcpNamespaceName, photonImageDisplayName) + Expect(err).ToNot(HaveOccurred(), "failed to get the VMI name in namespace %q", wcpNamespaceName) + + By("Deploying a Photon VM under the common WCP namespace to make it available for all test specs") + + linuxVMName = fmt.Sprintf("common-photon-vm-%s", capiutil.RandomString(4)) + // We only deploy the VM here and not block to wait for it to be ready. + deployVMWithCloudInit(ctx, wcpNamespaceName, linuxVMName, photonVMIName) + + if fssEnabled := utils.IsFssEnabled(ctx, svClusterProxy.GetClient(), + config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSWindowsSysprep")); fssEnabled { + // The Windows VM deployment and customization could take a while to complete. + // We only deploy the VM here and skip checking the VM status. + // The actual verification will be done in vm_guestcustomization spec. + By("Get the Windows VMI name from the image display name") + + windowsImageDisplayName := config.InfraConfig.ManagementClusterConfig.Resources.WindowsImageDisplayName + windowsVMIName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, svClusterProxy.GetClient(), wcpNamespaceName, windowsImageDisplayName) + Expect(err).ToNot(HaveOccurred(), "failed to get the VMI name in namespace %q", wcpNamespaceName) + + By("Deploying a Windows VM with the minimal Sysprep config") + // Keep Windows VM name under 15 chars so hostName inherited from vmName can adhere to + // the format specified in RFC-1034, Section 3.5 for DNS labels. + windowsServerVMName = fmt.Sprintf("%s-%s", "windows", capiutil.RandomString(4)) + deployWindowsVMWithSysprep(ctx, wcpNamespaceName, windowsServerVMName, windowsVMIName) + } + } + + // Deploy a jumpbox PodVM in the WCP namespace to be used across multiple tests in NSX setup. + if config.InfraConfig.NetworkingTopology == consts.NSX { + By("Deploying a jumpbox PodVM in the WCP namespace") + deployJumpboxPodVM(ctx) + } + + // If this is a stretch supervisor cluster and namespace scoped zone is introduced (>= VC 9.0), bind zones with supervisor. + // By default, zone-1 is already bound. + workloadIsolationFSSEnabled = utils.IsFssEnabled(ctx, svClusterProxy.GetClient(), + config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvWorkloadIsolation")) + + stretchedTestbed := false + + if workloadIsolationFSSEnabled && os.Getenv("STRETCHED_SUPERVISOR") == "true" { + supervisorID := vcenter.GetSupervisorIDFromKubeconfig(ctx, kubeconfigPath) + err := svClusterProxy.CreateMissingZoneBindingsWithSupervisor(supervisorID, []string{"zone-2", "zone-3"}) + Expect(err).ToNot(HaveOccurred(), "failed to update zone bindings with supervisor") + + // The VKS test we're using doesn't work with stretched since it does not set the + // zones on the worker pools. + stretchedTestbed = true + } + + if !checkSkipVKSTests(skipTests) && !stretchedTestbed { + vksTesting = true + } + + return []byte( + strings.Join([]string{ + artifactFolder, + kubeconfigPath, + configFilePath, + }, ","), + ) +}, func(data []byte) { +}) + +var _ = SynchronizedAfterSuite(func() { +}, func() { + // Clean up cluster role bindings (best effort) + if svClusterProxy != nil && !skipCleanup { + vmsvcClusterProxy := svClusterProxy.(*common.VMServiceClusterProxy) + + err := vmservice.CleanupClusterRoleBindings(vmsvcClusterProxy) + if err != nil { + framework.Byf("Warning: Failed to cleanup cluster role bindings: %v", err) + } + } + + // Do not delete the WCP namespace if the test failed for debugging purpose. + if !CurrentSpecReport().Failed() && + svClusterProxy != nil && + wcpNamespaceCtx.GetNamespace() != nil { + svClusterProxy.DeleteWCPNamespace(wcpNamespaceCtx) + } + + if workloadIsolationFSSEnabled && + os.Getenv("STRETCHED_SUPERVISOR") == "true" && + svClusterProxy != nil { + kubeconfigPath := config.InfraConfig.KubeconfigPath + supervisorID := vcenter.GetSupervisorIDFromKubeconfig(context.Background(), kubeconfigPath) + currentZoneList, err := svClusterProxy.GetZonesBoundWithSupervisor(supervisorID) + Expect(err).ToNot(HaveOccurred(), "failed to get zones bound with Supervisor") + + var zonesToDelete []string + + for _, item := range currentZoneList.Zones { + // Skip zone-1 as it's default zone which can not be deleted. + // Can only delete workload not management zones. + if item.Zone != "zone-1" && item.Type == "WORKLOAD" { + zonesToDelete = append(zonesToDelete, item.Zone) + } + } + + err = svClusterProxy.DeleteZoneBindingsWithSupervisor(supervisorID, zonesToDelete) + if err != nil && !strings.Contains(err.Error(), "are not bound to the Supervisor") { + Expect(err).NotTo(HaveOccurred(), "failed to delete zone bindings with supervisor") + } + } +}) + +// setupSupervisorClusterProxy creates a WCPClusterProxyInterface instance with kubeconfig and runtime scheme. +func setupSupervisorClusterProxy(kubeconfigPath string, sc *runtime.Scheme, config *e2eConfig.E2EConfig) wcpframework.WCPClusterProxyInterface { + var supervisorClusterProxy wcpframework.WCPClusterProxyInterface + if framework.InfraIs(config.InfraConfig.InfraName, consts.KIND) { + supervisorClusterProxy = wcpframework.NewSimulatedWCPClusterProxy("supervisor", kubeconfigPath, sc) + } else { + supervisorClusterProxy = common.NewVMServiceClusterProxy("supervisor", kubeconfigPath, sc) + wcpClient = supervisorClusterProxy.(*common.VMServiceClusterProxy).GetWorkloadManagementAPI() + Expect(wcpClient).ToNot(BeNil(), "wcpClient can't be nil when calling %s test suite", testSuiteName) + } + + Expect(supervisorClusterProxy).ToNot(BeNil(), "failed to get a supervisor cluster proxy") + + return supervisorClusterProxy +} + +// createWCPNamespaceCtx creates a WCP namespace with the default associations. +func createWCPNamespaceCtx(ctx context.Context, name, clName, vmClassName, storageClassName, workerStorageClassName string) wcpframework.NamespaceContext { + clID := vmservice.GetContentLibraryUUIDByName(clName, wcpClient) + vmsvcSpecs := wcp.NewVMServiceSpecDetails([]string{vmClassName}, []string{clID}) + wcpNamespaceCtx, err := svClusterProxy.CreateWCPNamespace(ctx, config, vmsvcSpecs, storageClassName, workerStorageClassName, name, artifactFolder) + Expect(err).ToNot(HaveOccurred(), "Failed to create the test WCP namespace") + + return wcpNamespaceCtx +} + +// deployWindowsVMWithSysprep deploys a Windows VM with the minimal Sysprep config passed. +// If WCP_VMService_v1alpha2 is enabled, it additionally deploys a v1a2 Windows VM with the same Sysprep config. +func deployWindowsVMWithSysprep(ctx context.Context, ns, vmName, vmiName string) { + vmsvcClusterProxy := svClusterProxy.(*common.VMServiceClusterProxy) + secretName := "sysprep-data" + secret := manifestbuilders.Secret{ + Namespace: ns, + Name: secretName, + } + + // Check if the secret already exists. + secretObj := corev1.Secret{} + + err := vmsvcClusterProxy.GetClient().Get(ctx, + types.NamespacedName{Namespace: ns, Name: secretName}, &secretObj) + if err == nil { + return + } + + secretYaml := manifestbuilders.GetSecretYamlSysprepConfig(secret) + Expect(vmsvcClusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create the Secret with Sysprep data", string(secretYaml)) + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: ns, + Name: vmName, + VMClassName: config.InfraConfig.ManagementClusterConfig.Resources.VMClassName, + StorageClassName: config.InfraConfig.ManagementClusterConfig.Resources.StorageClassName, + ResourcePolicy: config.InfraConfig.ManagementClusterConfig.Resources.VMResourcePolicyName, + ImageName: vmiName, + Transport: "Sysprep", + SecretName: secretName, + PowerState: "poweredOn", + } + vmYaml := manifestbuilders.GetVirtualMachineYaml(vmParameters) + Expect(vmsvcClusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create v1alpha1 Windows VM", string(vmYaml)) + + if utils.IsFssEnabled(ctx, svClusterProxy.GetClient(), + config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSV1alpha2")) { + secretName = "inline-sysprep-data" + secret = manifestbuilders.Secret{ + Namespace: ns, + Name: secretName, + } + secretYaml = manifestbuilders.GetSecretYamlInlineSysprepData(secret) + Expect(vmsvcClusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create the Secret with Sysprep data", string(secretYaml)) + + inlinedSysprep := fmt.Sprintf(` + guiUnattended: + autoLogon: true + autoLogonCount: 1 + password: + name: %s + key: vmsvc-pwd + timeZone: 004 + identification: + joinWorkgroup: vmware + guiRunOnce: + commands: + - "dir C:" + - "echo Hello" + - 'C:\sysprep\guestcustutil.exe restoreMountedDevices' + - 'C:\sysprep\guestcustutil.exe flagComplete' + - 'C:\sysprep\guestcustutil.exe deleteContainingFolder' + userData: + fullName: "First User" + orgName: "Broadcom"`, secretName) + // Keep Windows VM name under 15 chars so hostName inherited from vmName can adhere to + // the format specified in RFC-1034, Section 3.5 for DNS labels. + v1a2vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: ns, + Name: vmName + "-a2", + VMClassName: config.InfraConfig.ManagementClusterConfig.Resources.VMClassName, + StorageClassName: config.InfraConfig.ManagementClusterConfig.Resources.StorageClassName, + ImageName: vmiName, + PowerState: "PoweredOn", + Bootstrap: manifestbuilders.Bootstrap{ + Sysprep: &manifestbuilders.Sysprep{ + Sysprep: &inlinedSysprep, + }, + }, + } + vmYaml := manifestbuilders.GetVirtualMachineYamlA2(v1a2vmParameters) + Expect(vmsvcClusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create v1alpha2 Windows VM", string(vmYaml)) + } +} + +// deployVMWithCloudInit deploys a VM with the default cloud-init config passed. +func deployVMWithCloudInit(ctx context.Context, ns, vmName, vmiName string) { + vmsvcClusterProxy := svClusterProxy.(*common.VMServiceClusterProxy) + + secretName := vmName + "-cloud-config-data" + secret := manifestbuilders.Secret{ + Namespace: ns, + Name: secretName, + } + + // Check if the secret already exists. + secretObj := corev1.Secret{} + + err := vmsvcClusterProxy.GetClient().Get(ctx, + types.NamespacedName{Namespace: ns, Name: secretName}, &secretObj) + if err == nil { + return + } + + secretYaml := manifestbuilders.GetSecretYamlCloudConfig(secret) + Expect(vmsvcClusterProxy.CreateWithArgs(ctx, secretYaml)).To(Succeed(), "failed to create the Secret with cloud-config data", string(secretYaml)) + + vmParameters := manifestbuilders.VirtualMachineYaml{ + Namespace: ns, + Name: vmName, + ImageName: vmiName, + VMClassName: config.InfraConfig.ManagementClusterConfig.Resources.VMClassName, + StorageClassName: config.InfraConfig.ManagementClusterConfig.Resources.StorageClassName, + ResourcePolicy: config.InfraConfig.ManagementClusterConfig.Resources.VMResourcePolicyName, + Transport: "CloudInit", + SecretName: secretName, + PowerState: "poweredOn", + } + vmYaml := manifestbuilders.GetVirtualMachineYaml(vmParameters) + Expect(vmsvcClusterProxy.CreateWithArgs(ctx, vmYaml)).To(Succeed(), "failed to create VM:\n%s", string(vmYaml)) +} + +func checkSkipVKSTests(skipStr string) bool { + if skipStr == "" { + return false + } + + skippedTests := regexp.MustCompile(`\bVKS\b`) + // If 'VKS' is specified in test-skip, then do not create a VKS cluster. + return skippedTests.MatchString(skipStr) +} + +func deployJumpboxPodVM(ctx context.Context) { + podTemplateConfig := manifestbuilders.PodVMTemplateConfig{ + Name: consts.JumpboxPodVMName, + Namespace: wcpNamespaceName, + } + podVMYamlTemplate, err := manifestbuilders.BuildPodVMYamlTemplate(podTemplateConfig) + Expect(err).NotTo(HaveOccurred(), "failed to build jumpbox pod template due to %v", err) + Expect(podVMYamlTemplate).NotTo(BeEmpty()) + + vmsvcClusterProxy := svClusterProxy.(*common.VMServiceClusterProxy) + Expect(vmsvcClusterProxy.ApplyWithArgs(ctx, podVMYamlTemplate)).To(Succeed(), "failed to create jumpbox PodVM") +} diff --git a/test/e2e/vmservice/vmservice_test.go b/test/e2e/vmservice/vmservice_test.go new file mode 100644 index 000000000..c75c10373 --- /dev/null +++ b/test/e2e/vmservice/vmservice_test.go @@ -0,0 +1,226 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmservicee2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice/devops" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice/viadmin" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice/virtualmachine" +) + +var _ = Describe("Testing VM Services", Label("devops"), Label("viadmin"), Label("virtualmachine"), Label("vmservice"), func() { + Context("VI-ADMIN-VM-CLASS", func() { + viadmin.VIAdminVMClassSpec(context.TODO(), func() viadmin.VIAdminVMClassSpecInput { + return viadmin.VIAdminVMClassSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + } + }) + }) + + Context("VI-ADMIN-CL", func() { + viadmin.VIAdminCLSpec(context.TODO(), func() viadmin.VIAdminCLSpecInput { + return viadmin.VIAdminCLSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) + }) + + Context("VI-ADMIN-RegisterVM", func() { + viadmin.VIAdminRegisterVMSpec(context.TODO(), func() viadmin.VIAdminRegisterVMSpecInput { + return viadmin.VIAdminRegisterVMSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + WCPNamespaceName: wcpNamespaceName, + LinuxVMName: linuxVMName, + } + }) + }) + + Context("DEVOPS-NS", func() { + devops.DevOpsSpec(context.TODO(), func() devops.DevOpsSpecInput { + return devops.DevOpsSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + WCPNamespaceName: wcpNamespaceName, + } + }) + }) + + Context("VIRTUAL-MACHINE", func() { + Context("VM-LCM", func() { + virtualmachine.VMSpec(context.TODO(), func() virtualmachine.VMSpecInput { + return virtualmachine.VMSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + WCPNamespaceName: wcpNamespaceName, + LinuxVMName: linuxVMName, + } + }) + }) + + Context("VM-LONGEVITY", func() { + virtualmachine.VMLongevitySpec(context.TODO(), func() virtualmachine.VMLongevityInput { + return virtualmachine.VMLongevityInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + WCPNamespaceName: wcpNamespaceName, + } + }) + }) + + // Introduce vGPU in the context description so that we can skip vGPU related tests in TEST_SKIP + + Context("VM-GUEST-CUSTOMIZATION", func() { + virtualmachine.VMGOSCSpec(context.TODO(), func() virtualmachine.VMGOSCSpecInput { + return virtualmachine.VMGOSCSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + WCPNamespaceName: wcpNamespaceName, + WindowsServerVMName: windowsServerVMName, + } + }) + }) + + Context("VM-MULTIPLE-CLUSTER", func() { + virtualmachine.VMMultipleClusterSpec(context.TODO(), func() virtualmachine.VMMultipleClusterInput { + return virtualmachine.VMMultipleClusterInput{ + ClusterProxy: svClusterProxy, + Config: config, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + WCPNamespaceName: wcpNamespaceName, + } + }) + }) + + Context("VM-PUBLISH-REQUEST", func() { + virtualmachine.VMPublishRequestSpec(context.TODO(), func() virtualmachine.VMPublishRequestSpecInput { + return virtualmachine.VMPublishRequestSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + WCPNamespaceName: wcpNamespaceName, + LinuxVMName: linuxVMName, + } + }) + }) + + Context("VM-GROUP-PUBLISH-REQUEST", func() { + virtualmachine.VMGroupPublishRequestSpec(context.TODO(), func() virtualmachine.VMGroupPublishRequestSpecInput { + return virtualmachine.VMGroupPublishRequestSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + WCPNamespaceName: wcpNamespaceName, + } + }) + }) + + Context("VM-WEB-CONSOLE-REQUEST", func() { + virtualmachine.VMWebConsoleRequestSpec(context.TODO(), func() virtualmachine.VMWebConsoleRequestSpecInput { + return virtualmachine.VMWebConsoleRequestSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + WCPNamespaceName: wcpNamespaceName, + LinuxVMName: linuxVMName, + } + }) + }) + + Context("VM-VPC", func() { + virtualmachine.VMVPCSpec(context.TODO(), func() virtualmachine.VMVPCSpecInput { + return virtualmachine.VMVPCSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + WCPNamespaceName: wcpNamespaceName, + } + }) + }) + + Context("VM-ENCRYPTION", func() { + virtualmachine.VMEncryptionSpec(context.TODO(), func() virtualmachine.VMEncryptionInput { + return virtualmachine.VMEncryptionInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + } + }) + }) + + Context("VM-NETWORKING", func() { + virtualmachine.VMNetworkSpec(context.TODO(), func() virtualmachine.VMNetworkSpecInput { + return virtualmachine.VMNetworkSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + ArtifactFolder: artifactFolder, + WCPNamespaceName: wcpNamespaceName, + } + }) + }) + + Context("VM-GROUP", func() { + virtualmachine.VMGroupSpec(context.TODO(), func() virtualmachine.VMGroupSpecInput { + return virtualmachine.VMGroupSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + WCPNamespaceName: wcpNamespaceName, + } + }) + }) + + Context("VM-SNAPSHOT", func() { + virtualmachine.VMSnapshotSpec(context.TODO(), func() virtualmachine.VMSnapshotSpecInput { + return virtualmachine.VMSnapshotSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + WCPNamespaceName: wcpNamespaceName, + } + }) + }) + + Context("VM-HARDWARE", func() { + virtualmachine.VMHardwareSpec(context.TODO(), func() virtualmachine.VMHardwareSpecInput { + return virtualmachine.VMHardwareSpecInput{ + ClusterProxy: svClusterProxy, + Config: config, + WCPClient: wcpClient, + ArtifactFolder: artifactFolder, + WCPNamespaceName: wcpNamespaceName, + } + }) + }) + }) +}) diff --git a/test/e2e/vmservice/vmserviceapp/jenkins.go b/test/e2e/vmservice/vmserviceapp/jenkins.go new file mode 100644 index 000000000..d83a4f8f7 --- /dev/null +++ b/test/e2e/vmservice/vmserviceapp/jenkins.go @@ -0,0 +1,122 @@ +// Copyright (c) 2023 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmserviceapp + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" +) + +const ( + jenkinsSpecName = "vmservice-app-jenkins" + // This template file exists in the packer-plugin-vsphere repo. + // Make sure to update the repo if you are changing this file. + jenkinsSpecTemplateName = "jenkins-template.pkr.hcl" +) + +func JenkinsSpec(ctx context.Context, inputGetter func() SpecInput) { + var ( + input SpecInput + config *e2eConfig.E2EConfig + k8sClient ctrlclient.Client + clusterResources *e2eConfig.Resources + kubeconfigPath string + vmName string + ubuntuVMIName string + gatewayIP string + gatewayUsername string + gatewayPassword string + templateFilePath string + ) + + BeforeEach(func() { + By("Set up infrastructure related configs") + + input = inputGetter() + Expect(input.Config).NotTo(BeNil(), "Invalid argument. input.Config can't be nil when calling %s spec", jenkinsSpecName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", jenkinsSpecName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).NotTo(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling %s spec", jenkinsSpecName) + Expect(input.WCPNamespaceName).NotTo(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", jenkinsSpecName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", jenkinsSpecName) + + config = input.Config + k8sClient = input.ClusterProxy.GetClient() + clusterResources = config.InfraConfig.ManagementClusterConfig.Resources + kubeconfigPath = input.ClusterProxy.GetKubeconfigPath() + + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, + []string{config.GetVariable("VMOPNamespace")}, + input.ClusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, jenkinsSpecName)) + DeferCleanup(cancelPodWatches) + + vmName = fmt.Sprintf("source-%s", capiutil.RandomString(4)) + vmiName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, k8sClient, input.WCPNamespaceName, ubuntuImageName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VM Image name: %s", ubuntuImageName) + + ubuntuVMIName = vmiName + + By("Ensure the gateway VM has TCP forwarding enabled to allow SSH access") + + gatewayIP = os.Getenv("GATEWAY_IP") + Expect(gatewayIP).NotTo(BeEmpty(), "GATEWAY_IP environment variable is not set") + + gatewayUsername = os.Getenv("GATEWAY_VM_USERNAME") + Expect(gatewayUsername).NotTo(BeEmpty(), "GATEWAY_VM_USERNAME environment variable is not set") + + gatewayPassword = os.Getenv("GATEWAY_VM_PASSWORD") + Expect(gatewayPassword).NotTo(BeEmpty(), "GATEWAY_VM_PASSWORD environment variable is not set") + Expect(VerifyTCPForwarding(gatewayIP, gatewayUsername, gatewayPassword)).To(Succeed()) + + By("Ensure the Jenkins template file is available in the packer plugin directory") + + templateFilePath, err = GetTemplatePathInPackerPluginDir(jenkinsSpecTemplateName) + Expect(err).NotTo(HaveOccurred()) + Expect(templateFilePath).NotTo(BeEmpty()) + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, k8sClient, kubeconfigPath, input.WCPNamespaceName, vmName, "vm") + } + }) + + It("Should successfully run the Jenkins workload using Packer", func() { + opts := PackerBuildCmdOpts{ + TemplateFilePath: templateFilePath, + TemplateVariables: map[string]string{ + "keep_input_artifact": "true", + "kubeconfig_path": kubeconfigPath, + "supervisor_namespace": input.WCPNamespaceName, + "class_name": clusterResources.VMClassName, + "image_name": ubuntuVMIName, + "source_name": vmName, + "storage_class": clusterResources.StorageClassName, + "ssh_username": vmSSHUsername, + "ssh_password": vmSSHPassword, + "ssh_bastion_host": gatewayIP, + "ssh_bastion_username": gatewayUsername, + "ssh_bastion_password": gatewayPassword, + }, + } + + cmdOut, err := RunPackerBuildCmd(ctx, opts) + Expect(err).NotTo(HaveOccurred(), "failed to run the packer build command, output: %s", string(cmdOut)) + }) +} diff --git a/test/e2e/vmservice/vmserviceapp/packer.go b/test/e2e/vmservice/vmserviceapp/packer.go new file mode 100644 index 000000000..98fd38539 --- /dev/null +++ b/test/e2e/vmservice/vmserviceapp/packer.go @@ -0,0 +1,268 @@ +// Copyright (c) 2022-2023 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmserviceapp + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + capiutil "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/dcli" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/testbed" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/testutils" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" + e2eConfig "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/config" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/consts" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/lib/vmoperator" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/skipper" + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmservice" + "github.com/vmware-tanzu/vm-operator/test/e2e/wcpframework" +) + +// Package specific constants. +const ( + ubuntuImageName = "ubuntu-2004-cloud-init-21.4-kube-v1.20.10" + vmSSHUsername = "packer" + vmSSHPassword = "packer" +) + +type SpecInput struct { + Config *e2eConfig.E2EConfig + ClusterProxy wcpframework.WCPClusterProxyInterface + WCPClient wcp.WorkloadManagementAPI + ArtifactFolder string + SkipCleanup bool + WCPNamespaceName string +} + +// Spec specific constants. +const ( + packerSpecName = "vmservice-app-packer" + // This template file exists in the packer-plugin-vsphere repo. + // Make sure to update the repo if you are changing this file. + packerSpecTemplateName = "general-template.pkr.hcl" + randomSSOUserName = "packer-plugin-random-user" + randomSSOUserPassword = "Password!23" +) + +func PackerSpec(ctx context.Context, inputGetter func() SpecInput) { + var ( + input SpecInput + config *e2eConfig.E2EConfig + wcpClient wcp.WorkloadManagementAPI + k8sClient ctrlclient.Client + clusterResources *e2eConfig.Resources + kubeconfigPath string + vmserviceCLID string + vmName string + vmiName string + cvmiName string + gatewayIP string + gatewayUsername string + gatewayPassword string + vmImageRegistryEnabled bool + templateFilePath string + packerCmdOpts PackerBuildCmdOpts + ) + + BeforeEach(func() { + By("Set up infrastructure related configs") + + input = inputGetter() + Expect(input.Config).NotTo(BeNil(), "Invalid argument. input.Config can't be nil when calling %s spec", packerSpecName) + Expect(input.Config.InfraConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig.InfraConfig can't be nil when calling %s spec", packerSpecName) + skipper.SkipUnlessInfraIs(input.Config.InfraConfig.InfraName, consts.WCP) + + Expect(input.ClusterProxy).NotTo(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling %s spec", packerSpecName) + Expect(input.WCPClient).NotTo(BeNil(), "Invalid argument. input.WCPClient can't be nil when calling %s spec", packerSpecName) + Expect(input.WCPNamespaceName).NotTo(BeEmpty(), "Invalid argument. input.WCPNamespaceName can't be empty when calling %s spec", packerSpecName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", packerSpecName) + + config = input.Config + wcpClient = input.WCPClient + k8sClient = input.ClusterProxy.GetClient() + kubeconfigPath = input.ClusterProxy.GetKubeconfigPath() + clusterResources = config.InfraConfig.ManagementClusterConfig.Resources + + vmImageRegistryEnabled = utils.IsFssEnabled(ctx, k8sClient, + config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSVMImageRegistry")) + + cancelPodWatches := framework.WatchPodLogsAndEventsInNamespaces(ctx, + []string{config.GetVariable("VMOPNamespace")}, + input.ClusterProxy.GetClientSet(), filepath.Join(input.ArtifactFolder, packerSpecName)) + DeferCleanup(cancelPodWatches) + + vmName = fmt.Sprintf("source-%s", capiutil.RandomString(4)) + vmiObjName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, k8sClient, input.WCPNamespaceName, ubuntuImageName) + Expect(err).NotTo(HaveOccurred(), "failed to get the VM Image name: %s", ubuntuImageName) + + vmiName = vmiObjName + vmserviceCLID = vmservice.GetContentLibraryUUIDByName(consts.VMServiceCLName, wcpClient) + + if vmImageRegistryEnabled { + By("Attaching content library to cluster with Image-Registry FSS enabled") + + clusterMoID := vcenter.GetClusterMoIDFromKubeconfig(ctx, kubeconfigPath) + err := wcpClient.AssociateContentLibrariesToCluster(clusterMoID, wcp.ClusterContentLibrarySpec{ContentLibrary: vmserviceCLID}) + Expect(err).NotTo(HaveOccurred(), "failed to attach content library %s to cluster", vmserviceCLID) + cvmiObjName, err := vmoperator.WaitForClusterVirtualMachineImageName(ctx, &config.Config, k8sClient, ubuntuImageName) + Expect(err).NotTo(HaveOccurred(), "failed to get the CVMI with display name: %s", ubuntuImageName) + + cvmiName = cvmiObjName + } + + By("Ensure the gateway VM has TCP forwarding enabled to allow SSH access") + + gatewayIP = os.Getenv("GATEWAY_IP") + Expect(gatewayIP).NotTo(BeEmpty(), "GATEWAY_IP environment variable is not set") + + gatewayUsername = os.Getenv("GATEWAY_VM_USERNAME") + Expect(gatewayUsername).NotTo(BeEmpty(), "GATEWAY_VM_USERNAME environment variable is not set") + + gatewayPassword = os.Getenv("GATEWAY_VM_PASSWORD") + Expect(gatewayPassword).NotTo(BeEmpty(), "GATEWAY_VM_PASSWORD environment variable is not set") + Expect(VerifyTCPForwarding(gatewayIP, gatewayUsername, gatewayPassword)).To(Succeed()) + + By("Ensure the packer template file is available in the packer plugin directory") + + templateFilePath, err = GetTemplatePathInPackerPluginDir(packerSpecTemplateName) + Expect(err).NotTo(HaveOccurred()) + Expect(templateFilePath).NotTo(BeEmpty()) + // Populate all variables to avoid unset errors in running the packer build command. + packerCmdOpts = PackerBuildCmdOpts{ + TemplateFilePath: templateFilePath, + TemplateVariables: map[string]string{ + "keep_input_artifact": "true", + "kubeconfig_path": kubeconfigPath, + "supervisor_namespace": input.WCPNamespaceName, + "class_name": clusterResources.VMClassName, + "image_name": vmiName, + "source_name": vmName, + "storage_class": clusterResources.StorageClassName, + "ssh_username": vmSSHUsername, + "ssh_password": vmSSHPassword, + "ssh_bastion_host": gatewayIP, + "ssh_bastion_username": gatewayUsername, + "ssh_bastion_password": gatewayPassword, + "publish_location_name": "", + "publish_image_name": "", + }, + } + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + vmoperator.DescribeResourceIfExists(ctx, k8sClient, kubeconfigPath, input.WCPNamespaceName, vmName, "vm") + } + }) + + It("Packer command to build and deploy a VM from a view only access SSO user", func() { + // Create SSO user and associate to the namespace with view only access. + sshCommandRunner, _, supervisorClusterIP := testutils.GetHelpersFromKubeconfig(ctx, kubeconfigPath) + vCenterAdminCreds := dcli.VCenterUserCredentials{Username: testbed.AdminUsername, Password: testbed.AdminPassword} + nonAdminUser := vcenter.NewUser(randomSSOUserName, randomSSOUserPassword).WithAdminCreds(vCenterAdminCreds).WithSSHCommandRunner(sshCommandRunner) + kubectlPlugin := testutils.CreateUserAndLogin(nonAdminUser, supervisorClusterIP, "", "") + testutils.SetUserPermissionsOnNamespace(wcpClient, nonAdminUser, wcp.ViewAccessType, input.WCPNamespaceName) + + // Get the SSO user's kubeconfig path and set in the packer command variables. + kubeconfigAbsPath, err := filepath.Abs(kubectlPlugin.KubeconfigPath()) + Expect(err).ToNot(HaveOccurred()) + + packerCmdOpts.TemplateVariables["kubeconfig_path"] = kubeconfigAbsPath + cmdOut, err := RunPackerBuildCmd(ctx, packerCmdOpts) + Expect(err).To(HaveOccurred()) + Expect(string(cmdOut)).To(ContainSubstring((" cannot create resource"))) + + // Delete the SSO user. + vcenter.DeleteUserOrFail(nonAdminUser) + }) + + It("Packer command to build and deploy a VM from VMI with valid configs and keep_input_artifact set to true", func() { + // We run this spec regardless of the Image-Registry FSS state with a VMI (exist in both cases). + cmdOut, err := RunPackerBuildCmd(ctx, packerCmdOpts) + Expect(err).ToNot(HaveOccurred(), "failed to run packer build command, output: %s", string(cmdOut)) + Expect(string(cmdOut)).To(ContainSubstring("Build 'vsphere-supervisor' finished successfully")) + // Expect the source VM exists after the command completes. + sourceName := packerCmdOpts.TemplateVariables["source_name"] + vm, err := utils.GetVirtualMachine(ctx, k8sClient, input.WCPNamespaceName, sourceName) + Expect(err).ToNot(HaveOccurred(), "failed to get the source VM built from Packer") + Expect(vm).ToNot(BeNil()) + }) + + It("Packer command to build, deploy, and publish a VM from CVMI with Image-Registry FSS enabled", func() { + if !vmImageRegistryEnabled { + Skip("Skipping this test spec as FSS is disabled") + } + + // We are switching to CVMI here as VMI is already verified in the above test spec. + packerCmdOpts.TemplateVariables["image_name"] = cvmiName + + // We have multiple packer command runs in this single It to avoid setting up + // the similar environment (ns, cl, etc.) from calling the BeforeEach every time. + // Once we upgrade to ginkgo v2, we can considering having multiple Its with BeforeAll. + By("Publishing to a non-existing content library", func() { + packerCmdOpts.TemplateVariables["publish_location_name"] = "non-existing-content-library" + cmdOut, err := RunPackerBuildCmd(ctx, packerCmdOpts) + Expect(err).To(HaveOccurred()) + Expect(string(cmdOut)).To(ContainSubstring("not found"), "packer build error: ", err.Error()) + }) + + // Create a new content library used for the following packer command runs. + publishCLName := fmt.Sprintf("%s-%s-%s", packerSpecName, "content-library", capiutil.RandomString(4)) + publishCLID := vmservice.CreateLocalContentLibrary(publishCLName, wcpClient) + + By("Publishing to a non-writable content library", func() { + Expect(wcpClient.AssociateImageRegistryContentLibrariesToNamespace(input.WCPNamespaceName, wcp.ContentLibrarySpec{ + ContentLibrary: publishCLID, + Writable: false, + })).To(Succeed(), "failed to attach a non-writable content library to namespace, CL ID: %s", publishCLID) + + k8sContentLibraryName, err := vmservice.GetK8sContentLibraryNameByUUID(ctx, config, k8sClient, input.WCPNamespaceName, publishCLID) + Expect(err).NotTo(HaveOccurred()) + + packerCmdOpts.TemplateVariables["publish_location_name"] = k8sContentLibraryName + cmdOut, err := RunPackerBuildCmd(ctx, packerCmdOpts) + Expect(err).To(HaveOccurred()) + Expect(string(cmdOut)).To(ContainSubstring("not writable"), "packer build error: ", err.Error()) + + Expect(wcpClient.DisassociateImageRegistryContentLibrariesFromNamespace(input.WCPNamespaceName, publishCLID)).To(Succeed()) + }) + + By("Publishing to a writable content library", func() { + Expect(wcpClient.AssociateImageRegistryContentLibrariesToNamespace(input.WCPNamespaceName, wcp.ContentLibrarySpec{ + ContentLibrary: publishCLID, + Writable: true, + })).To(Succeed(), "failed to attach a writable content library to namespace, CL ID: %s", publishCLID) + + k8sContentLibraryName, err := vmservice.GetK8sContentLibraryNameByUUID(ctx, config, k8sClient, input.WCPNamespaceName, publishCLID) + Expect(err).NotTo(HaveOccurred()) + + packerCmdOpts.TemplateVariables["publish_location_name"] = k8sContentLibraryName + publishImageDisplayName := fmt.Sprintf("%s-%s-%s", packerSpecName, "publish-image", capiutil.RandomString(4)) + packerCmdOpts.TemplateVariables["publish_image_name"] = publishImageDisplayName + cmdOut, err := RunPackerBuildCmd(ctx, packerCmdOpts) + Expect(err).NotTo(HaveOccurred(), "failed to run packer build command, output: %s", string(cmdOut)) + + // Ensure the published image is available with expected display name under the namespace. + publishedImageCRName, err := vmoperator.WaitForVirtualMachineImageName(ctx, &config.Config, k8sClient, input.WCPNamespaceName, publishImageDisplayName) + Expect(err).NotTo(HaveOccurred(), "failed to get the published VM Image by displayed name %s", publishImageDisplayName) + Expect(publishedImageCRName).NotTo(BeEmpty(), "published VM Image resource name is empty") + + Expect(wcpClient.DisassociateImageRegistryContentLibrariesFromNamespace(input.WCPNamespaceName, publishCLID)).To(Succeed()) + }) + + Expect(wcpClient.DeleteLocalContentLibrary(publishCLID)).To(Succeed(), "failed to delete the publish content library, CL ID: %s", publishCLID) + }) +} diff --git a/test/e2e/vmservice/vmserviceapp/packer_utils.go b/test/e2e/vmservice/vmserviceapp/packer_utils.go new file mode 100644 index 000000000..58ad94956 --- /dev/null +++ b/test/e2e/vmservice/vmserviceapp/packer_utils.go @@ -0,0 +1,113 @@ +// Copyright (c) 2023 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmserviceapp + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "golang.org/x/crypto/ssh" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + + e2essh "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/ssh" +) + +const ( + tcpForwardEnabled = "\"AllowTcpForwarding yes\"" + tcpForwardDisabled = "\"AllowTcpForwarding no\"" + configCheckCmd = "grep -Fx %s /etc/ssh/sshd_config" + configChangeCmd = "sed -i s/%s/%s/g /etc/ssh/sshd_config" + restartServiceCmd = "systemctl restart sshd" + + // Path to the example template files in the packer-plugin-vsphere repo. + pluginDirTemplateExamplePath = "examples/builder/vsphere-supervisor" +) + +// PackerBuildCmdOpts contains the options to run a packer build command. +type PackerBuildCmdOpts struct { + TemplateFilePath string + TemplateVariables map[string]string +} + +// RunPackerBuildCmd runs a 'packer build' command with the given options. +func RunPackerBuildCmd(ctx context.Context, opts PackerBuildCmdOpts) ([]byte, error) { + var repoRoot, pluginDirPath string + if repoRoot = os.Getenv("REPO_ROOT"); repoRoot == "" { + return nil, errors.New("REPO_ROOT is required for running packer build command") + } + + if pluginDirPath = os.Getenv("PACKER_PLUGIN_DIR_PATH"); pluginDirPath == "" { + return nil, errors.New("PACKER_PLUGIN_DIR_PATH is required for running packer build command") + } + + cmdArgs := []string{"build"} + + // Add all the template variables to the packer build command. + for key, val := range opts.TemplateVariables { + cmdArgs = append(cmdArgs, "-var", fmt.Sprintf("%s=%s", key, val)) + } + + // Add the template file path to the end of packer build command. + cmdArgs = append(cmdArgs, opts.TemplateFilePath) + cmd := exec.Command("packer", cmdArgs...) + + // Set the command directory to ensure packer-plugin-vsphere binary is accessible. + cmd.Dir = pluginDirPath + e2eframework.Logf("Running command: %s (in %s)", cmd.String(), cmd.Dir) + + return cmd.Output() +} + +// GetTemplatePathInPackerPluginDir returns the path to the given template file in PACKER_PLUGIN_DIR_PATH. +func GetTemplatePathInPackerPluginDir(templateName string) (string, error) { + pluginDirPath := os.Getenv("PACKER_PLUGIN_DIR_PATH") + if pluginDirPath == "" { + return "", errors.New("PACKER_PLUGIN_DIR_PATH is empty") + } + + path := filepath.Join(pluginDirPath, pluginDirTemplateExamplePath, templateName) + if _, err := os.Stat(path); err == nil { //nolint:gosec // G703: path is built from known template dir + return path, nil + } else if os.IsNotExist(err) { + return "", fmt.Errorf("template file %s does not exist: %w", path, err) + } else { + return "", fmt.Errorf("failed to check if template file %s exists: %w", path, err) + } +} + +// VerifyTCPForwarding checks the TCP forwarding config on the given gateway VM. +// If disabled, it will enable it and restart the sshd service to allow Packer access to source VMs. +func VerifyTCPForwarding(gatewayIP, gatewayUsername, gatewayPassword string) error { + authPassword := []ssh.AuthMethod{ssh.Password(gatewayPassword)} + + gatewayCmdRunner, err := e2essh.NewSSHCommandRunner(gatewayIP, 22, gatewayUsername, authPassword) + if err != nil { + return err + } + + checkDisabledCmd := fmt.Sprintf(configCheckCmd, tcpForwardDisabled) + + disabledOut, err := gatewayCmdRunner.RunCommand(checkDisabledCmd) + if err == nil && len(disabledOut) != 0 { + e2eframework.Logf("TCP forwarding is disabled on the gateway VM, enabling it now...") + + enableConfigCmd := fmt.Sprintf(configChangeCmd, tcpForwardDisabled, tcpForwardEnabled) + if _, err := gatewayCmdRunner.RunCommand(enableConfigCmd); err != nil { + return fmt.Errorf("failed to enable tcp forwarding config on gateway VM: %w", err) + } + + if _, err := gatewayCmdRunner.RunCommand(restartServiceCmd); err != nil { + return fmt.Errorf("failed to restart sshd service on gateway VM: %w", err) + } + + time.Sleep(10 * time.Second) // time buffer for sshd service fully restarted + } + + return nil +} diff --git a/test/e2e/vmservice/vmserviceapp_test.go b/test/e2e/vmservice/vmserviceapp_test.go new file mode 100644 index 000000000..276a47782 --- /dev/null +++ b/test/e2e/vmservice/vmserviceapp_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2023 Broadcom. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmservicee2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + + "github.com/vmware-tanzu/vm-operator/test/e2e/vmservice/vmserviceapp" +) + +var _ = Describe("Testing VM Services App Workloads", FlakeAttempts(5), Label("jenkins"), Label("packer"), Label("vmserviceapp"), func() { + ctx := context.TODO() + specInputFunc := func() vmserviceapp.SpecInput { + return vmserviceapp.SpecInput{ + ArtifactFolder: artifactFolder, + ClusterProxy: svClusterProxy, + Config: config, + SkipCleanup: skipCleanup, + WCPClient: wcpClient, + WCPNamespaceName: wcpNamespaceName, + } + } + + Context("JENKINS", func() { + vmserviceapp.JenkinsSpec(ctx, specInputFunc) + }) + + Context("PACKER", func() { + vmserviceapp.PackerSpec(ctx, specInputFunc) + }) +}) diff --git a/test/e2e/wcpframework/wcp_cluster_proxy.go b/test/e2e/wcpframework/wcp_cluster_proxy.go new file mode 100644 index 000000000..a688f1430 --- /dev/null +++ b/test/e2e/wcpframework/wcp_cluster_proxy.go @@ -0,0 +1,596 @@ +package wcpframework + +import ( + "context" + "errors" + "fmt" + "path/filepath" + + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/test/e2e/framework" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/vcenter" + "github.com/vmware-tanzu/vm-operator/test/e2e/infrastructure/vsphere/wcp" + "github.com/vmware-tanzu/vm-operator/test/e2e/manifestbuilders" + "github.com/vmware-tanzu/vm-operator/test/e2e/utils" +) + +const ( + // Workload namespace annotations usually set by wcpsvc. Set here for simulated wcp + // to mimic actual wcp environment. + resourcePoolNsAnnotation = "vmware-system-resource-pool" + vmFolderNsAnnotation = "vmware-system-vm-folder" + + // Workload namespace labels usually set by wcpsvc. Set here for simulated wcp + // to mimic actual wcp environment. + userWorkloadNamespaceLabel = "vSphereClusterID" +) + +// WCP Cluster proxy interface is designed to better fit the architecture of WCP cluster +// WCO cluster contains Kubernetes behaviors and wcp api behaviors +// Cluster proxy interface encapsulates canonical kubernetes behaviors +// However, the interface here only contains key behaviors for E2E testing +// in which case the namespace creation and deletion are the most common use cases. +type WCPClusterProxyInterface interface { + framework.ClusterProxyInterface + CreateWCPNamespace(ctx context.Context, config framework.ConfigInterface, + vmsvcSpecs wcp.VMServiceSpecDetails, + scName, wscName, nsName, artifactFolder string) (NamespaceContext, error) + UpdateNamespaceWithZones(ctx context.Context, namespaceName string, zones []string, svClusterClient ctrlclient.Client) (ZoneContext, error) + DeleteZonesFromNamespace(ctx context.Context, namespaceName string, zones []string, svClusterClient ctrlclient.Client) error + GetZonesBoundWithSupervisor(supervisorID string) (wcp.ZoneList, error) + CreateZoneBindingsWithSupervisor(supervisorID string, zones []string) error + CreateMissingZoneBindingsWithSupervisor(supervisorID string, zones []string) error + DeleteZoneBindingsWithSupervisor(supervisorID string, zones []string) error + ListVSphereZones() (wcp.VSphereZoneList, error) + CreateWCPNamespaceWithNetwork(ctx context.Context, config framework.ConfigInterface, + vmsvcSpecs wcp.VMServiceSpecDetails, + scName, wscName, nsName, artifactFolder string, network wcp.NameSpaceNetworkInfo) (NamespaceContext, error) + CreateWCPNamespaceWithVMReservation(ctx context.Context, nsName, scName, zone, supervisorID string, vmsvcSpecs wcp.VMServiceSpecDetails, vmClassNameToReservedCount map[string]int) (NamespaceContext, error) + DeleteWCPNamespace(nsCtx NamespaceContext) +} + +// WCP cluster proxy is designed particular to operators that need to talk with wcp api. +type WCPClusterProxy struct { + framework.ClusterProxyInterface + + wcpAPI wcp.WorkloadManagementAPI +} + +// Instantiate a wcp cluster proxy object. +func NewWCPClusterProxy(name string, kubeconfigPath string, scheme *runtime.Scheme) *WCPClusterProxy { + kubeClusterProxy := framework.NewClusterProxy(name, kubeconfigPath, scheme) + wcpAPI := wcp.NewClientUsingKubeconfig(context.TODO(), kubeconfigPath) + + return &WCPClusterProxy{ + ClusterProxyInterface: kubeClusterProxy, + wcpAPI: wcpAPI, + } +} + +// Instantiate a wcp cluster proxy object with provided credentials. +func NewWCPClusterProxyWithCredentials(name string, kubeconfigPath string, user string, password string, scheme *runtime.Scheme) *WCPClusterProxy { + kubeClusterProxy := framework.NewClusterProxy(name, kubeconfigPath, scheme) + wcpAPI := wcp.NewClientUsingKubeconfigWithCredentials(context.TODO(), kubeconfigPath, user, password) + + return &WCPClusterProxy{ + ClusterProxyInterface: kubeClusterProxy, + wcpAPI: wcpAPI, + } +} + +// GetWorkloadManagementAPI returns the WCP API associated with the supervisor cluster. +func (w *WCPClusterProxy) GetWorkloadManagementAPI() wcp.WorkloadManagementAPI { + return w.wcpAPI +} + +// CreateWCPNamespace applies wcp api to create a namespace in wcp cluster with the given specs. +func (w *WCPClusterProxy) CreateWCPNamespace(ctx context.Context, config framework.ConfigInterface, + vmsvcSpecs wcp.VMServiceSpecDetails, + scName, wscName, nsName, artifactFolder string) (NamespaceContext, error) { + // we need to create namespace with vm class and bind it + // the CR of vm class will get deleted if the namespace owning it gets deleted + // we can assume that vcenter has default vm classes names present + namespace, cancelNsWatches := wcp.CreateNamespace(ctx, wcp.NamespaceCreateInput{ + SpecName: nsName, + ClientSet: w.ClusterProxyInterface.GetClientSet(), + Client: w.ClusterProxyInterface.GetClient(), + Kubeconfig: w.ClusterProxyInterface.GetKubeconfigPath(), + StorageClassName: scName, + WorkerStorageClassName: wscName, + Config: config, + WCPClient: w.wcpAPI, + ArtifactFolder: artifactFolder, + VMServiceSpec: vmsvcSpecs, + }) + + return NamespaceContext{ + namespace: namespace, + cancelNsWatches: cancelNsWatches, + }, nil +} + +// CreatePrivateRegistry creates a container image registry for the supervisor cluster. +func (w *WCPClusterProxy) CreatePrivateRegistry(ctx context.Context, registryName, hostname string, port int, username, password, certificateChain string, defaultRegistry bool) wcp.ContainerImageRegistryInfo { + return wcp.CreatePrivateRegistry(ctx, wcp.PrivateRegistryInput{ + Kubeconfig: w.ClusterProxyInterface.GetKubeconfigPath(), + WCPClient: w.wcpAPI, + RegistryName: registryName, + Hostname: hostname, + Port: port, + Username: username, + Password: password, + CertificateChain: certificateChain, + DefaultRegistry: defaultRegistry, + }) +} + +// DeletePrivateRegistry deletes a container image registry for the supervisor cluster. +func (w *WCPClusterProxy) DeletePrivateRegistry(ctx context.Context, registryName string) error { + return wcp.DeletePrivateRegistry(ctx, wcp.PrivateRegistryInput{ + Kubeconfig: w.ClusterProxyInterface.GetKubeconfigPath(), + WCPClient: w.wcpAPI, + RegistryName: registryName, + }) +} + +func (w *WCPClusterProxy) GetZonesBoundWithSupervisor(supervisorID string) (wcp.ZoneList, error) { + zoneList, _ := wcp.GetZonesBoundWithSupervisor(wcp.ZonesGetInput{ + SupervisorID: supervisorID, + WCPClient: w.wcpAPI, + }) + + return zoneList, nil +} + +func (w *WCPClusterProxy) CreateMissingZoneBindingsWithSupervisor(supervisorID string, zones []string) error { + boundZones, err := w.GetZonesBoundWithSupervisor(supervisorID) + if err != nil { + return err + } + + if len(boundZones.Zones) > 0 { + var missingZones []string + + for _, z := range zones { + missing := true + + for _, bz := range boundZones.Zones { + if z == bz.Zone { + missing = false + break + } + } + + if missing { + missingZones = append(missingZones, z) + } + } + + zones = missingZones + } + + if len(zones) > 0 { + return w.CreateZoneBindingsWithSupervisor(supervisorID, zones) + } + + return nil +} + +func (w *WCPClusterProxy) CreateZoneBindingsWithSupervisor(supervisorID string, zones []string) error { + return wcp.CreateZoneBindingsWithSupervisor(wcp.ZonesBindingInput{ + SupervisorID: supervisorID, + Zones: zones, + WCPClient: w.wcpAPI, + }) +} + +func (w *WCPClusterProxy) ListVSphereZones() (wcp.VSphereZoneList, error) { + return wcp.ListVSphereZones(w.wcpAPI) +} + +func (w *WCPClusterProxy) DeleteZoneBindingsWithSupervisor(supervisorID string, zones []string) error { + return wcp.DeleteZoneBindingsWithSupervisor(wcp.ZonesBindingInput{ + SupervisorID: supervisorID, + Zones: zones, + WCPClient: w.wcpAPI, + }) +} + +// UpdateNamespaceWithZones updates namespace with specified zones. +func (w *WCPClusterProxy) UpdateNamespaceWithZones(ctx context.Context, nsName string, zones []string, svClusterClient ctrlclient.Client) (ZoneContext, error) { + zoneList, cancelZoneWatches := wcp.UpdateNamespaceWithZones(ctx, wcp.BindZonesForNamespaceInput{ + Namespace: nsName, + Zones: zones, + SvClusterClient: svClusterClient, + ClientSet: w.ClusterProxyInterface.GetClientSet(), + Kubeconfig: w.ClusterProxyInterface.GetKubeconfigPath(), + WCPClient: w.wcpAPI, + }) + + return ZoneContext{ + zoneList: zoneList, + cancelZoneWatches: cancelZoneWatches, + }, nil +} + +// DeleteZonesFromNamespace deletes specified zones from namespace. +func (w *WCPClusterProxy) DeleteZonesFromNamespace(ctx context.Context, nsName string, zones []string, svClusterClient ctrlclient.Client) error { + return wcp.DeleteZonesFromNamespace(ctx, wcp.BindZonesForNamespaceInput{ + Namespace: nsName, + Zones: zones, + SvClusterClient: svClusterClient, + ClientSet: w.ClusterProxyInterface.GetClientSet(), + Kubeconfig: w.ClusterProxyInterface.GetKubeconfigPath(), + WCPClient: w.wcpAPI, + }) +} + +// CreateWCPNamespaceWithNetwork applies wcp api to create a namespace in wcp cluster with the given specs. +func (w *WCPClusterProxy) CreateWCPNamespaceWithNetwork(ctx context.Context, config framework.ConfigInterface, + vmsvcSpecs wcp.VMServiceSpecDetails, + scName, wscName, nsName, artifactFolder string, network wcp.NameSpaceNetworkInfo) (NamespaceContext, error) { + // we need to create namespace with vm class and bind it + // the CR of vm class will get deleted if the namespace owning it gets deleted + // we can assume that vcenter has default vm classes names present + namespace, cancelNsWatches := wcp.CreateNamespace(ctx, wcp.NamespaceCreateInput{ + SpecName: nsName, + ClientSet: w.ClusterProxyInterface.GetClientSet(), + Client: w.ClusterProxyInterface.GetClient(), + Kubeconfig: w.ClusterProxyInterface.GetKubeconfigPath(), + StorageClassName: scName, + WorkerStorageClassName: wscName, + Config: config, + WCPClient: w.wcpAPI, + ArtifactFolder: artifactFolder, + VMServiceSpec: vmsvcSpecs, + Network: &network, + }) + + return NamespaceContext{ + namespace: namespace, + cancelNsWatches: cancelNsWatches, + }, nil +} + +// CreateWCPNamespaceWithVMReservation creates a new namespace with VM reservations. +func (w *WCPClusterProxy) CreateWCPNamespaceWithVMReservation( + ctx context.Context, + nsName, scName, zone, supervisorID string, + vmsvcSpecs wcp.VMServiceSpecDetails, + vmClassNameToReservedCount map[string]int) (NamespaceContext, error) { + namespace, cancelNsWatches := wcp.CreateNamespace(ctx, wcp.NamespaceCreateInput{ + SpecName: nsName, + WCPClient: w.wcpAPI, + Kubeconfig: w.ClusterProxyInterface.GetKubeconfigPath(), + ClientSet: w.ClusterProxyInterface.GetClientSet(), + Client: w.ClusterProxyInterface.GetClient(), + StorageClassName: scName, + Zone: zone, + SupervisorID: supervisorID, + VMServiceSpec: vmsvcSpecs, + ReservedVMClassToCount: vmClassNameToReservedCount, + }) + + return NamespaceContext{ + namespace: namespace, + cancelNsWatches: cancelNsWatches, + }, nil +} + +// DeleteWCPNamespace in WCPClusterProxy type applies wcp api to delete a namespace in wcp cluster. +func (w *WCPClusterProxy) DeleteWCPNamespace(nsCtx NamespaceContext) { + wcp.DeleteNamespace(wcp.NamespaceDeleteInput{ + Namespace: nsCtx.namespace, + WCPClient: w.wcpAPI, + CancelWatches: nsCtx.cancelNsWatches, + }) +} + +type NamespaceContext struct { + namespace *corev1.Namespace + cancelNsWatches context.CancelFunc +} + +func (n *NamespaceContext) GetNamespace() *corev1.Namespace { + return n.namespace +} + +func (n *NamespaceContext) GetCancelNsWatches() context.CancelFunc { + return n.cancelNsWatches +} + +type ZoneContext struct { + zoneList *topologyv1.ZoneList + cancelZoneWatches context.CancelFunc +} + +func (z *ZoneContext) GetZoneList() *topologyv1.ZoneList { + return z.zoneList +} + +// SimulatedWCPClusterProxy implements WCP cluster proxy interface without workloadmanagement API +// Simulated wcp proxy is running in kind environment so it includes necessary steps to create +// a wcp namespace generated by canonical kubernetes behaviors. +type SimulatedWCPClusterProxy struct { + framework.ClusterProxyInterface +} + +// Instantiate a simulated wcp cluster proxy object. +func NewSimulatedWCPClusterProxy(name string, kubeconfigPath string, scheme *runtime.Scheme) *SimulatedWCPClusterProxy { + kubeClusterProxy := framework.NewClusterProxy(name, kubeconfigPath, scheme) + + return &SimulatedWCPClusterProxy{ + ClusterProxyInterface: kubeClusterProxy, + } +} + +// Apply wraps `kubectl apply ...` and prints the output so we can see what gets applied to the cluster. +func (s *SimulatedWCPClusterProxy) applyWithArgs(ctx context.Context, resources []byte, args ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Apply") + Expect(resources).NotTo(BeNil(), "resources is required for Apply") + + return framework.KubectlApplyWithArgs(ctx, s.GetKubeconfigPath(), resources, args...) +} + +// CreateWCPNamespace in simulatedwcpcluster type implements how to create a wcp namespace in kind cluster. +func (s *SimulatedWCPClusterProxy) CreateWCPNamespaceWithNetwork(ctx context.Context, config framework.ConfigInterface, + vmsvcSpecs wcp.VMServiceSpecDetails, + scName, wscName, nsName, artifactFolder string, network wcp.NameSpaceNetworkInfo) (NamespaceContext, error) { + return NamespaceContext{}, nil +} + +// CreateWCPNamespaceWithVMReservation creates a wcp namespace in kind cluster with VM reservation. +// Note: this is needed to compile gce2e but not implemented as it's not used in kind cluster. +func (s *SimulatedWCPClusterProxy) CreateWCPNamespaceWithVMReservation( + ctx context.Context, + nsName, scName, zone, supervisorID string, + vmsvcSpecs wcp.VMServiceSpecDetails, + vmClassNameToReservedCount map[string]int) (NamespaceContext, error) { + return NamespaceContext{}, nil +} + +// CreateWCPNamespace in simulatedwcpcluster type implements how to create a wcp namespace in kind cluster. +func (s *SimulatedWCPClusterProxy) CreateWCPNamespace(ctx context.Context, config framework.ConfigInterface, + vmsvcSpecs wcp.VMServiceSpecDetails, + scName, wscName, nsName, artifactFolder string) (NamespaceContext, error) { + namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + Creator: s.ClusterProxyInterface.GetClient(), + ClientSet: s.ClusterProxyInterface.GetClientSet(), + Name: nsName, + LogFolder: filepath.Join(artifactFolder, "wcp-simulated-clusters", s.ClusterProxyInterface.GetName()), + }) + + if err := s.setUserWorkloadNamespaceLabel(ctx, config, nsName); err != nil { + return NamespaceContext{}, fmt.Errorf("failed to set user workload namespace label: %w", err) + } + + // Only set namespace resource pool and folder when FSS_WCP_FAULTDOMAINS is not enabled. + // Otherwise we would get resource pool and folder info from availability zones. + haFssEnabled := utils.IsFssEnabled(ctx, s.ClusterProxyInterface.GetClient(), + config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSHA")) + if !haFssEnabled { + err := s.setNamespaceRPAndFolder(ctx, config, nsName) + if err != nil { + return NamespaceContext{}, fmt.Errorf("failed to set namespace resource pool and folder: %w", err) + } + } else { + // Add to AZs. + err := s.setAvailabilityZones(ctx, s.ClusterProxyInterface.GetClient(), config, nsName) + if err != nil { + return NamespaceContext{}, fmt.Errorf("failed to add namespace to availability zones: %w", err) + } + } + + storageQuotaYAML, err := manifestbuilders.GetStorageQuotaYAML() + if err != nil { + return NamespaceContext{}, fmt.Errorf("get storage quota YAML file failed, please check fixture folder and manifestbuilder package") + } + + err = s.applyWithArgs(ctx, storageQuotaYAML, "-n", namespace.Name) + if err != nil { + return NamespaceContext{}, fmt.Errorf("apply storage quota to namespace %q", namespace.Name) + } + + // When FSS_WCP_NAMESPACED_VM_CLASS is enabled, + // manually create namespaced VirtualMachineClass resources, + // else, create VirtualMachineClassBinding resources in the cluster. + // Note: in a real WCP env, when we associate VM class to a namespace, VM class binding will be created accordingly. + // In a kind cluster, there is no wcpsvc, manually create them as a hack. + NamespacedVMClassFssEnabled := utils.IsFssEnabled(ctx, s.ClusterProxyInterface.GetClient(), + config.GetVariable("VMOPNamespace"), config.GetVariable("VMOPDeploymentName"), + config.GetVariable("VMOPManagerCommand"), config.GetVariable("EnvFSSNamespacedVMClass")) + + e2eframework.Logf("vmsvcSpecs: %+v", vmsvcSpecs) + + if NamespacedVMClassFssEnabled { + for _, vmClass := range vmsvcSpecs.VMClasses { + e2eframework.Logf("Create Namespaced VM class: %s", vmClass) + vmclassYAML := manifestbuilders.GetVirtualMachineClassYaml(namespace.Name, vmClass) + e2eframework.Logf("%v", string(vmclassYAML)) + + err := s.applyWithArgs(ctx, vmclassYAML, "-n", namespace.Name) + if err != nil { + return NamespaceContext{}, fmt.Errorf("apply VM class to namespace %q failed", namespace.Name) + } + } + } else { + for _, vmClass := range vmsvcSpecs.VMClasses { + e2eframework.Logf("Create VM class binding: %s", vmClass) + vmclassBindingYAML := manifestbuilders.GetVirtualMachineClassBindingYaml(namespace.Name, vmClass) + e2eframework.Logf("%v", string(vmclassBindingYAML)) + + err := s.applyWithArgs(ctx, vmclassBindingYAML, "-n", namespace.Name) + if err != nil { + return NamespaceContext{}, fmt.Errorf("apply VM class binding to namespace %q failed", namespace.Name) + } + } + } + + for _, contentSource := range vmsvcSpecs.ContentLibraries { + e2eframework.Logf("Create content source binding: %s", contentSource) + contentSourceBindingYAML := manifestbuilders.GetContentSourceBindingYaml(namespace.Name, contentSource) + e2eframework.Logf("%v", string(contentSourceBindingYAML)) + + err := s.applyWithArgs(ctx, contentSourceBindingYAML, "-n", namespace.Name) + if err != nil { + return NamespaceContext{}, fmt.Errorf("apply content source binding to namespace %q failed", namespace.Name) + } + } + + return NamespaceContext{ + namespace: namespace, + cancelNsWatches: cancelWatches, + }, nil +} + +func (s *SimulatedWCPClusterProxy) GetZonesBoundWithSupervisor(supervisorID string) (wcp.ZoneList, error) { + return wcp.ZoneList{}, errors.New("getting Zones bound with Supervisor is not implemented") +} + +func (s *SimulatedWCPClusterProxy) UpdateNamespaceWithZones(ctx context.Context, nsName string, zones []string, svClusterClient ctrlclient.Client) (ZoneContext, error) { + return ZoneContext{}, errors.New("updating namespace with Zones is not implemented") +} + +// DeleteZonesFromNamespace deletes specified zones from namespace. +func (w *SimulatedWCPClusterProxy) DeleteZonesFromNamespace(ctx context.Context, nsName string, zones []string, svClusterClient ctrlclient.Client) error { + return errors.New("deleting Zone from namespace is not implemented") +} + +func (w *SimulatedWCPClusterProxy) CreateZoneBindingsWithSupervisor(supervisorID string, zones []string) error { + return errors.New("creating zone bindings with supervisor is not implemented") +} + +func (w *SimulatedWCPClusterProxy) CreateMissingZoneBindingsWithSupervisor(supervisorID string, zones []string) error { + return errors.New("creating missing zone bindings with supervisor is not implemented") +} + +func (w *SimulatedWCPClusterProxy) DeleteZoneBindingsWithSupervisor(supervisorID string, zones []string) error { + return errors.New("deleting zone bindings with supervisor is not implemented") +} + +func (w *SimulatedWCPClusterProxy) ListVSphereZones() (wcp.VSphereZoneList, error) { + return wcp.VSphereZoneList{}, errors.New("list VSphereZones is not implemented") +} + +func (s *SimulatedWCPClusterProxy) setUserWorkloadNamespaceLabel( + ctx context.Context, + config framework.ConfigInterface, + nsName string) error { + ctrlClient := s.ClusterProxyInterface.GetClient() + vcip := utils.GetVcPNID(ctx, ctrlClient, config.GetVariable("VMOPNamespace")) + vimClient := vcenter.NewVcSimClient(ctx, vcip) + finder := vcenter.NewVcsimFinder(ctx, vimClient) + clusters := vcenter.GetClusterRefs(ctx, finder) + + namespace := &corev1.Namespace{} + + err := ctrlClient.Get(ctx, ctrlclient.ObjectKey{Name: nsName}, namespace) + if err != nil { + return err + } + + if namespace.Labels == nil { + namespace.Labels = map[string]string{} + } + // Simply set the cluster value to the first cluster in a kind setup. + namespace.Labels[userWorkloadNamespaceLabel] = clusters[0].Reference().Value + + return ctrlClient.Update(ctx, namespace) +} + +func (s *SimulatedWCPClusterProxy) setNamespaceRPAndFolder( + ctx context.Context, + config framework.ConfigInterface, + nsName string) error { + ctrlClient := s.ClusterProxyInterface.GetClient() + vcip := utils.GetVcPNID(ctx, ctrlClient, config.GetVariable("VMOPNamespace")) + vimClient := vcenter.NewVcSimClient(ctx, vcip) + finder := vcenter.NewVcsimFinder(ctx, vimClient) + clusters := vcenter.GetClusterRefs(ctx, finder) + resourcePool, folder := vcenter.GetResourcePoolAndFolder(ctx, finder, clusters[0]) + + namespace := &corev1.Namespace{} + + err := ctrlClient.Get(ctx, ctrlclient.ObjectKey{Name: nsName}, namespace) + if err != nil { + return err + } + + if namespace.Annotations == nil { + namespace.Annotations = map[string]string{} + } + + namespace.Annotations[resourcePoolNsAnnotation] = resourcePool + namespace.Annotations[vmFolderNsAnnotation] = folder + + return ctrlClient.Update(ctx, namespace) +} + +// DeleteWCPNamespace in simulatedwcpcluster type implements how to delete a wcp namespace in kind cluster. +func (s *SimulatedWCPClusterProxy) DeleteWCPNamespace(nsCtx NamespaceContext) { + ctx := context.TODO() + c := s.ClusterProxyInterface.GetClient() + // Remove from availability zones. + azs, err := utils.ListAvailabilityZones(ctx, c) + Expect(err).ShouldNot(HaveOccurred()) + + for _, az := range azs.Items { + item := az + delete(item.Spec.Namespaces, nsCtx.namespace.Name) + Expect(c.Update(ctx, &item)).ShouldNot(HaveOccurred()) + } + + framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ + Deleter: c, + Name: nsCtx.namespace.Name, + }) + nsCtx.cancelNsWatches() +} + +func (s *SimulatedWCPClusterProxy) setAvailabilityZones(ctx context.Context, + c ctrlclient.Client, + config framework.ConfigInterface, + ns string) error { + azs, err := utils.ListAvailabilityZones(ctx, c) + if err != nil { + return err + } + + ctrlClient := s.ClusterProxyInterface.GetClient() + vcip := utils.GetVcPNID(ctx, ctrlClient, config.GetVariable("VMOPNamespace")) + vimClient := vcenter.NewVcSimClient(ctx, vcip) + finder := vcenter.NewVcsimFinder(ctx, vimClient) + + clusterRefs := vcenter.GetClusterRefs(ctx, finder) + refMapping := vcenter.GetClusterIDToRefMapping(clusterRefs) + + for _, az := range azs.Items { + item := az + + Expect(item.Spec.ClusterComputeResourceMoIDs).ToNot(BeEmpty()) + clusterID := item.Spec.ClusterComputeResourceMoIDs[0] + + rp, folder := vcenter.GetResourcePoolAndFolder(ctx, finder, refMapping[clusterID]) + + if item.Spec.Namespaces == nil { + item.Spec.Namespaces = map[string]topologyv1.NamespaceInfo{} + } + + item.Spec.Namespaces[ns] = topologyv1.NamespaceInfo{ + PoolMoId: rp, + PoolMoIDs: []string{rp}, + FolderMoId: folder, + } + Expect(c.Update(ctx, &item)).ShouldNot(HaveOccurred()) + } + + return nil +} diff --git a/webhooks/common/response_test.go b/webhooks/common/response_test.go index fe5b1c77a..3e6eec444 100644 --- a/webhooks/common/response_test.go +++ b/webhooks/common/response_test.go @@ -80,7 +80,7 @@ var _ = Describe( DescribeTable("", wellKnownError, Entry("NotFound", apierrors.NewNotFound(gr, ""), http.StatusNotFound), - Entry("Gone", apierrors.NewGone("gone"), http.StatusGone), + Entry("Gone", apierrors.NewResourceExpired("gone"), http.StatusGone), Entry("ResourceExpired", apierrors.NewResourceExpired("expired"), http.StatusGone), Entry("ServiceUnavailable", apierrors.NewServiceUnavailable("unavailable"), http.StatusServiceUnavailable), Entry("ServiceUnavailable", apierrors.NewServiceUnavailable("unavailable"), http.StatusServiceUnavailable), diff --git a/webhooks/conversion/v1alpha1/webhooks.go b/webhooks/conversion/v1alpha1/webhooks.go index dae974c33..06db1a5da 100644 --- a/webhooks/conversion/v1alpha1/webhooks.go +++ b/webhooks/conversion/v1alpha1/webhooks.go @@ -1,5 +1,5 @@ // © Broadcom. All Rights Reserved. -// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -14,43 +14,37 @@ import ( func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a1.VirtualMachine{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a1.VirtualMachine{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a1.VirtualMachineClass{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a1.VirtualMachineClass{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a1.VirtualMachineImage{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a1.VirtualMachineImage{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a1.VirtualMachinePublishRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a1.VirtualMachinePublishRequest{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a1.VirtualMachineService{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a1.VirtualMachineService{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a1.VirtualMachineSetResourcePolicy{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a1.VirtualMachineSetResourcePolicy{}). Complete(); err != nil { return err diff --git a/webhooks/conversion/v1alpha2/webhooks.go b/webhooks/conversion/v1alpha2/webhooks.go index 7b4002787..c73cc40fb 100644 --- a/webhooks/conversion/v1alpha2/webhooks.go +++ b/webhooks/conversion/v1alpha2/webhooks.go @@ -1,5 +1,5 @@ // © Broadcom. All Rights Reserved. -// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // SPDX-License-Identifier: Apache-2.0 package v1alpha2 @@ -14,50 +14,43 @@ import ( func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a2.VirtualMachine{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a2.VirtualMachine{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a2.VirtualMachineClass{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a2.VirtualMachineClass{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a2.VirtualMachineImage{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a2.VirtualMachineImage{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a2.VirtualMachinePublishRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a2.VirtualMachinePublishRequest{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a2.VirtualMachineService{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a2.VirtualMachineService{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a2.VirtualMachineSetResourcePolicy{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a2.VirtualMachineSetResourcePolicy{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a2.VirtualMachineWebConsoleRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a2.VirtualMachineWebConsoleRequest{}). Complete(); err != nil { return err diff --git a/webhooks/conversion/v1alpha3/webhooks.go b/webhooks/conversion/v1alpha3/webhooks.go index 1498eac9e..ecc596f29 100644 --- a/webhooks/conversion/v1alpha3/webhooks.go +++ b/webhooks/conversion/v1alpha3/webhooks.go @@ -1,5 +1,5 @@ // © Broadcom. All Rights Reserved. -// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // SPDX-License-Identifier: Apache-2.0 package v1alpha3 @@ -14,57 +14,49 @@ import ( func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a3.VirtualMachine{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a3.VirtualMachine{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a3.VirtualMachineClass{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a3.VirtualMachineClass{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a3.VirtualMachineImage{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a3.VirtualMachineImage{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a3.ClusterVirtualMachineImage{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a3.ClusterVirtualMachineImage{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a3.VirtualMachinePublishRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a3.VirtualMachinePublishRequest{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a3.VirtualMachineService{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a3.VirtualMachineService{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a3.VirtualMachineSetResourcePolicy{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a3.VirtualMachineSetResourcePolicy{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a3.VirtualMachineWebConsoleRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a3.VirtualMachineWebConsoleRequest{}). Complete(); err != nil { return err diff --git a/webhooks/conversion/v1alpha4/webhooks.go b/webhooks/conversion/v1alpha4/webhooks.go index e37c30f6b..d8f782e4b 100644 --- a/webhooks/conversion/v1alpha4/webhooks.go +++ b/webhooks/conversion/v1alpha4/webhooks.go @@ -1,5 +1,5 @@ // © Broadcom. All Rights Reserved. -// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // SPDX-License-Identifier: Apache-2.0 package v1alpha4 @@ -9,62 +9,63 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" vmopv1a4 "github.com/vmware-tanzu/vm-operator/api/v1alpha4" + pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config" pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context" ) func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a4.VirtualMachine{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a4.VirtualMachine{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a4.VirtualMachineClass{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a4.VirtualMachineClass{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a4.VirtualMachineImage{}). + if pkgcfg.FromContext(ctx).Features.ImmutableClasses { + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a4.VirtualMachineClassInstance{}). + Complete(); err != nil { + + return err + } + } + + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a4.VirtualMachineImage{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a4.ClusterVirtualMachineImage{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a4.ClusterVirtualMachineImage{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a4.VirtualMachinePublishRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a4.VirtualMachinePublishRequest{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a4.VirtualMachineService{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a4.VirtualMachineService{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a4.VirtualMachineSetResourcePolicy{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a4.VirtualMachineSetResourcePolicy{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1a4.VirtualMachineWebConsoleRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a4.VirtualMachineWebConsoleRequest{}). Complete(); err != nil { return err diff --git a/webhooks/conversion/v1alpha5/webhooks.go b/webhooks/conversion/v1alpha5/webhooks.go index e23682612..842b268be 100644 --- a/webhooks/conversion/v1alpha5/webhooks.go +++ b/webhooks/conversion/v1alpha5/webhooks.go @@ -1,5 +1,5 @@ // © Broadcom. All Rights Reserved. -// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // SPDX-License-Identifier: Apache-2.0 package v1alpha5 @@ -8,70 +8,94 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" - vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" + vmopv1a5 "github.com/vmware-tanzu/vm-operator/api/v1alpha5" + pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config" pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context" ) func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachine{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.ClusterVirtualMachineImage{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineClass{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachine{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineImage{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineClass{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.ClusterVirtualMachineImage{}). + if pkgcfg.FromContext(ctx).Features.ImmutableClasses { + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineClassInstance{}). + Complete(); err != nil { + + return err + } + } + + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineGroup{}). + Complete(); err != nil { + + return err + } + + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineGroupPublishRequest{}). + Complete(); err != nil { + + return err + } + + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineImage{}). + Complete(); err != nil { + + return err + } + + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineImageCache{}). + Complete(); err != nil { + + return err + } + + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachinePublishRequest{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineImageCache{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineReplicaSet{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachinePublishRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineService{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineService{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineSetResourcePolicy{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineSetResourcePolicy{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineSnapshot{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineWebConsoleRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1a5.VirtualMachineWebConsoleRequest{}). Complete(); err != nil { return err diff --git a/webhooks/conversion/v1alpha6/webhooks.go b/webhooks/conversion/v1alpha6/webhooks.go index a2c355262..b0c720d5a 100644 --- a/webhooks/conversion/v1alpha6/webhooks.go +++ b/webhooks/conversion/v1alpha6/webhooks.go @@ -9,104 +9,93 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" + pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config" pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context" ) func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.ClusterVirtualMachineImage{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.ClusterVirtualMachineImage{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachine{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachine{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineClass{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineClass{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineClassInstance{}). + if pkgcfg.FromContext(ctx).Features.ImmutableClasses { + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineClassInstance{}). Complete(); err != nil { - return err + return err + } } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineGroup{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineGroup{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineGroupPublishRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineGroupPublishRequest{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineImage{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineImage{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineImageCache{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineImageCache{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachinePublishRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachinePublishRequest{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineReplicaSet{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineReplicaSet{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineService{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineService{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineSetResourcePolicy{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineSetResourcePolicy{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineSnapshot{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineSnapshot{}). Complete(); err != nil { return err } - if err := ctrl.NewWebhookManagedBy(mgr). - For(&vmopv1.VirtualMachineWebConsoleRequest{}). + if err := ctrl.NewWebhookManagedBy(mgr, &vmopv1.VirtualMachineWebConsoleRequest{}). Complete(); err != nil { return err diff --git a/webhooks/unifiedstoragequota/validation/unifiedstoragequota_validator.go b/webhooks/unifiedstoragequota/validation/unifiedstoragequota_validator.go index 4ef86800b..3ac33451e 100644 --- a/webhooks/unifiedstoragequota/validation/unifiedstoragequota_validator.go +++ b/webhooks/unifiedstoragequota/validation/unifiedstoragequota_validator.go @@ -98,9 +98,9 @@ type VMSnapshotRequestedCapacityHandler struct { // AddToManager adds the webhook to the provided manager. func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) error { - webhookNameLong := fmt.Sprintf("%s/%s/%s", ctx.Namespace, ctx.Name, webhookName) logger := ctx.Logger.WithName(webhookName) + webhookNameLong := fmt.Sprintf("%s/%s/%s", ctx.Namespace, ctx.Name, webhookName) // Build the webhookContext. webhookContext := &pkgctx.WebhookContext{ @@ -108,7 +108,7 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr ctrlmgr.Manager) err Name: webhookName, Namespace: ctx.Namespace, ServiceAccountName: ctx.ServiceAccountName, - Recorder: record.New(mgr.GetEventRecorderFor(webhookNameLong)), + Recorder: record.New(mgr.GetEventRecorder(webhookNameLong)), Logger: logger, } // Initialize the webhook's decoder. diff --git a/webhooks/virtualmachine/mutation/virtualmachine_mutator_network_interface_type.go b/webhooks/virtualmachine/mutation/virtualmachine_mutator_network_interface_type.go index e814d7de1..c11976d6c 100644 --- a/webhooks/virtualmachine/mutation/virtualmachine_mutator_network_interface_type.go +++ b/webhooks/virtualmachine/mutation/virtualmachine_mutator_network_interface_type.go @@ -18,7 +18,7 @@ func mutateOnCreateDefaultNetworkInterfaceType( client ctrlclient.Client, vm *vmopv1.VirtualMachine) (bool, error) { - if !pkgcfg.FromContext(ctx).Features.VMExtraConfig { + if !pkgcfg.FromContext(ctx).Features.TelcoVMServiceAPI { return false, nil } return SetDefaultNetworkInterfaceTypesOnCreate(ctx, client, vm) @@ -29,7 +29,7 @@ func mutateOnUpdateDefaultNetworkInterfaceType( client ctrlclient.Client, newVM, oldVM *vmopv1.VirtualMachine) (bool, error) { - if !pkgcfg.FromContext(ctx).Features.VMExtraConfig { + if !pkgcfg.FromContext(ctx).Features.TelcoVMServiceAPI { return false, nil } return SetDefaultNetworkInterfaceTypesOnUpdate(ctx, client, newVM, oldVM) diff --git a/webhooks/virtualmachine/mutation/virtualmachine_mutator_network_interface_type_test.go b/webhooks/virtualmachine/mutation/virtualmachine_mutator_network_interface_type_test.go index fbdbbcf24..3d8fffd1f 100644 --- a/webhooks/virtualmachine/mutation/virtualmachine_mutator_network_interface_type_test.go +++ b/webhooks/virtualmachine/mutation/virtualmachine_mutator_network_interface_type_test.go @@ -2,55 +2,67 @@ // The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // SPDX-License-Identifier: Apache-2.0 -package mutation +package mutation_test import ( - "testing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" + "github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels" pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/mutation" ) -func TestSetDefaultNetworkInterfaceTypesOnCreate_DefaultsVMXNet3(t *testing.T) { - t.Parallel() - ctx := &pkgctx.WebhookRequestContext{WebhookContext: &pkgctx.WebhookContext{}} - vm := &vmopv1.VirtualMachine{} - vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ - Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{{Name: "eth0"}}, - } - ok, err := SetDefaultNetworkInterfaceTypesOnCreate(ctx, nil, vm) - if err != nil { - t.Fatal(err) - } - if !ok { - t.Fatal("expected mutation") - } - if vm.Spec.Network.Interfaces[0].Type != vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3 { - t.Fatalf("got %q", vm.Spec.Network.Interfaces[0].Type) - } -} - -func TestSetDefaultNetworkInterfaceTypesOnUpdate_PreservesOldType(t *testing.T) { - t.Parallel() - ctx := &pkgctx.WebhookRequestContext{WebhookContext: &pkgctx.WebhookContext{}} - newVM := &vmopv1.VirtualMachine{} - newVM.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ - Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{{Name: "eth0"}}, - } - oldVM := &vmopv1.VirtualMachine{} - oldVM.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ - Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ - {Name: "eth0", Type: vmopv1.VirtualMachineNetworkInterfaceTypeSRIOV}, - }, - } - ok, err := SetDefaultNetworkInterfaceTypesOnUpdate(ctx, nil, newVM, oldVM) - if err != nil { - t.Fatal(err) - } - if !ok { - t.Fatal("expected mutation") - } - if newVM.Spec.Network.Interfaces[0].Type != vmopv1.VirtualMachineNetworkInterfaceTypeSRIOV { - t.Fatalf("got %q", newVM.Spec.Network.Interfaces[0].Type) - } -} +var _ = Describe( + "SetDefaultNetworkInterfaceTypes", + Label( + testlabels.API, + testlabels.Mutation, + testlabels.Webhook, + ), + func() { + var ( + ctx *pkgctx.WebhookRequestContext + ) + + BeforeEach(func() { + ctx = &pkgctx.WebhookRequestContext{WebhookContext: &pkgctx.WebhookContext{}} + }) + + Context("OnCreate", func() { + It("should default to VMXNet3 when interface type is empty", func() { + vm := &vmopv1.VirtualMachine{} + vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{{Name: "eth0"}}, + } + + ok, err := mutation.SetDefaultNetworkInterfaceTypesOnCreate(ctx, nil, vm) + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeTrue(), "expected mutation to occur") + Expect(vm.Spec.Network.Interfaces[0].Type).To(Equal(vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3)) + }) + }) + + Context("OnUpdate", func() { + It("should preserve existing interface type from old VM", func() { + newVM := &vmopv1.VirtualMachine{} + newVM.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{{Name: "eth0"}}, + } + + oldVM := &vmopv1.VirtualMachine{} + oldVM.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + {Name: "eth0", Type: vmopv1.VirtualMachineNetworkInterfaceTypeSRIOV}, + }, + } + + ok, err := mutation.SetDefaultNetworkInterfaceTypesOnUpdate(ctx, nil, newVM, oldVM) + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeTrue(), "expected mutation to occur") + Expect(newVM.Spec.Network.Interfaces[0].Type).To(Equal(vmopv1.VirtualMachineNetworkInterfaceTypeSRIOV)) + }) + }) + }, +) diff --git a/webhooks/virtualmachine/validation/virtualmachine_validator.go b/webhooks/virtualmachine/validation/virtualmachine_validator.go index b01dc06b1..21394f354 100644 --- a/webhooks/virtualmachine/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/validation/virtualmachine_validator.go @@ -13,6 +13,7 @@ import ( "slices" "strconv" "strings" + "sync" "time" "github.com/google/uuid" @@ -35,6 +36,7 @@ import ( vpcv1alpha1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" + vmopv1common "github.com/vmware-tanzu/vm-operator/api/v1alpha6/common" "github.com/vmware-tanzu/vm-operator/api/v1alpha6/sysprep" "github.com/vmware-tanzu/vm-operator/pkg/builder" pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config" @@ -42,6 +44,7 @@ import ( pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context" pkglog "github.com/vmware-tanzu/vm-operator/pkg/log" "github.com/vmware-tanzu/vm-operator/pkg/providers/vsphere/config" + vsphereconst "github.com/vmware-tanzu/vm-operator/pkg/providers/vsphere/constants" "github.com/vmware-tanzu/vm-operator/pkg/topology" pkgutil "github.com/vmware-tanzu/vm-operator/pkg/util" cloudinitvalidate "github.com/vmware-tanzu/vm-operator/pkg/util/cloudinit/validate" @@ -99,6 +102,108 @@ const ( guestCustomizationVCDParityNotEnabled = "VC guest customization VCD parity capability is not enabled" bootstrapProviderTypeCannotBeChanged = "bootstrap provider type cannot be changed" forbiddenRemovableVolume = "cannot remove volume with removable=false" + + // ExtraConfig validation error messages. + extraConfigUseFirstClassFieldFmt = "%s: use the corresponding first-class field in %s instead" + extraConfigReservedForSystemFmt = "%s: this key is reserved for the system" + extraConfigUseSpecNetworkInterfacesFmt = "%s: use spec.network.interfaces[].vmxnet3 or advancedProperties instead" + extraConfigUseBareKeyNameFmt = "%s: use the bare key name without the network device prefix" +) + +var ( + ethernetDeviceKeyRE = regexp.MustCompile(`^ethernet\d+\.(.*)$`) + capvDefaultServiceAccount = regexp.MustCompile("^system:serviceaccount:svc-tkg-domain-[^:]+:default$") + + // firstClassVMAdvancedProperties is the set of vmx keys from first-class VM advanced fields. + firstClassVMAdvancedProperties = sync.OnceValue(func() map[string]bool { + result := make(map[string]bool) + specType := reflect.TypeOf((*vmopv1.VirtualMachineAdvancedSpec)(nil)).Elem() + for i := 0; i < specType.NumField(); i++ { + field := specType.Field(i) + if vmxTag, ok := field.Tag.Lookup("vmx"); ok && vmxTag != "" { + result[vmxTag] = true + } + } + return result + }) + + // firstClassNICAdvancedProperties is the set of vmx key suffixes after ethernet%d. on first-class NIC fields. + firstClassNICAdvancedProperties = sync.OnceValue(func() map[string]bool { + specType := reflect.TypeOf((*vmopv1.VirtualMachineNetworkInterfaceVMXNet3Spec)(nil)).Elem() + out := make(map[string]bool) + const ethernetPlaceholder = "ethernet%d." + for i := 0; i < specType.NumField(); i++ { + f := specType.Field(i) + vmxTag, ok := f.Tag.Lookup("vmx") + if !ok || vmxTag == "" { + continue + } + if !strings.HasPrefix(vmxTag, ethernetPlaceholder) { + continue + } + suffix := strings.TrimPrefix(vmxTag, ethernetPlaceholder) + if suffix == "" { + continue + } + out[suffix] = true + } + return out + }) + + // systemReservedNetworkDeviceProperties is the set of ethernet device suffixes reserved by the system. + systemReservedNetworkDeviceProperties = map[string]bool{ + "address": true, + "addresstype": true, + "allowguestconnectioncontrol": true, + "devname": true, + "dvs.connectionid": true, + "dvs.portgroupid": true, + "dvs.portid": true, + "dvs.switchid": true, + "externalid": true, + "filename": true, + "generatedaddress": true, + "key": true, + "limit": true, + "measurelatency": true, + "migrateconnect": true, + "name": true, + "networkname": true, + "opaquenetwork.id": true, + "opaquenetwork.type": true, + "present": true, + "pxm": true, + "realtime": true, + "reservation": true, + "rtdisableoffload": true, + "rtmaxrxqueues": true, + "rtmaxtxqueues": true, + "rtrxdataringdescsize": true, + "rttxdataringdescsize": true, + "shares": true, + "startconnected": true, + "upt": true, + "uptcompatibility": true, + "virtualdev": true, + "vnet": true, + "wakeonpcktrcv": true, + } + + // systemReservedExtraConfigKeys is the set of exact extraConfig keys reserved by the system. + systemReservedExtraConfigKeys = map[string]bool{ + vsphereconst.ExtraConfigReservedKeyVMXRebootPowerCycle: true, + vsphereconst.ExtraConfigReservedProfileID: true, + vsphereconst.ExtraConfigRunContainerKey: true, + vsphereconst.ExtraConfigVMServiceNamespacedName: true, + vsphereconst.GOSCPendingExtraConfigKey: true, + vsphereconst.GOSCIgnoreToolsCheckExtraConfigKey: true, + } + + // systemReservedExtraConfigPrefixes is the set of extraConfig key prefixes reserved by the system. + systemReservedExtraConfigPrefixes = []string{ + vsphereconst.ExtraConfigReservedPrefixVMService, + vsphereconst.ExtraConfigGuestInfoPrefix, + } ) // +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha6-virtualmachine,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachines,versions=v1alpha6,name=default.validating.virtualmachine.v1alpha6.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 @@ -784,13 +889,13 @@ func (v validator) validateNetwork( p := networkPath.Child("interfaces") for i, interfaceSpec := range networkSpec.Interfaces { - allErrs = append(allErrs, v.validateNetworkInterfaceSpec(p.Index(i), interfaceSpec, vm.Name)...) + allErrs = append(allErrs, v.validateNetworkInterfaceSpec(ctx, p.Index(i), interfaceSpec, vm.Name)...) allErrs = append(allErrs, v.validateNetworkInterfaceSpecWithBootstrap(ctx, p.Index(i), interfaceSpec, vm)...) } } if len(networkSpec.VLANs) > 0 { - allErrs = append(allErrs, v.validateNetworkVLANs(networkPath.Child("vlans"), vm)...) + allErrs = append(allErrs, v.validateNetworkVLANs(ctx, networkPath.Child("vlans"), vm)...) } if oldVM != nil { @@ -853,6 +958,7 @@ func (v validator) validateNetworkInterfaceMacAddressNotChanged( // validateNetworkVLANs validates the VLANs configuration in the network spec. // VLANs are only supported with CloudInit bootstrap provider. func (v validator) validateNetworkVLANs( + ctx *pkgctx.WebhookRequestContext, vlansPath *field.Path, vm *vmopv1.VirtualMachine) field.ErrorList { @@ -860,6 +966,13 @@ func (v validator) validateNetworkVLANs( networkSpec := vm.Spec.Network + if !pkgcfg.FromContext(ctx).Features.VMVlanSubinterface { + allErrs = append(allErrs, field.Forbidden( + vlansPath, fmt.Sprintf(featureNotEnabled, "VLAN Sub Interface"), + )) + return allErrs + } + if vm.Spec.Bootstrap == nil || vm.Spec.Bootstrap.CloudInit == nil { allErrs = append(allErrs, field.Forbidden( vlansPath, "vlans is available only with the following bootstrap providers: CloudInit", @@ -939,6 +1052,7 @@ var macAddressSupportNetworkGroups = []string{ //nolint:gocyclo func (v validator) validateNetworkInterfaceSpec( + ctx *pkgctx.WebhookRequestContext, interfacePath *field.Path, interfaceSpec vmopv1.VirtualMachineNetworkInterfaceSpec, vmName string) field.ErrorList { @@ -1084,6 +1198,21 @@ func (v validator) validateNetworkInterfaceSpec( } } + if pkgcfg.FromContext(ctx).Features.TelcoVMServiceAPI && len(interfaceSpec.AdvancedProperties) > 0 { + allErrs = append(allErrs, v.validateNetworkInterfaceAdvancedProperties( + interfacePath.Child("advancedProperties"), interfaceSpec.AdvancedProperties)...) + } + + // Validate VMXNet3 configuration if present + if interfaceSpec.VMXNet3 != nil { + if interfaceSpec.Type != vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3 { + allErrs = append(allErrs, field.Forbidden(interfacePath.Child("vmxnet3"), + "vmxnet3 configuration is only allowed when type is VMXNet3")) + } else { + allErrs = append(allErrs, v.validateVMXNet3Spec(interfacePath.Child("vmxnet3"), interfaceSpec.VMXNet3)...) + } + } + return allErrs } @@ -1145,6 +1274,96 @@ func (v validator) validateNetworkSpecWithBootStrap( return allErrs } +// validateNetworkInterfaceAdvancedProperties validates spec.network.interfaces[].advancedProperties. +// Rejects keys that duplicate first-class VMX keys or reserved/device-scoped prefixes. +func (v validator) validateNetworkInterfaceAdvancedProperties( + advancedPropsPath *field.Path, + advancedProps []vmopv1common.KeyValuePair) field.ErrorList { + + var allErrs field.ErrorList + + for i, kv := range advancedProps { + keyPath := advancedPropsPath.Index(i).Child("key") + + if isFirstClassNICAdvancedProperty(kv.Key) { + allErrs = append(allErrs, field.Forbidden(keyPath, + fmt.Sprintf(extraConfigUseFirstClassFieldFmt, kv.Key, "spec.network.interfaces[].vmxnet3"))) + continue + } + + if isNetworkDeviceProperty(kv.Key) { + allErrs = append(allErrs, field.Forbidden(keyPath, + fmt.Sprintf(extraConfigUseBareKeyNameFmt, kv.Key))) + continue + } + + // Reject reserved prefixes that system controls for network devices. + if isSystemReservedNetworkDeviceProperty(kv.Key) { + allErrs = append(allErrs, field.Forbidden(keyPath, + fmt.Sprintf(extraConfigReservedForSystemFmt, kv.Key))) + continue + } + } + + return allErrs +} + +func (v validator) validateVMXNet3Spec( + vmxnet3Path *field.Path, + vmxnet3Spec *vmopv1.VirtualMachineNetworkInterfaceVMXNet3Spec) field.ErrorList { + + var allErrs field.ErrorList + + // Validate PNICFeatures for power-of-2 integers + if vmxnet3Spec.PNICFeatures != nil { + pnicPath := vmxnet3Path.Child("pnicFeatures") + for i, feature := range vmxnet3Spec.PNICFeatures { + featurePath := pnicPath.Index(i) + + // Check if it's a known enum value + switch feature { + case vmopv1.PNICQueueFeatureLargeReceiveOffload, vmopv1.PNICQueueFeatureReceiveSideScaling: + // Valid enum values + continue + default: + // Check if it's a valid power-of-2 integer (1, 2, 4, 8, 16, 32, etc.) + if isPowerOf2String(string(feature)) { + continue + } + // Invalid value + allErrs = append(allErrs, field.Invalid(featurePath, feature, + "must be a known enum value (LargeReceiveOffload, ReceiveSideScaling) or a power-of-2 integer (1,2,4,8,16,32,...)")) + } + } + } + + // Validate CoalescingParams when CoalescingScheme is RateBasedCoalescing + if vmxnet3Spec.CoalescingScheme != nil && vmxnet3Spec.CoalescingParams != nil { + if *vmxnet3Spec.CoalescingScheme == vmopv1.CoalescingSchemeRateBasedCoalescing { + // For RateBasedCoalescing, coalescingParams must be a valid unsigned 32-bit integer + if _, err := strconv.ParseUint(*vmxnet3Spec.CoalescingParams, 10, 32); err != nil { + coalescingPath := vmxnet3Path.Child("coalescingParams") + allErrs = append(allErrs, field.Invalid(coalescingPath, *vmxnet3Spec.CoalescingParams, + "must be a valid 32-bit unsigned integer when coalescingScheme is RateBasedCoalescing")) + } + } + } + + return allErrs +} + +// isPowerOf2String checks if a string represents a power-of-2 integer. +func isPowerOf2String(s string) bool { + // Parse as integer + n, err := strconv.Atoi(s) + if err != nil || n <= 0 { + return false + } + + // Check if it's a power of 2: n > 0 && (n & (n-1)) == 0 + return (n & (n - 1)) == 0 +} + // MTU, routes, and searchDomains are available only with CloudInit. // Nameservers is available only with CloudInit and Sysprep. func (v validator) validateNetworkInterfaceSpecWithBootstrap( @@ -1840,7 +2059,7 @@ func (v validator) validateReadinessProbe( var megaByte = resource.MustParse("1Mi") func (v validator) validateAdvanced( - _ *pkgctx.WebhookRequestContext, + ctx *pkgctx.WebhookRequestContext, vm *vmopv1.VirtualMachine) field.ErrorList { var allErrs field.ErrorList @@ -1859,6 +2078,46 @@ func (v validator) validateAdvanced( } } + // Validate extraConfig when TelcoVMServiceAPI is enabled + if pkgcfg.FromContext(ctx).Features.TelcoVMServiceAPI { + allErrs = append(allErrs, v.validateAdvancedExtraConfig(ctx, advancedPath, advanced)...) + } + + return allErrs +} + +// validateAdvancedExtraConfig validates spec.advanced.extraConfig keys. +func (v validator) validateAdvancedExtraConfig( + _ *pkgctx.WebhookRequestContext, + advancedPath *field.Path, + advanced *vmopv1.VirtualMachineAdvancedSpec) field.ErrorList { + + var allErrs field.ErrorList + if len(advanced.ExtraConfig) == 0 { + return allErrs + } + + ecPath := advancedPath.Child("extraConfig") + for i, kv := range advanced.ExtraConfig { + keyPath := ecPath.Index(i).Child("key") + + if isFirstClassVMAdvancedProperty(kv.Key) { + allErrs = append(allErrs, field.Forbidden(keyPath, + fmt.Sprintf(extraConfigUseFirstClassFieldFmt, kv.Key, "spec.advanced"))) + continue + } + if isNetworkDeviceProperty(kv.Key) { + allErrs = append(allErrs, field.Forbidden(keyPath, + fmt.Sprintf(extraConfigUseSpecNetworkInterfacesFmt, kv.Key))) + continue + } + if isSystemReservedProperty(kv.Key) { + allErrs = append(allErrs, field.Forbidden(keyPath, + fmt.Sprintf(extraConfigReservedForSystemFmt, kv.Key))) + continue + } + } + return allErrs } @@ -2581,8 +2840,6 @@ func (v validator) validateCdromWhenPoweredOn( return allErrs } -var capvDefaultServiceAccount = regexp.MustCompile("^system:serviceaccount:svc-tkg-domain-[^:]+:default$") - // isCAPVServiceAccount checks if the username matches that of the CAPV service account. func isCAPVServiceAccount(username string) bool { return capvDefaultServiceAccount.Match([]byte(username)) @@ -2963,9 +3220,20 @@ func (v validator) validateVMAffinity( } } - if rs.TopologyKey != corev1.LabelTopologyZone { - allErrs = append(allErrs, field.NotSupported( - p.Child("topologyKey"), rs.TopologyKey, []string{corev1.LabelTopologyZone})) + if pkgcfg.FromContext(ctx).Features.VMAffinityDuringExecution { + // When VMAffinityDuringExecution capability is enabled, + // either zone or host topology key is allowed for affinity required terms. + if rs.TopologyKey != corev1.LabelTopologyZone && rs.TopologyKey != corev1.LabelHostname { + allErrs = append(allErrs, field.NotSupported( + p.Child("topologyKey"), rs.TopologyKey, []string{corev1.LabelTopologyZone, corev1.LabelHostname})) + } + } else { + // Without VMAffinityDuringExecution capability enabled, + // only zone topology key is allowed for affinity required terms. + if rs.TopologyKey != corev1.LabelTopologyZone { + allErrs = append(allErrs, field.NotSupported( + p.Child("topologyKey"), rs.TopologyKey, []string{corev1.LabelTopologyZone})) + } } } } @@ -2997,9 +3265,20 @@ func (v validator) validateVMAffinity( } } - if rs.TopologyKey != corev1.LabelTopologyZone { - allErrs = append(allErrs, field.NotSupported( - p.Child("topologyKey"), rs.TopologyKey, []string{corev1.LabelTopologyZone})) + if pkgcfg.FromContext(ctx).Features.VMAffinityDuringExecution { + // When VMAffinityDuringExecution capability is enabled, + // either zone or host topology key is allowed for affinity required terms. + if rs.TopologyKey != corev1.LabelTopologyZone && rs.TopologyKey != corev1.LabelHostname { + allErrs = append(allErrs, field.NotSupported( + p.Child("topologyKey"), rs.TopologyKey, []string{corev1.LabelTopologyZone, corev1.LabelHostname})) + } + } else { + // Without VMAffinityDuringExecution capability enabled, + // only zone topology key is allowed for affinity required terms. + if rs.TopologyKey != corev1.LabelTopologyZone { + allErrs = append(allErrs, field.NotSupported( + p.Child("topologyKey"), rs.TopologyKey, []string{corev1.LabelTopologyZone})) + } } } } @@ -3193,3 +3472,44 @@ func (v validator) validateBiosUUID(_ *pkgctx.WebhookRequestContext, vm *vmopv1. return allErrs } + +func isFirstClassVMAdvancedProperty(key string) bool { + return firstClassVMAdvancedProperties()[key] +} + +// isSystemReservedProperty returns true if the key is reserved and controlled by the system. +func isSystemReservedProperty(key string) bool { + // Check exact keys first (fastest lookup) + if systemReservedExtraConfigKeys[key] { + return true + } + + // Check prefixes + for _, prefix := range systemReservedExtraConfigPrefixes { + if strings.HasPrefix(key, prefix) { + return true + } + } + + return false +} + +func isSystemReservedNetworkDeviceProperty(key string) bool { + return systemReservedNetworkDeviceProperties[key] +} + +func isFirstClassNICAdvancedProperty(key string) bool { + firstClassProperties := firstClassNICAdvancedProperties() + if firstClassProperties[key] { + return true + } + if m := ethernetDeviceKeyRE.FindStringSubmatch(key); m != nil { + _, ok := firstClassProperties[m[1]] + return ok + } + return false +} + +func isNetworkDeviceProperty(key string) bool { + return ethernetDeviceKeyRE.MatchString(key) +} diff --git a/webhooks/virtualmachine/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/validation/virtualmachine_validator_unit_test.go index b134a4499..8ce8ffbc8 100644 --- a/webhooks/virtualmachine/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/validation/virtualmachine_validator_unit_test.go @@ -110,6 +110,31 @@ func doValidateWithMsg(msgs ...string) func(admission.Response) { } } +// doTestWithContext runs a table-style validating webhook test using the provided ctx. +// It calls ValidateUpdate when ctx.oldVM is set, otherwise ValidateCreate. Use when a nested Context owns a +// dedicated ctx and the outer commonCreateAndUpdateValidations doTest closure would bind the wrong ctx. +func doTestWithContext(ctx *unitValidatingWebhookContext, args testParams) { + args.setup(ctx) + + var err error + ctx.WebhookRequestContext.Obj, err = builder.ToUnstructured(ctx.vm) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + + var response admission.Response + if ctx.oldVM != nil { + ctx.WebhookRequestContext.OldObj, err = builder.ToUnstructured(ctx.oldVM) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + response = ctx.ValidateUpdate(&ctx.WebhookRequestContext) + } else { + response = ctx.ValidateCreate(&ctx.WebhookRequestContext) + } + ExpectWithOffset(1, response.Allowed).To(Equal(args.expectAllowed)) + + if args.validate != nil { + args.validate(response) + } +} + func unitTests() { Describe( "Create", @@ -2855,9 +2880,45 @@ func unitTestsValidateCreate() { }, ), + Entry("disallow spec.crypto when FeatureGate VMVlanSubinterface is disabled (by default)", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMVlanSubinterface = false + }) + ctx.vm.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ + CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, + } + ctx.vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + }, + { + Name: "eth1", + }, + }, + VLANs: []vmopv1.VirtualMachineNetworkVLANSpec{ + { + Name: "vlan100a", + ID: 100, + Link: "eth1", + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.network.vlans: Forbidden: the VLAN Sub Interface feature is not enabled`, + ), + }, + ), + Entry("allow valid VLANs parameter", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMVlanSubinterface = true + }) ctx.vm.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, } @@ -2899,6 +2960,9 @@ func unitTestsValidateCreate() { Entry("disallow VLANs without CloudInit bootstrap", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMVlanSubinterface = true + }) ctx.vm.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ LinuxPrep: &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{}, } @@ -2926,6 +2990,9 @@ func unitTestsValidateCreate() { Entry("disallow VLANs without any bootstrap", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMVlanSubinterface = true + }) ctx.vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ { @@ -2950,6 +3017,9 @@ func unitTestsValidateCreate() { Entry("disallow VLAN with invalid link reference", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMVlanSubinterface = true + }) ctx.vm.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, } @@ -2977,6 +3047,9 @@ func unitTestsValidateCreate() { Entry("disallow VLAN with empty link", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMVlanSubinterface = true + }) ctx.vm.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, } @@ -3004,6 +3077,9 @@ func unitTestsValidateCreate() { Entry("disallow duplicate VLAN IDs on the same parent link", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMVlanSubinterface = true + }) ctx.vm.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, } @@ -3039,6 +3115,9 @@ func unitTestsValidateCreate() { Entry("disallow VLAN name conflicting with interface name", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMVlanSubinterface = true + }) ctx.vm.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, } @@ -3069,6 +3148,9 @@ func unitTestsValidateCreate() { Entry("disallow VLAN name conflicting with interface guestDeviceName", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMVlanSubinterface = true + }) ctx.vm.Spec.Bootstrap = &vmopv1.VirtualMachineBootstrapSpec{ CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, } @@ -3996,6 +4078,178 @@ func unitTestsValidateCreate() { }, ), + Entry("disallow VM Affinity with RequiredDuringSchedulingPreferredDuringExecution and Host topology key when VMAffinityDuringExecution is disabled", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Affinity.VMAffinity = &vmopv1.VMAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + TopologyKey: corev1.LabelHostname, + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.affinity.vmAffinity.requiredDuringSchedulingPreferredDuringExecution[0].topologyKey: Unsupported value: "kubernetes.io/hostname": supported values: "topology.kubernetes.io/zone"`), + }, + ), + + Entry("allow VM Affinity with RequiredDuringSchedulingPreferredDuringExecution and Host topology key when VMAffinityDuringExecution is enabled", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMAffinityDuringExecution = true + }) + ctx.vm.Spec.Affinity.VMAffinity = &vmopv1.VMAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + TopologyKey: corev1.LabelHostname, + }, + }, + } + }, + expectAllowed: true, + }, + ), + + Entry("disallow VM Affinity with RequiredDuringSchedulingPreferredDuringExecution and unsupported topology key even with VMAffinityDuringExecution enabled", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMAffinityDuringExecution = true + }) + ctx.vm.Spec.Affinity.VMAffinity = &vmopv1.VMAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + TopologyKey: "unsupported-key", + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.affinity.vmAffinity.requiredDuringSchedulingPreferredDuringExecution[0].topologyKey: Unsupported value: "unsupported-key": supported values: "topology.kubernetes.io/zone", "kubernetes.io/hostname"`), + }, + ), + + Entry("disallow VM Affinity with RequiredDuringSchedulingPreferredDuringExecution and unsupported operator", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Affinity.VMAffinity = &vmopv1.VMAffinitySpec{ + RequiredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "foo", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"bar"}, + }, + }, + }, + TopologyKey: corev1.LabelTopologyZone, + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.affinity.vmAffinity.requiredDuringSchedulingPreferredDuringExecution[0].labelSelector.matchExpressions[0].operator: Unsupported value: "NotIn": supported values: "In"`), + }, + ), + + Entry("disallow VM Affinity PreferredDuringSchedulingPreferredDuringExecution with Host topology key", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Affinity.VMAffinity = &vmopv1.VMAffinitySpec{ + PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + TopologyKey: corev1.LabelHostname, + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.affinity.vmAffinity.preferredDuringSchedulingPreferredDuringExecution[0].topologyKey: Unsupported value: "kubernetes.io/hostname": supported values: "topology.kubernetes.io/zone"`), + }, + ), + + Entry("allow VM Affinity PreferredDuringSchedulingPreferredDuringExecution with Host topology key when VMAffinityDuringExecution is enabled", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMAffinityDuringExecution = true + }) + ctx.vm.Spec.Affinity.VMAffinity = &vmopv1.VMAffinitySpec{ + PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + TopologyKey: corev1.LabelHostname, + }, + }, + } + }, + expectAllowed: true, + }, + ), + + Entry("disallow VM Affinity PreferredDuringSchedulingPreferredDuringExecution with unsupported topology key even with VMAffinityDuringExecution enabled", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMAffinityDuringExecution = true + }) + ctx.vm.Spec.Affinity.VMAffinity = &vmopv1.VMAffinitySpec{ + PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + TopologyKey: "unsupported-key", + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.affinity.vmAffinity.preferredDuringSchedulingPreferredDuringExecution[0].topologyKey: Unsupported value: "unsupported-key": supported values: "topology.kubernetes.io/zone", "kubernetes.io/hostname"`), + }, + ), + + Entry("disallow VM Affinity PreferredDuringSchedulingPreferredDuringExecution with empty topology key", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Affinity.VMAffinity = &vmopv1.VMAffinitySpec{ + PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + TopologyKey: "", + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.affinity.vmAffinity.preferredDuringSchedulingPreferredDuringExecution[0].topologyKey: Unsupported value: "": supported values: "topology.kubernetes.io/zone"`), + }, + ), + + Entry("disallow VM Affinity PreferredDuringSchedulingPreferredDuringExecution with unsupported operator", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Affinity.VMAffinity = &vmopv1.VMAffinitySpec{ + PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "foo", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"bar"}, + }, + }, + }, + TopologyKey: corev1.LabelTopologyZone, + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.affinity.vmAffinity.preferredDuringSchedulingPreferredDuringExecution[0].labelSelector.matchExpressions[0].operator: Unsupported value: "NotIn": supported values: "In"`), + }, + ), + Entry("allow VM Anti Affinity with RequiredDuringSchedulingPreferredDuringExecution and Zone topology key for non-privileged users", testParams{ setup: func(ctx *unitValidatingWebhookContext) { @@ -4246,6 +4500,25 @@ func unitTestsValidateCreate() { `spec.affinity.vmAntiAffinity.preferredDuringSchedulingPreferredDuringExecution[0].labelSelector.matchExpressions[0].key: Forbidden: label selector can not contain VM Operator managed labels (vmoperator.vmware.com)`), }, ), + + Entry("disallow VM Anti Affinity PreferredDuringSchedulingPreferredDuringExecution with unsupported topology key when VMAffinityDuringExecution is enabled", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.VMAffinityDuringExecution = true + }) + ctx.vm.Spec.Affinity.VMAntiAffinity = &vmopv1.VMAntiAffinitySpec{ + PreferredDuringSchedulingPreferredDuringExecution: []vmopv1.VMAffinityTerm{ + { + TopologyKey: "unsupported-key", + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.affinity.vmAntiAffinity.preferredDuringSchedulingPreferredDuringExecution[0].topologyKey: Unsupported value: "unsupported-key": supported values: "topology.kubernetes.io/zone", "kubernetes.io/hostname"`), + }, + ), ) }) @@ -9994,4 +10267,356 @@ func commonCreateAndUpdateValidations( ), ) }) + + Context("TelcoVMServiceAPI extraConfig validation", func() { + var ( + ctx *unitValidatingWebhookContext + ) + + // Not the outer commonCreateAndUpdateValidations doTest: that closure binds a different ctx. + doTest := func(args testParams) { + doTestWithContext(ctx, args) + } + + BeforeEach(func() { + ctx = newUnitTestContextForValidatingWebhook(true) + + // Enable TelcoVMServiceAPI feature for all tests in this context. + pkgcfg.SetContext(ctx, func(config *pkgcfg.Config) { + config.Features.TelcoVMServiceAPI = true + }) + + bypassUpgradeCheck(&ctx.Context, ctx.vm, ctx.oldVM) + }) + + Context("VM-level extraConfig validation", func() { + DescribeTable("should validate VM advanced extraConfig", doTest, + Entry("should allow non-reserved extraConfig keys", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Advanced = &vmopv1.VirtualMachineAdvancedSpec{ + ExtraConfig: []common.KeyValuePair{ + {Key: "user.custom.setting", Value: "value1"}, + {Key: "custom.app.config", Value: "value2"}, + }, + } + }, + expectAllowed: true, + }, + ), + Entry("should reject first-class VMX keys", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Advanced = &vmopv1.VirtualMachineAdvancedSpec{ + ExtraConfig: []common.KeyValuePair{ + {Key: "numa.vcpu.preferHT", Value: "TRUE"}, + }, + } + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.advanced.extraConfig[0].key: " + + "Forbidden: numa.vcpu.preferHT: use the corresponding first-class field " + + "in spec.advanced instead"), + }, + ), + Entry("should reject vmservice.* prefix", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Advanced = &vmopv1.VirtualMachineAdvancedSpec{ + ExtraConfig: []common.KeyValuePair{ + {Key: "vmservice.test.key", Value: "value"}, + }, + } + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.advanced.extraConfig[0].key: " + + "Forbidden: vmservice.test.key: this key is reserved for the system"), + }, + ), + Entry("should reject guestinfo.* prefix", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Advanced = &vmopv1.VirtualMachineAdvancedSpec{ + ExtraConfig: []common.KeyValuePair{ + {Key: "guestinfo.custom.data", Value: "value"}, + }, + } + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.advanced.extraConfig[0].key: " + + "Forbidden: guestinfo.custom.data: this key is reserved for the system"), + }, + ), + Entry("should reject vmx.reboot.powerCycle exact key", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Advanced = &vmopv1.VirtualMachineAdvancedSpec{ + ExtraConfig: []common.KeyValuePair{ + {Key: "vmx.reboot.powerCycle", Value: "TRUE"}, + }, + } + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.advanced.extraConfig[0].key: " + + "Forbidden: vmx.reboot.powerCycle: this key is reserved for the system"), + }, + ), + Entry("should reject GOSC reserved keys", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Advanced = &vmopv1.VirtualMachineAdvancedSpec{ + ExtraConfig: []common.KeyValuePair{ + {Key: "tools.deployPkg.fileName", Value: "package.tar"}, + }, + } + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.advanced.extraConfig[0].key: " + + "Forbidden: tools.deployPkg.fileName: this key is reserved for the system"), + }, + ), + Entry("should reject ethernet device-scoped keys", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Advanced = &vmopv1.VirtualMachineAdvancedSpec{ + ExtraConfig: []common.KeyValuePair{ + {Key: "ethernet0.ctxPerDev", Value: "1"}, + }, + } + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.advanced.extraConfig[0].key: " + + "Forbidden: ethernet0.ctxPerDev: use spec.network.interfaces[].vmxnet3 or advancedProperties instead"), + }, + ), + ) + }) + + Context("Network interface advancedProperties validation", func() { + DescribeTable("should validate NIC advancedProperties", doTest, + Entry("should allow non-conflicting advancedProperties", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Type: vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + AdvancedProperties: []common.KeyValuePair{ + {Key: "custom.nic.setting", Value: "value1"}, + {Key: "user.network.config", Value: "value2"}, + }, + }, + }, + } + }, + expectAllowed: true, + }, + ), + Entry("should reject first-class NIC properties (bare key)", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Type: vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + AdvancedProperties: []common.KeyValuePair{ + {Key: "ctxPerDev", Value: "1"}, + }, + }, + }, + } + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.network.interfaces[0].advancedProperties[0].key: " + + "Forbidden: ctxPerDev: use the corresponding first-class field in spec.network.interfaces[].vmxnet3 instead"), + }, + ), + Entry("should reject first-class NIC properties (device-prefixed)", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Type: vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + AdvancedProperties: []common.KeyValuePair{ + {Key: "ethernet0.ctxPerDev", Value: "1"}, + }, + }, + }, + } + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.network.interfaces[0].advancedProperties[0].key: " + + "Forbidden: ethernet0.ctxPerDev: use the corresponding first-class field in spec.network.interfaces[].vmxnet3 instead"), + }, + ), + Entry("should reject generic ethernet device-scoped keys", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Type: vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + AdvancedProperties: []common.KeyValuePair{ + {Key: "ethernet1.customSetting", Value: "value"}, + }, + }, + }, + } + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.network.interfaces[0].advancedProperties[0].key: " + + "Forbidden: ethernet1.customSetting: use the bare key name without the network device prefix"), + }, + ), + Entry("should reject system reserved network device properties", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Type: vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + AdvancedProperties: []common.KeyValuePair{ + {Key: "present", Value: "TRUE"}, + }, + }, + }, + } + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.network.interfaces[0].advancedProperties[0].key: Forbidden: present: this key is reserved for the system"), + }, + ), + ) + }) + + Context("VMXNet3 validation", func() { + setupVMXNet3Test := func(ctx *unitValidatingWebhookContext, interfaceType vmopv1.VirtualMachineNetworkInterfaceType, pnicFeatures []vmopv1.PNICQueueFeature) { + ctx.vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Type: interfaceType, + VMXNet3: &vmopv1.VirtualMachineNetworkInterfaceVMXNet3Spec{ + PNICFeatures: pnicFeatures, + }, + }, + }, + } + } + + DescribeTable("should validate PNICFeatures", doTest, + // Valid values + Entry("should allow enum values", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + setupVMXNet3Test(ctx, vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + []vmopv1.PNICQueueFeature{vmopv1.PNICQueueFeatureLargeReceiveOffload, vmopv1.PNICQueueFeatureReceiveSideScaling}) + }, + expectAllowed: true, + }, + ), + Entry("should allow power-of-2 integers", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + setupVMXNet3Test(ctx, vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + []vmopv1.PNICQueueFeature{"1", "2", "4", "8", "16", "1024"}) + }, + expectAllowed: true, + }, + ), + + // Invalid values + Entry("should reject zero", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + setupVMXNet3Test(ctx, vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + []vmopv1.PNICQueueFeature{"0"}) + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.network.interfaces[0].vmxnet3.pnicFeatures[0]: Invalid value: \"0\": must be a known enum value (LargeReceiveOffload, ReceiveSideScaling) or a power-of-2 integer (1,2,4,8,16,32,...)"), + }, + ), + Entry("should reject non-power-of-2 integers", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + setupVMXNet3Test(ctx, vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + []vmopv1.PNICQueueFeature{"3"}) + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.network.interfaces[0].vmxnet3.pnicFeatures[0]: Invalid value: \"3\": must be a known enum value (LargeReceiveOffload, ReceiveSideScaling) or a power-of-2 integer (1,2,4,8,16,32,...)"), + }, + ), + Entry("should reject non-integers", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + setupVMXNet3Test(ctx, vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + []vmopv1.PNICQueueFeature{"abc"}) + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.network.interfaces[0].vmxnet3.pnicFeatures[0]: Invalid value: \"abc\": must be a known enum value (LargeReceiveOffload, ReceiveSideScaling) or a power-of-2 integer (1,2,4,8,16,32,...)"), + }, + ), + Entry("should reject VMXNet3 config with SRIOV type", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + setupVMXNet3Test(ctx, vmopv1.VirtualMachineNetworkInterfaceTypeSRIOV, + []vmopv1.PNICQueueFeature{vmopv1.PNICQueueFeatureLargeReceiveOffload}) + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.network.interfaces[0].vmxnet3: Forbidden: vmxnet3 configuration is only allowed when type is VMXNet3"), + }, + ), + ) + + setupCoalescingTest := func(ctx *unitValidatingWebhookContext, scheme vmopv1.CoalescingScheme, params string) { + ctx.vm.Spec.Network = &vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Type: vmopv1.VirtualMachineNetworkInterfaceTypeVMXNet3, + VMXNet3: &vmopv1.VirtualMachineNetworkInterfaceVMXNet3Spec{ + CoalescingScheme: &scheme, + CoalescingParams: ¶ms, + }, + }, + }, + } + } + + DescribeTable("should validate CoalescingParams", doTest, + Entry("should allow valid 32-bit uint with RateBasedCoalescing", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + setupCoalescingTest(ctx, vmopv1.CoalescingSchemeRateBasedCoalescing, "100") + }, + expectAllowed: true, + }, + ), + Entry("should reject non-integer with RateBasedCoalescing", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + setupCoalescingTest(ctx, vmopv1.CoalescingSchemeRateBasedCoalescing, "abc") + }, + expectAllowed: false, + validate: doValidateWithMsg("spec.network.interfaces[0].vmxnet3.coalescingParams: Invalid value: \"abc\": must be a valid 32-bit unsigned integer when coalescingScheme is RateBasedCoalescing"), + }, + ), + Entry("should allow params string with non-RateBasedCoalescing", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + setupCoalescingTest(ctx, vmopv1.CoalescingSchemeStatic, "64,64,64") + }, + expectAllowed: true, + }, + ), + ) + }) + + }) } diff --git a/webhooks/virtualmachineservice/validation/virtualmachineservice_validator_intg_test.go b/webhooks/virtualmachineservice/validation/virtualmachineservice_validator_intg_test.go index a45b97d9b..445a4201f 100644 --- a/webhooks/virtualmachineservice/validation/virtualmachineservice_validator_intg_test.go +++ b/webhooks/virtualmachineservice/validation/virtualmachineservice_validator_intg_test.go @@ -8,6 +8,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha6" "github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels" "github.com/vmware-tanzu/vm-operator/test/builder" @@ -99,6 +102,88 @@ func intgTestsValidateCreate() { Entry("should work", true, "", nil), Entry("should not work for invalid", false, "spec.type: Required value", nil), ) + + Describe("CRD schema and CEL (ipFamilies and ipFamilyPolicy)", func() { + It("allows LoadBalancer with dual-stack ipFamilies and PreferDualStack policy", func() { + vmSvc := ctx.vmService.DeepCopy() + vmSvc.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} + p := corev1.IPFamilyPolicyPreferDualStack + vmSvc.Spec.IPFamilyPolicy = &p + Expect(ctx.Client.Create(ctx, vmSvc)).To(Succeed()) + }) + + It("rejects ExternalName when ipFamilies is set", func() { + vmSvc := ctx.vmService.DeepCopy() + vmSvc.Spec.Type = vmopv1.VirtualMachineServiceTypeExternalName + vmSvc.Spec.ExternalName = "backend.example.com." + vmSvc.Spec.Ports = nil + vmSvc.Spec.Selector = nil + vmSvc.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol} + err := ctx.Client.Create(ctx, vmSvc) + Expect(apierrors.IsInvalid(err)).To(BeTrue()) + Expect(err.Error()).To(ContainSubstring("ipFamilies and ipFamilyPolicy may not be set when type is ExternalName")) + }) + + It("rejects ExternalName when ipFamilyPolicy is set", func() { + vmSvc := ctx.vmService.DeepCopy() + vmSvc.Spec.Type = vmopv1.VirtualMachineServiceTypeExternalName + vmSvc.Spec.ExternalName = "backend.example.com." + vmSvc.Spec.Ports = nil + vmSvc.Spec.Selector = nil + p := corev1.IPFamilyPolicySingleStack + vmSvc.Spec.IPFamilyPolicy = &p + err := ctx.Client.Create(ctx, vmSvc) + Expect(apierrors.IsInvalid(err)).To(BeTrue()) + Expect(err.Error()).To(ContainSubstring("ipFamilies and ipFamilyPolicy may not be set when type is ExternalName")) + }) + + It("rejects duplicate ipFamilies entries", func() { + vmSvc := ctx.vmService.DeepCopy() + vmSvc.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv4Protocol} + err := ctx.Client.Create(ctx, vmSvc) + Expect(apierrors.IsInvalid(err)).To(BeTrue()) + Expect(err.Error()).To(ContainSubstring("ipFamilies must not contain duplicate entries")) + }) + + It("rejects ipFamilies values outside IPv4 and IPv6", func() { + vmSvc := ctx.vmService.DeepCopy() + vmSvc.Spec.IPFamilies = []corev1.IPFamily{"bogus"} + err := ctx.Client.Create(ctx, vmSvc) + Expect(apierrors.IsInvalid(err)).To(BeTrue()) + // OpenAPI items.enum may reject before CEL; either error is acceptable. + Expect(err.Error()).To(Or( + ContainSubstring("each ipFamilies entry must be IPv4 or IPv6"), + ContainSubstring("bogus"), + ContainSubstring("supported values"), + )) + }) + + It("rejects more than two ipFamilies entries", func() { + vmSvc := ctx.vmService.DeepCopy() + vmSvc.Spec.IPFamilies = []corev1.IPFamily{ + corev1.IPv4Protocol, + corev1.IPv6Protocol, + corev1.IPv4Protocol, + } + err := ctx.Client.Create(ctx, vmSvc) + Expect(apierrors.IsInvalid(err)).To(BeTrue()) + // OpenAPI maxItems vs CEL duplicate rule: message text differs by apiserver version. + Expect(err.Error()).To(ContainSubstring("ipFamilies")) + }) + + It("rejects unknown ipFamilyPolicy value", func() { + vmSvc := ctx.vmService.DeepCopy() + bad := corev1.IPFamilyPolicy("bogus") + vmSvc.Spec.IPFamilyPolicy = &bad + err := ctx.Client.Create(ctx, vmSvc) + Expect(apierrors.IsInvalid(err)).To(BeTrue()) + Expect(err.Error()).To(Or( + ContainSubstring("ipFamilyPolicy"), + ContainSubstring("Supported values"), + ContainSubstring("enum"), + )) + }) + }) } func intgTestsValidateUpdate() { @@ -120,10 +205,8 @@ func intgTestsValidateUpdate() { ctx = nil }) - When("update is performed with changed ... placeholder", func() { - BeforeEach(func() { - }) - It("should deny the request", func() { + When("update is performed without spec changes", func() { + It("should allow the request", func() { Expect(err).ToNot(HaveOccurred()) }) })