test(e2e): add nested-cluster e2e suite for sds-object

.github/workflows/e2e-tests.yml

@@ -0,0 +1,43 @@
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Thin caller for the reusable storage-e2e pipeline. The heavy e2e run only
+# fires when the PR carries the `e2e/run` label (set it on the PR to trigger).
+# Other labels: `e2e/keep-cluster` (skip teardown), `e2e/label:<suite>` (Ginkgo
+# label filter). See storage-e2e docs/CI.md for the secrets/vars to configure.
+
+name: E2E
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+
+jobs:
+  e2e:
+    # Gate: run only when the PR is labeled e2e/run.
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'e2e/run') }}
+    uses: deckhouse/storage-e2e/.github/workflows/e2e.yml@main
+    with:
+      module_slug: sds-object
+      # The e2e Go module lives under e2e/; tests are in e2e/tests/.
+      module_path: e2e
+      test_package: ./tests/
+      # CI config: modulePullOverride is ${E2E_MODULE_IMAGE_TAG}, resolved by the
+      # enable-modules step to the PR image tag below.
+      cluster_config: e2e/tests/cluster_config.ci.yml
+      # Create a fresh cluster through Deckhouse Commander for each run.
+      cluster_provider: commander
+      # Install the image built for this PR (build_dev publishes the pr<N> tag).
+      module_image_tag: pr${{ github.event.pull_request.number }}
+    secrets: inherit

e2e/.gitignore

@@ -0,0 +1,7 @@
+# Local env files with secrets (DKP license, registry cfg, SSH passphrase).
+# Keep them out of git.
+config/
+test_exports*
+.tmp-*.log
+e2e-artifacts-*/
+.gomodcache/

e2e/Makefile

@@ -0,0 +1,69 @@
+# Makefile for sds-object e2e tests.
+
+.PHONY: help
+help: ## Show help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-28s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: deps
+deps: ## Download dependencies
+	go mod download
+	go mod tidy
+
+.PHONY: test
+test: ## Run the full nested-cluster E2E suite
+	go test -v -count=1 -timeout 120m ./tests/ -ginkgo.timeout 120m
+
+.PHONY: test-focus
+test-focus: ## Run a focused Ginkgo spec (make test-focus FOCUS="reaches Ready")
+	@if [ -z "$(FOCUS)" ]; then \
+		echo "Error: specify FOCUS=<Ginkgo focus string>"; \
+		exit 1; \
+	fi
+	go test -v -count=1 -timeout 120m ./tests/ -ginkgo.timeout 120m -ginkgo.focus "$(FOCUS)"
+
+.PHONY: vet
+vet: ## go vet
+	go vet ./...
+
+.PHONY: build
+build: ## Compile-check the suite (no cluster)
+	go build ./...
+	go test -c -o /dev/null ./tests/
+
+.PHONY: lint
+lint: ## golangci-lint (if installed)
+	golangci-lint run ./...
+
+.PHONY: check-env
+check-env: ## Print relevant env vars for the nested flow
+	@echo "=== Nested cluster (storage-e2e) ==="
+	@echo "TEST_CLUSTER_CREATE_MODE:        $${TEST_CLUSTER_CREATE_MODE}  # alwaysCreateNew | alwaysUseExisting | commander"
+	@echo "TEST_CLUSTER_CLEANUP:            $${TEST_CLUSTER_CLEANUP}      # set true to delete VMs after the run"
+	@echo "TEST_CLUSTER_NAMESPACE:          $${TEST_CLUSTER_NAMESPACE}    # also the in-cluster namespace for OB/Secret/probe"
+	@echo "TEST_CLUSTER_STORAGE_CLASS:      $${TEST_CLUSTER_STORAGE_CLASS}  # base-cluster SC for the VM OS disks"
+	@echo "SSH_HOST:                        $${SSH_HOST}"
+	@echo "SSH_USER:                        $${SSH_USER}"
+	@echo "SSH_PRIVATE_KEY:                 $${SSH_PRIVATE_KEY}"
+	@echo "SSH_PUBLIC_KEY:                  $${SSH_PUBLIC_KEY}  # required for alwaysCreateNew (VM authorized key)"
+	@echo "SSH_VM_USER:                     $${SSH_VM_USER}  # required for alwaysCreateNew (SSH user inside the VMs)"
+	@echo "DKP_LICENSE_KEY:                 $${DKP_LICENSE_KEY:+<set>}"
+	@echo "REGISTRY_DOCKER_CFG:             $${REGISTRY_DOCKER_CFG:+<set>}"
+	@echo "SDS_OBJECT_MODULE_PULL_OVERRIDE: $${SDS_OBJECT_MODULE_PULL_OVERRIDE}  # overrides modulePullOverride (default main)"
+	@echo "YAML_CONFIG_FILENAME:            $${YAML_CONFIG_FILENAME}  # default cluster_config.yml"
+	@echo ""
+	@echo "=== sds-object suite knobs ==="
+	@echo "E2E_OSC_NAME:                    $${E2E_OSC_NAME}"
+	@echo "E2E_OSC_TYPE:                    $${E2E_OSC_TYPE}  # System | Lightweight | Full | Heavy (default System)"
+	@echo "E2E_REDUNDANCY:                  $${E2E_REDUNDANCY}  # Single | Replicated | HighRedundancy"
+	@echo "E2E_STORAGE_CLASS:               $${E2E_STORAGE_CLASS}  # required for Lightweight/Full"
+	@echo "E2E_OSC_SIZE:                    $${E2E_OSC_SIZE}"
+	@echo "E2E_ELASTIC_CLUSTER_REF:         $${E2E_ELASTIC_CLUSTER_REF}  # required for Heavy"
+	@echo "E2E_BUCKET_NAME:                 $${E2E_BUCKET_NAME}"
+	@echo "E2E_OSC_READY_TIMEOUT:           $${E2E_OSC_READY_TIMEOUT}"
+	@echo "E2E_OB_READY_TIMEOUT:            $${E2E_OB_READY_TIMEOUT}"
+	@echo "E2E_MODULE_READY_TIMEOUT:        $${E2E_MODULE_READY_TIMEOUT}"
+	@echo "E2E_PROBE_IMAGE:                 $${E2E_PROBE_IMAGE}  # image with mc for the S3 round-trip job"
+	@echo "E2E_PROBE_JOB_TIMEOUT:           $${E2E_PROBE_JOB_TIMEOUT}"
+	@echo "E2E_KEEP_CLUSTER_ON_FAILURE:     $${E2E_KEEP_CLUSTER_ON_FAILURE}  # true|1|yes -> skip teardown if a spec failed"
+
+.DEFAULT_GOAL := help

e2e/README.md

@@ -0,0 +1,176 @@
+# E2E tests for sds-object
+
+End-to-end coverage for the documented `ObjectStorageCluster` / `ObjectBucket`
+lifecycle: cluster creation, bucket provisioning with the standardised S3
+credentials `Secret`, a real S3 write/list/read round-trip through the generated
+credentials, the admission guards (validating webhooks + CRD CEL rules), and
+finalizer-driven deletion.
+
+1. `storage-e2e` brings up a nested cluster from `tests/cluster_config.yml`
+   (1 master + 2 workers) with the `sds-object` module enabled.
+2. `BeforeSuite` waits for the module to become Ready and ensures the in-cluster
+   test namespace exists.
+3. A single shared `ObjectStorageCluster` is created and the ordered specs run
+   on top of it: create → bucket + creds `Secret` + S3 round-trip → validation
+   guards → delete.
+4. `AfterSuite` hands the cluster back to `storage-e2e` for teardown.
+
+## Running in CI (PR label)
+
+The suite is wired into the reusable `storage-e2e` pipeline via
+[`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml). It is
+**gated by the `e2e/run` PR label**: add that label to a pull request to trigger
+a run (the job graph is `resolve → bootstrap → run-tests → teardown`). The
+pipeline is configured with `cluster_provider: commander`, so each run creates a
+fresh cluster through the Deckhouse Commander API and deletes it on teardown.
+
+Other labels:
+
+- `e2e/keep-cluster` — skip teardown so you can inspect / re-run on the cluster.
+- `e2e/label:<suite>` — Ginkgo label filter (multiple are OR-joined).
+
+The Commander endpoint, token and template come from inherited org/repo secrets
+and vars (`E2E_COMMANDER_*`); see `storage-e2e` `docs/CI.md` for the full list.
+
+Pipeline flow (`resolve → bootstrap → enable-modules → run-tests → teardown`):
+`bootstrap` creates the cluster via Commander and hands off its kubeconfig;
+`enable-modules` installs `sds-object` from `tests/cluster_config.ci.yml`
+(`modulePullOverride: "${E2E_MODULE_IMAGE_TAG}"`, resolved to `pr<N>`);
+`run-tests` connects to that cluster from the kubeconfig (no SSH) and runs the
+suite. This requires the PR's dev image (`pr<N>`) to be built and pushed to the
+dev-registry **before** the e2e run (the `build_dev` workflow), and the Commander
+cluster must be able to pull from that registry.
+
+## Running locally
+
+The suite drives the CRDs through the dynamic client and reads the
+`storage.deckhouse.io/v1alpha1` Go types from the in-repo `sds-object/api`
+module (`replace github.com/deckhouse/sds-object/api => ../api`). All generic
+nested-cluster plumbing (`pkg/cluster`, `pkg/kubernetes`) comes from
+`storage-e2e`, consumed as a pinned pseudo-version.
+
+## Profiles
+
+The shared cluster's profile is selectable via `E2E_OSC_TYPE` (default
+`System`). The default is intentionally the cheapest, most self-contained
+profile so CI needs no extra storage modules:
+
+| `E2E_OSC_TYPE` | Backend | Extra requirements |
+|----------------|---------|--------------------|
+| `System` (default) | Garage (DaemonSet on control-plane, hostPath) | none |
+| `Lightweight` | Garage (StatefulSet + PVC) | `E2E_STORAGE_CLASS` + a CSI/local-volume module enabled in `cluster_config.yml` |
+| `Full` | SeaweedFS | `E2E_STORAGE_CLASS` + `managed-postgres` module |
+| `Heavy` | Ceph RGW | `sds-elastic` module + a Ready `ElasticCluster` (`E2E_ELASTIC_CLUSTER_REF`) |
+
+When you point `E2E_OSC_TYPE` at a heavier profile, enable the corresponding
+modules in `tests/cluster_config.yml` first (see the comments there). The suite
+fails fast in `BeforeSuite` if a profile's required env knob is missing.
+
+## Why one shared cluster + Ordered specs
+
+The validation and delete specs build on the cluster and bucket created by the
+first specs, so the suite uses a **single shared `ObjectStorageCluster`** inside
+one `Describe(..., Ordered)`. Spec registration goes through builder functions
+called in explicit order from the root container
+(`createSpecs → validationSpecs → deleteSpecs`); the deletion specs run last.
+`RandomizeAllSpecs` stays **off**.
+
+## Requirements
+
+- Go **1.26+**
+- A base Deckhouse cluster with the `virtualization` module enabled.
+- SSH access to the master node of the base cluster.
+- A Deckhouse license and a docker config for the dev registry.
+- A block-mode `StorageClass` on the base cluster for the VM OS disks.
+- Outbound access to Docker Hub for the `minio/mc` probe image (override with
+  `E2E_PROBE_IMAGE` if you mirror it).
+
+## Environment variables
+
+### `storage-e2e` (nested cluster)
+
+- `TEST_CLUSTER_CREATE_MODE` (**required**):
+  one of `alwaysCreateNew`, `alwaysUseExisting`, `commander`.
+- `TEST_CLUSTER_CLEANUP`:
+  set to `true` to delete the VMs after the run.
+- `TEST_CLUSTER_NAMESPACE`:
+  the VM namespace on the base cluster **and** the in-cluster namespace the
+  suite uses for ObjectBuckets / credentials Secrets / probe Pods (single source
+  of truth — no separate `E2E_NAMESPACE`).
+- `TEST_CLUSTER_STORAGE_CLASS`:
+  base-cluster `StorageClass` for the VM OS disks.
+- `YAML_CONFIG_FILENAME`:
+  defaults to `cluster_config.yml`.
+- `SSH_HOST`, `SSH_USER`, `SSH_PRIVATE_KEY`
+- `SSH_PUBLIC_KEY`:
+  SSH public key injected as the VMs' authorized key. **Required in
+  `alwaysCreateNew` mode**.
+- `SSH_VM_USER`:
+  SSH user inside the created VMs (must match the VM image, usually `cloud`).
+  **Required in `alwaysCreateNew` mode**.
+- `SSH_JUMP_HOST`, `SSH_JUMP_USER`, `SSH_JUMP_KEY_PATH`:
+  jump-host (bastion) SSH settings used by `alwaysUseExisting`.
+- `TEST_CLUSTER_FORCE_LOCK_RELEASE`:
+  set to `true` to steal a stale `e2e-cluster-lock` left by a crashed run.
+- `DKP_LICENSE_KEY`
+- `REGISTRY_DOCKER_CFG`
+- `SDS_OBJECT_MODULE_PULL_OVERRIDE`:
+  overrides `modulePullOverride` for `sds-object` from
+  `tests/cluster_config.yml` (which keeps a literal `main` default). Set to
+  `prN` on GitHub, `mrN` on GitLab, or `main` for nightly. (This is
+  storage-e2e's generic per-module convention: `<MODULE>_MODULE_PULL_OVERRIDE`.)
+
+### `sds-object` suite knobs
+
+- `E2E_OSC_NAME`: name of the shared `ObjectStorageCluster`, defaults to `e2e-osc`.
+- `E2E_OSC_TYPE`: profile, one of `System` (default) / `Lightweight` / `Full` / `Heavy`.
+- `E2E_REDUNDANCY`: `Single` (default) / `Replicated` / `HighRedundancy`.
+- `E2E_STORAGE_CLASS`: StorageClass for the PVCs; **required** for `Lightweight`/`Full`.
+- `E2E_OSC_SIZE`: cluster storage size, defaults to `5Gi`.
+- `E2E_ELASTIC_CLUSTER_REF`: `ElasticCluster` name; **required** for `Heavy`.
+- `E2E_BUCKET_NAME`: name of the shared `ObjectBucket`, defaults to `e2e-bucket`.
+- `E2E_OSC_READY_TIMEOUT`: Go duration bounding the cluster Ready wait, defaults to 15m.
+- `E2E_OB_READY_TIMEOUT`: Go duration bounding the bucket Ready wait, defaults to 5m.
+- `E2E_MODULE_READY_TIMEOUT`: Go duration bounding the module Ready wait, defaults to 15m.
+- `E2E_PROBE_IMAGE`: container image carrying `mc` for the S3 round-trip Job, defaults to `minio/mc:latest`.
+- `E2E_PROBE_JOB_TIMEOUT`: Go duration bounding the probe Job, defaults to 5m.
+- `E2E_KEEP_CLUSTER_ON_FAILURE`: when truthy and at least one spec failed, the
+  nested cluster is **not** torn down in `AfterSuite`, so you can inspect it.
+
+## Quick start
+
+```bash
+export TEST_CLUSTER_CREATE_MODE=alwaysCreateNew
+export TEST_CLUSTER_CLEANUP=true
+export TEST_CLUSTER_NAMESPACE=e2e-sds-object
+export TEST_CLUSTER_STORAGE_CLASS=linstor-r2
+
+export SSH_HOST=<master-ip>
+export SSH_USER=<ssh-user>
+export SSH_PRIVATE_KEY=~/.ssh/id_rsa
+export SSH_PUBLIC_KEY=~/.ssh/id_rsa.pub   # required for alwaysCreateNew
+export SSH_VM_USER=cloud                  # required for alwaysCreateNew
+
+export DKP_LICENSE_KEY=<license>
+export REGISTRY_DOCKER_CFG=<base64-docker-config>
+
+# Override the sds-object image tag; optional, defaults to the literal "main".
+export SDS_OBJECT_MODULE_PULL_OVERRIDE=main   # or prN / mrN to test a specific PR/MR
+
+cd e2e
+make deps
+make test
+```
+
+For local debugging you can run a subset of specs:
+
+```bash
+make test-focus FOCUS="round-trip"
+```
+
+## Compile check (no cluster)
+
+```bash
+make build
+make vet
+```

e2e/go.mod

@@ -0,0 +1,83 @@
+module github.com/deckhouse/sds-object/e2e
+
+go 1.26.0
+
+require (
+	github.com/deckhouse/sds-object/api v0.0.0-00010101000000-000000000000
+	github.com/deckhouse/storage-e2e v0.0.0-20260615225534-f681188c4aa9
+	github.com/onsi/ginkgo/v2 v2.23.3
+	github.com/onsi/gomega v1.37.0
+	k8s.io/api v0.34.2
+	k8s.io/apimachinery v0.34.2
+	k8s.io/client-go v0.34.2
+)
+
+require (
+	github.com/Masterminds/semver/v3 v3.3.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/deckhouse/deckhouse v1.74.0 // indirect
+	github.com/deckhouse/sds-node-configurator/api v0.0.0-20260114125558-7fd7152586ff // indirect
+	github.com/deckhouse/virtualization/api v1.8.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-openapi/jsonpointer v0.22.1 // indirect
+	github.com/go-openapi/jsonreference v0.21.2 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-openapi/swag/jsonname v0.25.1 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/spdystream v0.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/openshift/custom-resource-status v1.1.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/sftp v1.13.10 // indirect
+	github.com/spf13/pflag v1.0.7 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.46.0 // indirect
+	golang.org/x/net v0.47.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/sys v0.39.0 // indirect
+	golang.org/x/term v0.38.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
+	golang.org/x/tools v0.39.0 // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.34.2 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
+	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
+	kubevirt.io/api v1.6.2 // indirect
+	kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 // indirect
+	kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
+	sigs.k8s.io/controller-runtime v0.22.4 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
+)
+
+// storage-e2e carries the generic nested-cluster + pkg/kubernetes helpers this
+// suite relies on; it is consumed as a pinned pseudo-version from origin (no
+// local checkout needed). Re-pin to a tagged release once one is published.
+
+// sds-object/api is always consumed from this repo's source via replace.
+replace github.com/deckhouse/sds-object/api => ../api