Introduce KUTTL tests

Makefile

@@ -224,6 +224,7 @@ openstack-lightspeed-deploy: ## Deploy using a catalog image.
 	oc apply -f $(OUTPUT_DIR)/catalog
 	bash scripts/gen-rhosls.sh $(CATALOG_NAME) $(CATALOG_CHANNEL)
 	oc apply -f $(OUTPUT_DIR)/rhosls
+	bash scripts/confirm-rhosls-running.sh
 
 # Deploy using the catalog image.
 .PHONY: openstack-lightspeed-undeploy
@@ -247,12 +248,14 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
+KUTTL ?= $(LOCALBIN)/kubectl-kuttl
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.4.2
 CONTROLLER_TOOLS_VERSION ?= v0.16.5
 ENVTEST_VERSION ?= release-0.18
 GOLANGCI_LINT_VERSION ?= v2.6.0
+KUTTL_VERSION ?= 0.22.0
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
@@ -274,6 +277,19 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(LOCALBIN) $(GOLANGCI_LINT_VERSION)
 
+.PHONY: kuttl
+kuttl: $(KUTTL) ## Download kubectl-kuttl locally if necessary.
+$(KUTTL): $(LOCALBIN)
+	test -s $(LOCALBIN)/kubectl-kuttl || curl -L -o $(LOCALBIN)/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v$(KUTTL_VERSION)/kubectl-kuttl_$(KUTTL_VERSION)_linux_x86_64
+	chmod +x $(LOCALBIN)/kubectl-kuttl
+
+.PHONY: kuttl-test
+kuttl-test: kuttl ## Run kuttl tests
+	$(LOCALBIN)/kubectl-kuttl test --config kuttl-test.yaml test/kuttl/tests $(KUTTL_ARGS)
+
+.PHONY: kuttl-test-run
+kuttl-test-run: kuttl openstack-lightspeed-deploy kuttl-test openstack-lightspeed-undeploy
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary
 # $2 - package url which can be installed

README.md

@@ -220,3 +220,27 @@ Run all hooks manually:
 ```bash
 pre-commit run --all-files
 ```
+
+### Running KUTTL Tests
+
+KUTTL (KUbernetes Test TooL) tests validate the operator's behavior in a real
+OpenShift environment.
+
+Before running the tests ensure that:
+- `oc` CLI tool is available in your PATH and you can access an OpenShift cluster
+(e.g., deployed with `crc`) with it
+- The `openshift-lightspeed` namespace is empty or non-existing to prevent collisions
+
+Once you are ready you can run the KUTTL tests using:
+
+```bash
+make kuttl-test-run
+```
+
+**Important Notes:**
+- The tests use the `openshift-lightspeed` namespace to test in the exact namespace
+where the OLS operator is expected to operate.
+- The correct behavior of the OLS operator is not guaranteed outside of the
+`openshift-lightspeed` namespace.
+- Ensure the namespace is clean before running tests to avoid resource conflicts
+or test failures.

kuttl-test.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+reportFormat: xml
+reportName: kuttl-report-openstack-lightspeed
+reportGranularity: test
+namespace: openshift-lightspeed
+timeout: 600
+parallel: 1
+suppress:
+  - events

scripts/confirm-rhosls-running.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+while true; do
+    csv=$(oc get subscription openstack-lightspeed-operator -n openshift-lightspeed -o jsonpath='{.status.installedCSV}' 2>/dev/null)
+    if [ -n "$csv" ]; then
+        echo "Found installedCSV: $csv"
+        break
+    fi
+    echo "Waiting for openstack-lightspeed-operator Subscription installedCSV to be populated ..."
+    sleep 5
+done
+
+# Wait for the CSV to succeed
+oc wait csv $csv --for=jsonpath='{.status.phase}'=Succeeded --timeout=300s -n openshift-lightspeed

test/kuttl/common/mock-objects/assert-mock-objects-created.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openshift-lightspeed
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: openstack-lightspeed-apitoken
+  namespace: openshift-lightspeed
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: openstack-lightspeed-cert
+  namespace: openshift-lightspeed
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: mock-llm-api-server-pod
+  namespace: openshift-lightspeed
+status:
+  phase: Running

test/kuttl/common/mock-objects/cleanup-mock-objects.yaml

@@ -0,0 +1,24 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: v1
+    kind: Pod
+    name: mock-llm-api-server-pod
+    namespace: openshift-lightspeed
+  - apiVersion: v1
+    kind: Service
+    name: mock-llm-api-server-pod
+    namespace: openshift-lightspeed
+  - apiVersion: v1
+    kind: ConfigMap
+    name: mock-llm-code
+    namespace: openshift-lightspeed
+  - apiVersion: v1
+    kind: Secret
+    name: openstack-lightspeed-apitoken
+    namespace: openshift-lightspeed
+  - apiVersion: v1
+    kind: ConfigMap
+    name: openstack-lightspeed-cert
+    namespace: openshift-lightspeed

test/kuttl/common/mock-objects/errors-mock-objects.yaml

@@ -0,0 +1,30 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: mock-llm-api-server-pod
+  namespace: openshift-lightspeed
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mock-llm-api-server-pod
+  namespace: openshift-lightspeed
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mock-llm-code
+  namespace: openshift-lightspeed
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: openstack-lightspeed-apitoken
+  namespace: openshift-lightspeed
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: openstack-lightspeed-cert
+  namespace: openshift-lightspeed

test/kuttl/common/mock-objects/mock-resources.yaml

@@ -0,0 +1,135 @@
+##############################################################################
+#        Mock API tokens and certificates required for OLSConfig tests       #
+##############################################################################
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: openstack-lightspeed-apitoken
+  namespace: openshift-lightspeed
+stringData:
+  apitoken: secret
+---
+apiVersion: v1
+kind: ConfigMap
+type: Opaque
+metadata:
+  name: openstack-lightspeed-cert
+  namespace: openshift-lightspeed
+data:
+  cert: |
+    -----BEGIN CERTIFICATE-----
+    MIIEMDCCAxigAwIBAgIJANqb7HHzA7AZMA0GCSqGSIb3DQEBCwUAMIGkMQswCQYD
+    VQQGEwJQQTEPMA0GA1UECAwGUGFuYW1hMRQwEgYDVQQHDAtQYW5hbWEgQ2l0eTEk
+    MCIGA1UECgwbVHJ1c3RDb3IgU3lzdGVtcyBTLiBkZSBSLkwuMScwJQYDVQQLDB5U
+    cnVzdENvciBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHzAdBgNVBAMMFlRydXN0Q29y
+    IFJvb3RDZXJ0IENBLTEwHhcNMTYwMjA0MTIzMjE2WhcNMjkxMjMxMTcyMzE2WjCB
+    pDELMAkGA1UEBhMCUEExDzANBgNVBAgMBlBhbmFtYTEUMBIGA1UEBwwLUGFuYW1h
+    IENpdHkxJDAiBgNVBAoMG1RydXN0Q29yIFN5c3RlbXMgUy4gZGUgUi5MLjEnMCUG
+    A1UECwweVHJ1c3RDb3IgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MR8wHQYDVQQDDBZU
+    cnVzdENvciBSb290Q2VydCBDQS0xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+    CgKCAQEAv463leLCJhJrMxnHQFgKq1mqjQCj/IDHUHuO1CAmujIS2CNUSSUQIpid
+    RtLByZ5OGy4sDjjzGiVoHKZaBeYei0i/mJZ0PmnK6bV4pQa81QBeCQryJ3pS/C3V
+    seq0iWEk8xoT26nPUu0MJLq5nux+AHT6k61sKZKuUbS701e/s/OojZz0JEsq1pme
+    9J7+wH5COucLlVPat2gOkEz7cD+PSiyU8ybdY2mplNgQTsVHCJCZGxdNuWxu72CV
+    EY4hgLW9oHPY0LJ3xEXqWib7ZnZ2+AYfYW0PVcWDtxBWcgYHpfOxGgMFZA6dWorW
+    hnAbJN7+KIor0Gqw/Hqi3LJ5DotlDwIDAQABo2MwYTAdBgNVHQ4EFgQU7mtJPHo/
+    DeOxCbeKyKsZn3MzUOcwHwYDVR0jBBgwFoAU7mtJPHo/DeOxCbeKyKsZn3MzUOcw
+    DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD
+    ggEBACUY1JGPE+6PHh0RU9otRCkZoB5rMZ5NDp6tPVxBb5UrJKF5mDo4Nvu7Zp5I
+    /5CQ7z3UuJu0h3U/IJvOcs+hVcFNZKIZBqEHMwwLKeXx6quj7LUKdJDHfXLy11yf
+    ke+Ri7fc7Waiz45mO7yfOgLgJ90WmMCV1Aqk5IGadZQ1nJBfiDcGrVmVCrDRZ9MZ
+    yonnMlo2HD6CqFqTvsbQZJG2z9m2GM/bftJlo6bEjhcxwft+dtvTheNYsnd6djts
+    L1Ac59v2Z3kf9YKVmgenFK+P3CghZwnS1k1aHBkcjndcw5QkPTJrS37UeJSDvjdN
+    zl/HHk484IkzlQsPpTLWPFp5LBk=
+    -----END CERTIFICATE-----
+
+##############################################################################
+#         Mock Pod to simulate OpenAI /chat/completions API endpoints        #
+#         Used by OpenShiftLightspeed for LLM connection verification        #
+##############################################################################
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: mock-llm-api-server-pod
+  labels:
+    app: mock-llm-api-server-pod
+spec:
+  containers:
+    - name: mock-llm-api-server
+      image: registry.redhat.io/ubi8/python-311:latest
+      ports:
+        - containerPort: 8000
+      volumeMounts:
+        - name: app-code
+          mountPath: /app
+      workingDir: /app
+      command:
+        - sh
+        - -c
+        - |
+          pip install "fastapi[standard]" && \
+          uvicorn app:app --host 0.0.0.0 --port 8000
+  volumes:
+    - name: app-code
+      configMap:
+        name: mock-llm-code
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mock-llm-api-server-pod
+spec:
+  selector:
+    app: mock-llm-api-server-pod
+  ports:
+    - protocol: TCP
+      port: 8000
+      targetPort: 8000
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mock-llm-code
+data:
+  app.py: |
+    from fastapi import FastAPI, Request
+    from fastapi.responses import JSONResponse
+
+    app = FastAPI()
+    @app.post("/v1/chat/completions")
+    async def completions_post(request: Request):
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+
+        # Always return a valid OpenAI-like response, handle missing/None body gracefully
+        model = body.get("model", "gpt-3.5-turbo") if isinstance(body, dict) else "gpt-3.5-turbo"
+
+        # The OpenAI API expects 'messages' (for chat) or 'prompt' (for completions), but for mock, we will accept either
+        response = {
+            "id": "cmpl-123",
+            "object": "chat.completion",
+            "created": 1234567890,
+            "model": model,
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "content": "Hello, this is a dummy chat completion."
+                    },
+                    "finish_reason": "stop"
+                }
+            ],
+            "usage": {
+                "prompt_tokens": 5,
+                "completion_tokens": 7,
+                "total_tokens": 12
+            }
+        }
+        return JSONResponse(content=response)
+

test/kuttl/common/openstack-lightspeed-instance/assert-openstack-lightspeed-instance.yaml

@@ -0,0 +1,71 @@
+---
+apiVersion: ols.openshift.io/v1alpha1
+kind: OLSConfig
+metadata:
+  name: cluster
+spec:
+  llm:
+    providers:
+      - name: openstack-lightspeed-provider
+        type: openai
+        url: http://mock-llm-api-server-pod:8000/v1
+        credentialsSecretRef:
+          name: openstack-lightspeed-apitoken
+        models:
+          - name: ibm-granite/granite-3.1-8b-instruct
+            parameters:
+              maxTokensForResponse: 2048
+  ols:
+    defaultProvider: openstack-lightspeed-provider
+    defaultModel: ibm-granite/granite-3.1-8b-instruct
+    byokRAGOnly: true
+    logLevel: INFO
+    additionalCAConfigMapRef:
+      name: openstack-lightspeed-cert
+    rag:
+      - image: quay.io/openstack-lightspeed/rag-content:os-docs-2025.2
+        indexID: ""
+        indexPath: /rag/vector_db/os_product_docs
+status:
+  conditions:
+    - type: ConsolePluginReady
+      status: "True"
+      reason: Reconciling
+    - type: CacheReady
+      status: "True"
+      reason: Reconciling
+    - type: ApiReady
+      status: "True"
+      reason: Reconciling
+    - type: Reconciled
+      status: "True"
+      reason: Reconciling
+---
+apiVersion: lightspeed.openstack.org/v1beta1
+kind: OpenStackLightspeed
+metadata:
+  name: openstack-lightspeed
+  namespace: openshift-lightspeed
+spec:
+  catalogSourceName: redhat-operators
+  catalogSourceNamespace: openshift-marketplace
+  llmCredentials: openstack-lightspeed-apitoken
+  llmEndpoint: http://mock-llm-api-server-pod:8000/v1
+  llmEndpointType: openai
+  modelName: ibm-granite/granite-3.1-8b-instruct
+  tlsCACertBundle: openstack-lightspeed-cert
+status:
+  conditions:
+    - type: Ready
+      status: "True"
+      reason: Ready
+      message: Setup complete
+    - type: OpenShiftLightspeedOperatorReady
+      status: "True"
+      reason: Ready
+      message: OpenShift Lightspeed operator is ready.
+    - type: OpenStackLightspeedReady
+      status: "True"
+      reason: Ready
+      message: OpenStack Lightspeed created
+

test/kuttl/common/openstack-lightspeed-instance/cleanup-openstack-lightspeed-instance.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: lightspeed.openstack.org/v1beta1
+    kind: OpenStackLightspeed
+    name: openstack-lightspeed
+    namespace: openshift-lightspeed
+