OCPBUGS-87161: [release-4.22] OCPBUGS-87027: MCO-2212: MCO-2213: Migrate mco_observability and mco_daemon test-suite

test/extended-priv/const.go

@@ -10,6 +10,10 @@ const (
 	MachineConfigNamespace = "openshift-machine-config-operator"
 	// MachineConfigDaemon mcd container name
 	MachineConfigDaemon = "machine-config-daemon"
+	// MachineConfigOperator mco container name
+	MachineConfigOperator = "machine-config-operator"
+	// MachineConfigDaemonEvents cluster role binding
+	MachineConfigDaemonEvents = "machine-config-daemon-events"
 
 	// MachineConfigPoolMaster master pool name
 	MachineConfigPoolMaster = "master"
@@ -47,10 +51,22 @@ const (
 	GCPPlatform = "gcp"
 	// AzurePlatform value used to identify azure infrastructure
 	AzurePlatform = "azure"
+	// NonePlatform value used to identify a None Platform value
+	NonePlatform = "none"
 	// VspherePlatform value used to identify Vsphere infrastructure
 	VspherePlatform = "vsphere"
-	// BaremetalPlatform value used to identify BareMetal infrastructure
+	// BaremetalPlatform value used to identify baremetal infrastructure
 	BaremetalPlatform = "baremetal"
+	// KniPlatform value used to identify KNI infrastructure
+	KniPlatform = "kni"
+	// NutanixPlatform value used to identify Nutanix infrastructure
+	NutanixPlatform = "nutanix"
+	// OpenstackPlatform value used to identify Openstack infrastructure
+	OpenstackPlatform = "openstack"
+	// OvirtPlatform value used to identify Ovirt infrastructure
+	OvirtPlatform = "ovirt"
+	// AlibabaCloudPlatform value used to identify AlibabaCloud infrastructure
+	AlibabaCloudPlatform = "alibabacloud"
 
 	// ExpirationDockerfileLabel Expiration label in Dockerfile
 	ExpirationDockerfileLabel = `LABEL maintainer="mco-qe-team" quay.expires-after=24h`
@@ -160,6 +176,16 @@ const (
 )
 
 var (
+	// OnPremPlatforms describes all the on-prem platforms
+	// xref: https://github.com/openshift/machine-config-operator/blob/752667ba9dfcdefd12222ab422201fa3f9846aca/pkg/controller/template/render.go#L593
+	OnPremPlatforms = map[string]string{
+		BaremetalPlatform: "openshift-kni-infra",
+		NutanixPlatform:   "openshift-nutanix-infra",
+		OpenstackPlatform: "openshift-openstack-infra",
+		OvirtPlatform:     "openshift-ovirt-infra",
+		VspherePlatform:   "openshift-vsphere-infra",
+	}
+
 	// Map with all available extensions and the packages they install
 	AllExtenstions = map[string][]string{
 		ipsecExtension:               {"NetworkManager-libreswan", "libreswan"},

test/extended-priv/controller.go

@@ -133,6 +133,50 @@ func (mcc *Controller) HasAcquiredLease() (bool, error) {
 	return re.MatchString(podAllLogs), nil
 }
 
+// GetLogsAsList returns the MCO controller logs as a list strings. One string per line
+func (mcc *Controller) GetLogsAsList() ([]string, error) {
+	logs, err := mcc.GetLogs()
+	if err != nil {
+		return nil, err
+	}
+
+	return strings.Split(logs, "\n"), nil
+}
+
+// GetFilteredLogsAsList returns the filtered logs as a lit of strings, one string per line.
+func (mcc *Controller) GetFilteredLogsAsList(regex string) ([]string, error) {
+	logs, err := mcc.GetLogsAsList()
+	if err != nil {
+		return nil, err
+	}
+
+	filteredLogs := []string{}
+	for _, line := range logs {
+		match, err := regexp.MatchString(regex, line)
+		if err != nil {
+			logger.Errorf("Error filtering log lines. Error: %s", err)
+			return nil, err
+		}
+
+		if match {
+			filteredLogs = append(filteredLogs, line)
+		}
+	}
+
+	return filteredLogs, nil
+}
+
+// GetFilteredLogs returns the logs filtered by a regexp applied to every line. If the match is ok the log line is accepted.
+// This function can return big log so, please, try not to print the returned value in your tests
+func (mcc *Controller) GetFilteredLogs(regex string) (string, error) {
+	logs, err := mcc.GetFilteredLogsAsList(regex)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.Join(logs, "\n"), nil
+}
+
 // GetNode return the node where the machine controller is running
 func (mcc *Controller) GetNode() (*Node, error) {
 	controllerPodName, err := mcc.GetCachedPodName()

test/extended-priv/node.go

@@ -1386,9 +1386,20 @@ func (nl NodeList) GetAllCoreOsWokerNodesOrFail() []*Node {
 
 	workers, err := nl.GetAll()
 	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(len(workers)).NotTo(o.Equal(0), "no RHCOS worker nodes found for selector node-role.kubernetes.io/worker=,node.openshift.io/os_id=rhel")
 	return workers
 }
 
+// GetAllCoreOsNodesOrFail returns all RHCOS nodes. Fails the test if an error happens.
+func (nl NodeList) GetAllCoreOsNodesOrFail() []*Node {
+	nl.ByLabel("node.openshift.io/os_id=rhel")
+
+	allRhcos, err := nl.GetAll()
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(len(allRhcos)).NotTo(o.Equal(0), "no RHCOS nodes found for selector node.openshift.io/os_id=rhel")
+	return allRhcos
+}
+
 // GetAllReady returns all nodes that are in Ready status
 func (nl NodeList) GetAllReady() ([]*Node, error) {
 	allNodes, err := nl.GetAll()
@@ -1586,3 +1597,15 @@ func (n *Node) GetRHCOSVersion() (string, error) {
 
 	return rhcosVersion, nil
 }
+
+// FilterSchedulableNodesOrFail filters a list of nodes to return only schedulable ones
+func FilterSchedulableNodesOrFail(nodes []*Node) []*Node {
+	returnNodes := []*Node{}
+	for _, item := range nodes {
+		node := item
+		if node.IsSchedulableOrFail() {
+			returnNodes = append(returnNodes, node)
+		}
+	}
+	return returnNodes
+}

test/extended-priv/testdata/files/add-mc-to-trigger-node-drain.yaml

@@ -0,0 +1,25 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+ name: mc-template
+objects:
+- kind: MachineConfig
+  apiVersion: machineconfiguration.openshift.io/v1
+  metadata:
+    labels:
+      machineconfiguration.openshift.io/role: "${POOL}"
+    name: "${NAME}"
+  spec:
+    config:
+      ignition:
+        version: 3.2.0
+      storage:
+        files:
+          - contents:
+              source: data:text/plain;charset=utf;base64,c2VydmVyIGZvby5leGFtcGxlLm5ldCBtYXhkZWxheSAwLjQgb2ZmbGluZQpzZXJ2ZXIgYmFyLmV4YW1wbGUubmV0IG1heGRlbGF5IDAuNCBvZmZsaW5lCnNlcnZlciBiYXouZXhhbXBsZS5uZXQgbWF4ZGVsYXkgMC40IG9mZmxpbmUK
+            filesystem: root
+            mode: 0644
+            path: /etc/test
+parameters:
+  - name: NAME
+  - name: POOL

test/extended-priv/testdata/files/create-pod.yaml

@@ -0,0 +1,33 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  name: create-pod
+objects:
+  - kind: Pod
+    apiVersion: v1
+    metadata:
+      labels:
+        app: "${NAME}"
+      name: "${NAME}"
+    spec:
+      containers:
+        - args:
+            - sleep
+            - 4h
+          image: quay.io/openshifttest/busybox@sha256:c5439d7db88ab5423999530349d327b04279ad3161d7596d2126dfb5b02bfd1f
+          imagePullPolicy: Always
+          name: "${NAME}"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+      nodeSelector:
+        kubernetes.io/hostname: "${HOSTNAME}"
+      restartPolicy: Never
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+            type: "RuntimeDefault"
+parameters:
+  - name: NAME
+  - name: HOSTNAME

test/extended-priv/testdata/files/pod-disruption-budget.yaml

@@ -0,0 +1,16 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  name: pod-disruption-budget
+objects:
+  - kind: PodDisruptionBudget
+    apiVersion: policy/v1
+    metadata:
+      name: "${NAME}"
+    spec:
+      minAvailable: 1
+      selector:
+        matchLabels:
+          app: "${NAME}"
+parameters:
+  - name: NAME

test/extended-priv/util.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
+	"net"
 	"net/url"
 	"os"
 	"os/exec"
@@ -1267,3 +1268,78 @@ func skipTestIfOsIsNotCoreOs(oc *exutil.CLI) *Node {
 	}
 	return allCoreOs[0]
 }
+
+// getSATokenFromContainer gets the service account token from a container
+func getSATokenFromContainer(oc *exutil.CLI, podName, podNamespace, container string) string {
+	podOut, err := exutil.RemoteShContainer(oc, podNamespace, podName, container, "cat", "/var/run/secrets/kubernetes.io/serviceaccount/token")
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	return strings.TrimSpace(podOut)
+}
+
+// getHostFromRoute gets the host from a route
+func getHostFromRoute(oc *exutil.CLI, routeName, routeNamespace string) string {
+	stdout, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("route", routeName, "-n", routeNamespace, "-o", "jsonpath={.spec.host}").Output()
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	return stdout
+}
+
+// getPrometheusQueryResults executes a Prometheus query and returns the results
+func getPrometheusQueryResults(oc *exutil.CLI, query string) string {
+
+	token := getSATokenFromContainer(oc, "prometheus-k8s-0", "openshift-monitoring", "prometheus")
+
+	routeHost := getHostFromRoute(oc, "prometheus-k8s", "openshift-monitoring")
+	url := fmt.Sprintf("https://%s/api/v1/query?query=%s", routeHost, query)
+	headers := fmt.Sprintf("Authorization: Bearer %s", token)
+
+	curlCmd := fmt.Sprintf("curl -ks -H '%s' %s", headers, url)
+	logger.Infof("curl cmd:\n %s", curlCmd)
+
+	curlOutput, cmdErr := exec.Command("bash", "-c", curlCmd).Output()
+	o.Expect(cmdErr).NotTo(o.HaveOccurred())
+
+	return string(curlOutput)
+}
+
+// getMachineConfigOperatorPod gets the machine-config-operator pod name
+func getMachineConfigOperatorPod(oc *exutil.CLI) (string, error) {
+	pods, err := exutil.GetAllPodsWithLabel(oc.AsAdmin(), MachineConfigNamespace, "k8s-app=machine-config-operator")
+	if err != nil {
+		return "", err
+	}
+	if len(pods) == 0 {
+		return "", fmt.Errorf("no machine-config-operator pod found")
+	}
+	logger.Infof("machine-config-operator pod name is %s", pods[0])
+	return pods[0], nil
+}
+
+// WrapWithBracketsIfIpv6 wraps IPv6 addresses with brackets for use in URLs
+func WrapWithBracketsIfIpv6(ip string) (string, error) {
+	parsedIP := net.ParseIP(ip)
+	if parsedIP == nil {
+		return "", fmt.Errorf("The string %s is not a valid IP", ip)
+	}
+
+	// If it is an IPV6 address, wrap it
+	if parsedIP.To4() == nil {
+		return "[" + ip + "]", nil
+	}
+
+	return ip, nil
+}
+
+// ToInterfaceSlice converts a gjson.Result array to a slice of interfaces
+func ToInterfaceSlice(r gjson.Result) []interface{} {
+	if !r.IsArray() {
+		return nil
+	}
+	arr := r.Array()
+	result := make([]interface{}, len(arr))
+	for i, elem := range arr {
+		result[i] = elem.Value()
+	}
+	return result
+}