diff --git a/test/extended/edge_topologies/tnf_fencing_credentials.go b/test/extended/edge_topologies/tnf_fencing_credentials.go new file mode 100644 index 000000000000..6f0409e5033e --- /dev/null +++ b/test/extended/edge_topologies/tnf_fencing_credentials.go @@ -0,0 +1,256 @@ +package edge_topologies + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + mathrand "math/rand" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + v1 "github.com/openshift/api/config/v1" + "github.com/openshift/origin/test/extended/edge_topologies/utils" + "github.com/openshift/origin/test/extended/edge_topologies/utils/apis" + "github.com/openshift/origin/test/extended/edge_topologies/utils/core" + "github.com/openshift/origin/test/extended/edge_topologies/utils/services" + "github.com/openshift/origin/test/extended/etcd/helpers" + exutil "github.com/openshift/origin/test/extended/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + fencingHealthTimeout = time.Minute +) + +func secureRandomString(n int) (string, error) { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b)[:n], nil +} + +var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:DualReplica][Suite:openshift/two-node][Serial] Fencing credentials", func() { + defer g.GinkgoRecover() + + var ( + oc = exutil.NewCLIWithoutNamespace("").AsAdmin() + etcdClientFactory *helpers.EtcdClientFactoryImpl + peerNode, targetNode corev1.Node + ) + + g.BeforeEach(func() { + utils.SkipIfNotTopology(oc, v1.DualReplicaTopologyMode) + + etcdClientFactory = helpers.NewEtcdClientFactory(oc.KubeClient()) + + utils.SkipIfClusterIsNotHealthy(oc, etcdClientFactory) + + nodes, err := utils.GetNodes(oc, utils.AllNodes) + o.Expect(err).ShouldNot(o.HaveOccurred(), "Expected to retrieve nodes without error") + o.Expect(nodes.Items).To(o.HaveLen(2), "Expected exactly two nodes for dual-replica fencing test") + + randomIndex := mathrand.Intn(len(nodes.Items)) + peerNode = nodes.Items[randomIndex] + targetNode = nodes.Items[(randomIndex+1)%len(nodes.Items)] + + g.DeferCleanup(func() { + logFinalClusterStatus([]corev1.Node{peerNode, targetNode}) + }) + }) + + g.It("should update fencing credentials and validate stonith health", func() { + bmcNode := targetNode + survivedNode := peerNode + + g.By(fmt.Sprintf("Reading current fencing credentials for node %s", bmcNode.Name)) + creds, err := apis.FindFencingCredentialsByNodeName(oc, bmcNode.Name) + o.Expect(err).ToNot(o.HaveOccurred(), "expected to find fencing credentials secret") + framework.Logf("Found fencing credentials secret %s (address: %s, username: %s)", + creds.SecretName, creds.Address, creds.Username) + + g.By("Parsing Redfish address from fencing credentials") + redfishHost, redfishPort, redfishPath, err := apis.ParseRedfishAddress(creds.Address) + o.Expect(err).ToNot(o.HaveOccurred(), "expected to parse Redfish address") + framework.Logf("Redfish endpoint: host=%s port=%s path=%s", redfishHost, redfishPort, redfishPath) + + isSushy := apis.IsSushyEmulator(redfishPath) + var hypervisorSSH *core.SSHConfig + var hypervisorKnownHosts string + if isSushy { + if !exutil.HasHypervisorConfig() { + g.Skip("sushy-tools detected but no hypervisor SSH config available") + } + sshCfg := exutil.GetHypervisorConfig() + o.Expect(sshCfg).ToNot(o.BeNil(), "expected hypervisor config to parse") + hypervisorSSH = &core.SSHConfig{ + IP: sshCfg.HypervisorIP, + User: sshCfg.SSHUser, + PrivateKeyPath: sshCfg.PrivateKeyPath, + } + var khErr error + hypervisorKnownHosts, khErr = core.PrepareLocalKnownHostsFile(hypervisorSSH) + o.Expect(khErr).ToNot(o.HaveOccurred(), "expected to prepare hypervisor known_hosts") + framework.Logf("Using sushy-tools password change via hypervisor SSH (%s)", hypervisorSSH.IP) + } + + changeBMCPassword := func(currentPw, newPw string) error { + if isSushy { + return apis.ChangeSushyToolsPassword(creds.Username, newPw, hypervisorSSH, hypervisorKnownHosts) + } + return apis.ChangeBMCPasswordViaRedfish(oc, bmcNode.Name, redfishHost, redfishPort, + creds.Username, currentPw, newPw) + } + + hasPacemakerCR := apis.IsPacemakerClusterAvailable(oc) + if hasPacemakerCR { + g.By("Verifying PacemakerCluster CR is healthy before credential change") + pc, pcErr := apis.GetPacemakerCluster(oc) + o.Expect(pcErr).ToNot(o.HaveOccurred(), "expected to get PacemakerCluster CR") + o.Expect(apis.ExpectClusterHealthy(pc)).ToNot(o.HaveOccurred(), "expected PacemakerCluster to be healthy before credential change") + o.Expect(apis.ExpectNodeFencingHealthy(pc, bmcNode.Name)).ToNot(o.HaveOccurred(), + "expected fencing to be healthy for %s before credential change", bmcNode.Name) + } else { + framework.Logf("PacemakerCluster CRD not available, skipping CR health checks") + } + + sslInsecure := creds.CertificateVerification == "Disabled" + originalPassword := creds.Password + newPassword, err := secureRandomString(32) + o.Expect(err).ToNot(o.HaveOccurred(), "expected to generate a secure BMC password") + nodeIdentifier := strings.TrimPrefix(creds.SecretName, "fencing-credentials-") + + scriptPath := "/etc/kubernetes/static-pod-resources/etcd-certs/configmaps/etcd-scripts/update-fencing-credentials.sh" + bashCmd := scriptPath + ` --node "$1" --username "$2" --password "$3" --address "$4"` + if sslInsecure { + bashCmd += " --ssl-insecure" + } + + // On sushy-tools, a single htpasswd file serves all BMC endpoints, so changing + // the password affects all nodes. Fetch the survived node's credentials so we + // can update its stonith device and secret in lockstep. + var survivedNodeCreds *apis.FencingCredentials + var survivedNodeIdentifier string + var survivedBashCmd string + if isSushy { + survivedNodeCreds, err = apis.FindFencingCredentialsByNodeName(oc, survivedNode.Name) + o.Expect(err).ToNot(o.HaveOccurred(), "expected to find survived node fencing credentials") + survivedNodeIdentifier = strings.TrimPrefix(survivedNodeCreds.SecretName, "fencing-credentials-") + survivedBashCmd = scriptPath + ` --node "$1" --username "$2" --password "$3" --address "$4"` + if survivedNodeCreds.CertificateVerification == "Disabled" { + survivedBashCmd += " --ssl-insecure" + } + framework.Logf("sushy-tools: will also update survived node %s credentials (secret: %s)", + survivedNode.Name, survivedNodeCreds.SecretName) + } + + bmcPasswordChanged := false + g.DeferCleanup(func() { + var cleanupFailed bool + + if bmcPasswordChanged { + framework.Logf("Restoring original BMC password") + if restoreErr := changeBMCPassword(newPassword, originalPassword); restoreErr != nil { + fmt.Fprintf(g.GinkgoWriter, "Warning: failed to restore BMC password: %v\n", restoreErr) + cleanupFailed = true + } + } else { + framework.Logf("Skipping BMC password restore because the password change did not complete") + } + + scriptPassword := originalPassword + if bmcPasswordChanged && cleanupFailed { + scriptPassword = newPassword + } + + framework.Logf("Re-running update-fencing-credentials.sh with original credentials") + output, restoreErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, bmcNode.Name, "openshift-etcd", + "bash", "-c", bashCmd, "update-fencing-credentials", + nodeIdentifier, creds.Username, scriptPassword, creds.Address) + if restoreErr != nil { + fmt.Fprintf(g.GinkgoWriter, "Warning: failed to restore fencing credentials via script: %v\noutput: %s\n", + restoreErr, output) + } + + if isSushy { + framework.Logf("Restoring survived node %s fencing credentials (sushy-tools shares credentials)", survivedNode.Name) + survivedOutput, survivedErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, survivedNode.Name, "openshift-etcd", + "bash", "-c", survivedBashCmd, "update-fencing-credentials", + survivedNodeIdentifier, survivedNodeCreds.Username, scriptPassword, survivedNodeCreds.Address) + if survivedErr != nil { + fmt.Fprintf(g.GinkgoWriter, "Warning: failed to restore survived node fencing credentials: %v\noutput: %s\n", + survivedErr, survivedOutput) + } + } + }) + + g.By(fmt.Sprintf("Changing BMC password on %s", bmcNode.Name)) + err = changeBMCPassword(originalPassword, newPassword) + o.Expect(err).ToNot(o.HaveOccurred(), "expected to change BMC password") + bmcPasswordChanged = true + + g.By(fmt.Sprintf("Validating new BMC credentials via fence_redfish on %s", bmcNode.Name)) + err = apis.ValidateBMCCredentials(oc, bmcNode.Name, redfishHost, redfishPort, redfishPath, + creds.Username, newPassword, sslInsecure) + o.Expect(err).ToNot(o.HaveOccurred(), "expected new BMC credentials to be valid") + + g.By(fmt.Sprintf("Running update-fencing-credentials.sh on %s with new credentials", bmcNode.Name)) + output, err := exutil.DebugNodeRetryWithOptionsAndChroot(oc, bmcNode.Name, "openshift-etcd", + "bash", "-c", bashCmd, "update-fencing-credentials", + nodeIdentifier, creds.Username, newPassword, creds.Address) + o.Expect(err).ToNot(o.HaveOccurred(), "expected update-fencing-credentials.sh to succeed") + framework.Logf("update-fencing-credentials.sh output:\n%s", output) + + if isSushy { + g.By(fmt.Sprintf("Updating survived node %s fencing credentials (sushy-tools shares credentials)", survivedNode.Name)) + survivedOutput, survivedErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, survivedNode.Name, "openshift-etcd", + "bash", "-c", survivedBashCmd, "update-fencing-credentials", + survivedNodeIdentifier, survivedNodeCreds.Username, newPassword, survivedNodeCreds.Address) + o.Expect(survivedErr).ToNot(o.HaveOccurred(), + "expected update-fencing-credentials.sh for survived node to succeed") + framework.Logf("update-fencing-credentials.sh output for survived node:\n%s", survivedOutput) + } + + g.By("Validating pacemaker health after credential update") + ctx, cancel := context.WithTimeout(context.Background(), fencingHealthTimeout) + defer cancel() + pcsOutput, err := services.PcsStatusViaDebug(ctx, oc, bmcNode.Name) + o.Expect(err).ToNot(o.HaveOccurred(), "expected pcs status to succeed") + failedActions := services.ExtractPcsFailedActions(pcsOutput) + o.Expect(failedActions).To(o.BeEmpty(), "expected no failed pacemaker resource actions after credential update") + + g.By("Ensuring etcd members remain healthy after fencing credentials update") + o.Eventually(func() error { + if err := helpers.EnsureHealthyMember(g.GinkgoT(), etcdClientFactory, survivedNode.Name); err != nil { + return err + } + if err := helpers.EnsureHealthyMember(g.GinkgoT(), etcdClientFactory, bmcNode.Name); err != nil { + return err + } + return nil + }, fencingHealthTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), + "etcd members should be healthy after fencing credentials update") + + if hasPacemakerCR { + g.By("Verifying PacemakerCluster CR remains healthy after credential update") + o.Eventually(func() error { + pc, pcErr := apis.GetPacemakerCluster(oc) + if pcErr != nil { + return pcErr + } + if pcErr = apis.ExpectClusterHealthy(pc); pcErr != nil { + return pcErr + } + if pcErr = apis.ExpectNodeFencingHealthy(pc, bmcNode.Name); pcErr != nil { + return pcErr + } + return apis.ExpectNodeFencingHealthy(pc, survivedNode.Name) + }, fencingHealthTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), + "expected PacemakerCluster to remain healthy after credential update") + } + }) +}) diff --git a/test/extended/edge_topologies/tnf_recovery.go b/test/extended/edge_topologies/tnf_recovery.go index bbccdc2a9016..af226d0502c4 100644 --- a/test/extended/edge_topologies/tnf_recovery.go +++ b/test/extended/edge_topologies/tnf_recovery.go @@ -21,7 +21,6 @@ import ( exutil "github.com/openshift/origin/test/extended/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8srand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/kubernetes/test/e2e/framework" ) @@ -415,230 +414,6 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual memberPromotedVotingTimeout, utils.FiveSecondPollInterval) }) - g.It("should update fencing credentials and validate fencing with updated credentials", func() { - bmcNode := targetNode - survivedNode := peerNode - - g.By(fmt.Sprintf("Reading current fencing credentials for node %s", bmcNode.Name)) - creds, err := apis.FindFencingCredentialsByNodeName(oc, bmcNode.Name) - o.Expect(err).ToNot(o.HaveOccurred(), "expected to find fencing credentials secret") - framework.Logf("Found fencing credentials secret %s (address: %s, username: %s)", - creds.SecretName, creds.Address, creds.Username) - - g.By("Parsing Redfish address from fencing credentials") - redfishHost, redfishPort, redfishPath, err := apis.ParseRedfishAddress(creds.Address) - o.Expect(err).ToNot(o.HaveOccurred(), "expected to parse Redfish address") - framework.Logf("Redfish endpoint: host=%s port=%s path=%s", redfishHost, redfishPort, redfishPath) - - isSushy := apis.IsSushyEmulator(redfishPath) - var hypervisorSSH *core.SSHConfig - var hypervisorKnownHosts string - if isSushy { - if !exutil.HasHypervisorConfig() { - g.Skip("sushy-tools detected but no hypervisor SSH config available") - } - sshCfg := exutil.GetHypervisorConfig() - o.Expect(sshCfg).ToNot(o.BeNil(), "expected hypervisor config to parse") - hypervisorSSH = &core.SSHConfig{ - IP: sshCfg.HypervisorIP, - User: sshCfg.SSHUser, - PrivateKeyPath: sshCfg.PrivateKeyPath, - } - var khErr error - hypervisorKnownHosts, khErr = core.PrepareLocalKnownHostsFile(hypervisorSSH) - o.Expect(khErr).ToNot(o.HaveOccurred(), "expected to prepare hypervisor known_hosts") - framework.Logf("Using sushy-tools password change via hypervisor SSH (%s)", hypervisorSSH.IP) - } - - changeBMCPassword := func(currentPw, newPw string) error { - if isSushy { - return apis.ChangeSushyToolsPassword(creds.Username, newPw, hypervisorSSH, hypervisorKnownHosts) - } - return apis.ChangeBMCPasswordViaRedfish(oc, bmcNode.Name, redfishHost, redfishPort, - creds.Username, currentPw, newPw) - } - - hasPacemakerCR := apis.IsPacemakerClusterAvailable(oc) - if hasPacemakerCR { - g.By("Verifying PacemakerCluster CR is healthy before credential change") - pc, pcErr := apis.GetPacemakerCluster(oc) - o.Expect(pcErr).ToNot(o.HaveOccurred(), "expected to get PacemakerCluster CR") - o.Expect(apis.ExpectClusterHealthy(pc)).ToNot(o.HaveOccurred(), "expected PacemakerCluster to be healthy before credential change") - o.Expect(apis.ExpectNodeFencingHealthy(pc, bmcNode.Name)).ToNot(o.HaveOccurred(), - "expected fencing to be healthy for %s before credential change", bmcNode.Name) - } else { - framework.Logf("PacemakerCluster CRD not available, skipping CR health checks") - } - - sslInsecure := creds.CertificateVerification == "Disabled" - originalPassword := creds.Password - newPassword := k8srand.String(32) - nodeIdentifier := strings.TrimPrefix(creds.SecretName, "fencing-credentials-") - - scriptPath := "/etc/kubernetes/static-pod-resources/etcd-certs/configmaps/etcd-scripts/update-fencing-credentials.sh" - bashCmd := scriptPath + ` --node "$1" --username "$2" --password "$3" --address "$4"` - if sslInsecure { - bashCmd += " --ssl-insecure" - } - - // On sushy-tools, a single htpasswd file serves all BMC endpoints, so changing - // the password affects all nodes. Fetch the survived node's credentials so we - // can update its stonith device and secret in lockstep. - var survivedNodeCreds *apis.FencingCredentials - var survivedNodeIdentifier string - var survivedBashCmd string - if isSushy { - survivedNodeCreds, err = apis.FindFencingCredentialsByNodeName(oc, survivedNode.Name) - o.Expect(err).ToNot(o.HaveOccurred(), "expected to find survived node fencing credentials") - survivedNodeIdentifier = strings.TrimPrefix(survivedNodeCreds.SecretName, "fencing-credentials-") - survivedBashCmd = scriptPath + ` --node "$1" --username "$2" --password "$3" --address "$4"` - if survivedNodeCreds.CertificateVerification == "Disabled" { - survivedBashCmd += " --ssl-insecure" - } - framework.Logf("sushy-tools: will also update survived node %s credentials (secret: %s)", - survivedNode.Name, survivedNodeCreds.SecretName) - } - - g.DeferCleanup(func() { - var cleanupFailed bool - - framework.Logf("Restoring original BMC password") - if restoreErr := changeBMCPassword(newPassword, originalPassword); restoreErr != nil { - fmt.Fprintf(g.GinkgoWriter, "Warning: failed to restore BMC password: %v\n", restoreErr) - cleanupFailed = true - } - - scriptPassword := originalPassword - if cleanupFailed { - scriptPassword = newPassword - } - - framework.Logf("Re-running update-fencing-credentials.sh with original credentials") - output, restoreErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, bmcNode.Name, "openshift-etcd", - "bash", "-c", bashCmd, "update-fencing-credentials", - nodeIdentifier, creds.Username, scriptPassword, creds.Address) - if restoreErr != nil { - fmt.Fprintf(g.GinkgoWriter, "Warning: failed to restore fencing credentials via script: %v\noutput: %s\n", - restoreErr, output) - } - - if isSushy { - framework.Logf("Restoring survived node %s fencing credentials (sushy-tools shares credentials)", survivedNode.Name) - survivedOutput, survivedErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, survivedNode.Name, "openshift-etcd", - "bash", "-c", survivedBashCmd, "update-fencing-credentials", - survivedNodeIdentifier, survivedNodeCreds.Username, scriptPassword, survivedNodeCreds.Address) - if survivedErr != nil { - fmt.Fprintf(g.GinkgoWriter, "Warning: failed to restore survived node fencing credentials: %v\noutput: %s\n", - survivedErr, survivedOutput) - } - } - }) - - g.By(fmt.Sprintf("Changing BMC password on %s", bmcNode.Name)) - err = changeBMCPassword(originalPassword, newPassword) - o.Expect(err).ToNot(o.HaveOccurred(), "expected to change BMC password") - - g.By(fmt.Sprintf("Validating new BMC credentials via fence_redfish on %s", bmcNode.Name)) - err = apis.ValidateBMCCredentials(oc, bmcNode.Name, redfishHost, redfishPort, redfishPath, - creds.Username, newPassword, sslInsecure) - o.Expect(err).ToNot(o.HaveOccurred(), "expected new BMC credentials to be valid") - - g.By(fmt.Sprintf("Running update-fencing-credentials.sh on %s with new credentials", bmcNode.Name)) - output, err := exutil.DebugNodeRetryWithOptionsAndChroot(oc, bmcNode.Name, "openshift-etcd", - "bash", "-c", bashCmd, "update-fencing-credentials", - nodeIdentifier, creds.Username, newPassword, creds.Address) - o.Expect(err).ToNot(o.HaveOccurred(), "expected update-fencing-credentials.sh to succeed") - framework.Logf("update-fencing-credentials.sh output:\n%s", output) - - if isSushy { - g.By(fmt.Sprintf("Updating survived node %s fencing credentials (sushy-tools shares credentials)", survivedNode.Name)) - survivedOutput, survivedErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, survivedNode.Name, "openshift-etcd", - "bash", "-c", survivedBashCmd, "update-fencing-credentials", - survivedNodeIdentifier, survivedNodeCreds.Username, newPassword, survivedNodeCreds.Address) - o.Expect(survivedErr).ToNot(o.HaveOccurred(), - "expected update-fencing-credentials.sh for survived node to succeed") - framework.Logf("update-fencing-credentials.sh output for survived node:\n%s", survivedOutput) - } - - g.By("Validating pacemaker health after credential update") - ctx, cancel := context.WithTimeout(context.Background(), nodeIsHealthyTimeout) - defer cancel() - pcsOutput, err := services.PcsStatusViaDebug(ctx, oc, bmcNode.Name) - o.Expect(err).ToNot(o.HaveOccurred(), "expected pcs status to succeed") - failedActions := services.ExtractPcsFailedActions(pcsOutput) - o.Expect(failedActions).To(o.BeEmpty(), "expected no failed pacemaker resource actions after credential update") - - g.By("Ensuring etcd members remain healthy after fencing credentials update") - o.Eventually(func() error { - if err := helpers.EnsureHealthyMember(g.GinkgoT(), etcdClientFactory, survivedNode.Name); err != nil { - return err - } - if err := helpers.EnsureHealthyMember(g.GinkgoT(), etcdClientFactory, bmcNode.Name); err != nil { - return err - } - return nil - }, nodeIsHealthyTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), - "etcd members should be healthy after fencing credentials update") - - if hasPacemakerCR { - g.By("Verifying PacemakerCluster CR remains healthy after credential update") - o.Eventually(func() error { - pc, pcErr := apis.GetPacemakerCluster(oc) - if pcErr != nil { - return pcErr - } - if pcErr = apis.ExpectClusterHealthy(pc); pcErr != nil { - return pcErr - } - return apis.ExpectNodeFencingHealthy(pc, bmcNode.Name) - }, nodeIsHealthyTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), - "expected PacemakerCluster to remain healthy after credential update") - } - - g.By(fmt.Sprintf("Triggering fencing-style network disruption between %s and %s", bmcNode.Name, survivedNode.Name)) - command, err := exutil.TriggerNetworkDisruption(oc.KubeClient(), &bmcNode, &survivedNode, networkDisruptionDuration) - o.Expect(err).To(o.BeNil(), "Expected to disrupt network without errors") - framework.Logf("network disruption command: %q", command) - - g.By(fmt.Sprintf("Ensuring cluster recovery after network disruption (timeout: %v)", memberIsLeaderTimeout)) - leaderNode, learnerNode, learnerStarted := validateEtcdRecoveryStateWithoutAssumingLeader(oc, etcdClientFactory, - &survivedNode, &bmcNode, memberIsLeaderTimeout, utils.FiveSecondPollInterval) - - if learnerStarted { - framework.Logf("Learner node %q already started as learner after disruption", learnerNode.Name) - } else { - g.By(fmt.Sprintf("Ensuring '%s' rejoins as learner (timeout: %v)", learnerNode.Name, memberRejoinedLearnerTimeout)) - validateEtcdRecoveryState(oc, etcdClientFactory, - leaderNode, - learnerNode, true, true, - memberRejoinedLearnerTimeout, utils.FiveSecondPollInterval) - } - - g.By(fmt.Sprintf("Ensuring learner node '%s' is promoted back as voting member (timeout: %v)", learnerNode.Name, memberPromotedVotingTimeout)) - validateEtcdRecoveryState(oc, etcdClientFactory, - leaderNode, - learnerNode, true, false, - memberPromotedVotingTimeout, utils.FiveSecondPollInterval) - - if hasPacemakerCR { - g.By("Verifying PacemakerCluster CR is healthy after full recovery") - o.Eventually(func() error { - pc, pcErr := apis.GetPacemakerCluster(oc) - if pcErr != nil { - return pcErr - } - if pcErr = apis.ExpectClusterHealthy(pc); pcErr != nil { - return pcErr - } - if pcErr = apis.ExpectNodeFencingHealthy(pc, bmcNode.Name); pcErr != nil { - return pcErr - } - return apis.ExpectNodeFencingHealthy(pc, survivedNode.Name) - }, nodeIsHealthyTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), - "expected PacemakerCluster to be fully healthy after recovery") - } - }) - g.It("should compute etcd revision bump and preserve backup container after kernel panic recovery", func() { // Note: This test triggers a kernel panic on one node via sysrq trigger, then verifies // the surviving node computes the etcd revision bump as floor(maxRaftIndex * 0.2) per