WIP: OCPBUGS-80958: Kube apiserver node replace

go.mod

@@ -2,6 +2,10 @@ module github.com/openshift/library-go
 
 go 1.25.0
 
+replace github.com/openshift/api => github.com/bhperry/openshift-api v0.0.0-20260430171308-07f1fbedc28f
+
+replace github.com/openshift/client-go => github.com/bhperry/openshift-client-go v0.0.0-20260430171600-439d308fac91
+
 require (
 	github.com/RangelReale/osincli v0.0.0-20160924135400-fababb0555f2
 	github.com/blang/semver/v4 v4.0.0

go.sum

@@ -20,6 +20,10 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bhperry/openshift-api v0.0.0-20260430171308-07f1fbedc28f h1:R930pInp3YsxDUcvx8nXgQyga1z2f/xcnDEbtM44rRo=
+github.com/bhperry/openshift-api v0.0.0-20260430171308-07f1fbedc28f/go.mod h1:pyVjK0nZ4sRs4fuQVQ4rubsJdahI1PB94LnQ8sGdvxo=
+github.com/bhperry/openshift-client-go v0.0.0-20260430171600-439d308fac91 h1:UeTG9/NiL9hTpcnLvucw0wkgCdG9evZJ/fIhwVjQemQ=
+github.com/bhperry/openshift-client-go v0.0.0-20260430171600-439d308fac91/go.mod h1:fcdSl1rcCdZOKgn3zH8C7UbIykq6PeXVRB1EFL6xJKI=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@@ -225,12 +229,8 @@ github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrB
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/selinux v1.13.0 h1:Zza88GWezyT7RLql12URvoxsbLfjFx988+LGaWfbL84=
 github.com/opencontainers/selinux v1.13.0/go.mod h1:XxWTed+A/s5NNq4GmYScVy+9jzXhGBVEOAyucdRUY8s=
-github.com/openshift/api v0.0.0-20260429122012-1180c0f5c3e9 h1:lZw6pYY7El1giNk1lYvkp6hLungiqwIOqLlH+Hm7w9g=
-github.com/openshift/api v0.0.0-20260429122012-1180c0f5c3e9/go.mod h1:pyVjK0nZ4sRs4fuQVQ4rubsJdahI1PB94LnQ8sGdvxo=
 github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee h1:+Sp5GGnjHDhT/a/nQ1xdp43UscBMr7G5wxsYotyhzJ4=
 github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE=
-github.com/openshift/client-go v0.0.0-20260429123927-c81f86abfa6a h1:4GR6seHvlfv0rADe+LCQx63FqSExx6gaSo8uNiyWq+c=
-github.com/openshift/client-go v0.0.0-20260429123927-c81f86abfa6a/go.mod h1:Lm7X7aYbAaKhGsNhgYaowP7hiLKwfN/w0r+Q6VlQoI8=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

pkg/operator/staticpod/controller/node/node_controller.go

@@ -7,6 +7,7 @@ import (
 
 	coreapiv1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/informers"
 	corelisterv1 "k8s.io/client-go/listers/core/v1"
 
@@ -84,22 +85,31 @@ func (c *NodeController) sync(ctx context.Context, syncCtx factory.SyncContext)
 		nodes = append(nodes, extraNodes...)
 	}
 
+	nodeUIDs := map[string]types.UID{}
+	for _, node := range nodes {
+		nodeUIDs[node.Name] = node.UID
+	}
+
 	jsonPatch := jsonpatch.New()
 	var removedNodeStatusesCounter int
 	newTargetNodeStates := []*applyoperatorv1.NodeStatusApplyConfiguration{}
 	// remove entries for missing nodes
 	for i, nodeState := range originalOperatorStatus.NodeStatuses {
-		found := false
-		for _, node := range nodes {
-			if nodeState.NodeName == node.Name {
-				found = true
+		currentUID, found := nodeUIDs[nodeState.NodeName]
+		if found && (nodeState.NodeUID == currentUID || nodeState.NodeUID == "") {
+			if nodeState.NodeUID == "" {
+				syncCtx.Recorder().Warningf("MasterNodeUnknownUID", "Node %s has no recorded UID. If this node was replaced its status may need to be manually removed from operator objects.")
 			}
-		}
-		if found {
-			newTargetNodeState := applyoperatorv1.NodeStatus().WithNodeName(originalOperatorStatus.NodeStatuses[i].NodeName)
+			newTargetNodeState := applyoperatorv1.NodeStatus().
+				WithNodeName(nodeState.NodeName).
+				WithNodeUID(currentUID)
 			newTargetNodeStates = append(newTargetNodeStates, newTargetNodeState)
 		} else {
-			syncCtx.Recorder().Warningf("MasterNodeRemoved", "Observed removal of master node %s", nodeState.NodeName)
+			if found {
+				syncCtx.Recorder().Warningf("MasterNodeReplaced", "Observed replacement of master node %s (old UID: %s, new UID: %s)", nodeState.NodeName, nodeState.NodeUID, currentUID)
+			} else {
+				syncCtx.Recorder().Warningf("MasterNodeRemoved", "Observed removal of master node %s", nodeState.NodeName)
+			}
 			// each delete operation is applied to the object,
 			// which modifies the array. Thus, we need to
 			// adjust the indices to find the correct node to remove.
@@ -116,16 +126,17 @@ func (c *NodeController) sync(ctx context.Context, syncCtx factory.SyncContext)
 	for _, node := range nodes {
 		found := false
 		for _, nodeState := range originalOperatorStatus.NodeStatuses {
-			if nodeState.NodeName == node.Name {
+			if nodeState.NodeName == node.Name && (nodeState.NodeUID == node.UID || nodeState.NodeUID == "") {
 				found = true
+				break
 			}
 		}
 		if found {
 			continue
 		}
 
-		syncCtx.Recorder().Eventf("MasterNodeObserved", "Observed new master node %s", node.Name)
-		newTargetNodeState := applyoperatorv1.NodeStatus().WithNodeName(node.Name)
+		syncCtx.Recorder().Eventf("MasterNodeObserved", "Observed new master node %s (UID: %s)", node.Name, node.UID)
+		newTargetNodeState := applyoperatorv1.NodeStatus().WithNodeName(node.Name).WithNodeUID(node.UID)
 		newTargetNodeStates = append(newTargetNodeStates, newTargetNodeState)
 	}
 

pkg/operator/staticpod/controller/node/node_controller_test.go

@@ -13,6 +13,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/fake"
 	clientgotesting "k8s.io/client-go/testing"
 	clocktesting "k8s.io/utils/clock/testing"
@@ -28,6 +29,7 @@ import (
 func fakeMasterNode(name string) *corev1.Node {
 	n := &corev1.Node{}
 	n.Name = name
+	n.UID = types.UID(name + "-uid")
 	n.Labels = map[string]string{
 		"node-role.kubernetes.io/master": "",
 	}
@@ -729,3 +731,143 @@ func TestNewNodeController(t *testing.T) {
 
 	}
 }
+
+func TestNodeControllerTrackNodeUIDs(t *testing.T) {
+	tests := []struct {
+		name               string
+		startNodes         []runtime.Object
+		startNodeStatus    []operatorv1.NodeStatus
+		expectedNodes      [][2]string
+		evaluateNodeStatus func([]operatorv1.NodeStatus) error
+	}{
+		{
+			name:       "node exists missing uid in status",
+			startNodes: []runtime.Object{fakeMasterNode("test-node-1")},
+			startNodeStatus: []operatorv1.NodeStatus{
+				{
+					NodeName: "test-node-1",
+				},
+			},
+			evaluateNodeStatus: func(s []operatorv1.NodeStatus) error {
+				if len(s) != 1 {
+					return fmt.Errorf("expected 1 node status, got %d", len(s))
+				}
+				if s[0].NodeName != "test-node-1" {
+					return fmt.Errorf("expected 'test-node-1' as node name, got %q", s[0].NodeName)
+				}
+				if s[0].NodeUID != "test-node-1-uid" {
+					return fmt.Errorf("expected 'test-node-1-uid' as node UID, got %q", s[0].NodeUID)
+				}
+				return nil
+			},
+		},
+		{
+			name:       "node exists matching uid in status",
+			startNodes: []runtime.Object{fakeMasterNode("test-node-1")},
+			startNodeStatus: []operatorv1.NodeStatus{
+				{
+					NodeName: "test-node-1",
+					NodeUID:  "test-node-1-uid",
+				},
+			},
+			evaluateNodeStatus: func(s []operatorv1.NodeStatus) error {
+				if len(s) != 1 {
+					return fmt.Errorf("expected 1 node status, got %d", len(s))
+				}
+				if s[0].NodeName != "test-node-1" {
+					return fmt.Errorf("expected 'test-node-1' as node name, got %q", s[0].NodeName)
+				}
+				if s[0].NodeUID != "test-node-1-uid" {
+					return fmt.Errorf("expected 'test-node-1-uid' as node UID, got %q", s[0].NodeUID)
+				}
+				return nil
+			},
+		},
+		{
+			name:       "node replaced mismatched uid in status",
+			startNodes: []runtime.Object{fakeMasterNode("test-node-1")},
+			startNodeStatus: []operatorv1.NodeStatus{
+				{
+					NodeName: "test-node-1",
+					NodeUID:  "test-node-1-uid-init",
+				},
+			},
+			evaluateNodeStatus: func(s []operatorv1.NodeStatus) error {
+				if len(s) != 1 {
+					return fmt.Errorf("expected 1 node status, got %d", len(s))
+				}
+				if s[0].NodeName != "test-node-1" {
+					return fmt.Errorf("expected 'test-node-1' as node name, got %q", s[0].NodeName)
+				}
+				if s[0].NodeUID != "test-node-1-uid" {
+					return fmt.Errorf("expected 'test-node-1-uid' as node UID, got %q", s[0].NodeUID)
+				}
+				return nil
+			},
+		},
+		{
+			name:       "node removed",
+			startNodes: []runtime.Object{fakeMasterNode("test-node-2")},
+			startNodeStatus: []operatorv1.NodeStatus{
+				{
+					NodeName: "test-node-1",
+					NodeUID:  "test-node-1-uid",
+				},
+			},
+			evaluateNodeStatus: func(s []operatorv1.NodeStatus) error {
+				if len(s) != 1 {
+					return fmt.Errorf("expected 1 node status, got %d", len(s))
+				}
+				if s[0].NodeName != "test-node-2" {
+					return fmt.Errorf("expected 'test-node-2' as node name, got %q", s[0].NodeName)
+				}
+				if s[0].NodeUID != "test-node-2-uid" {
+					return fmt.Errorf("expected 'test-node-2-uid' as node UID, got %q", s[0].NodeUID)
+				}
+				return nil
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			kubeClient := fake.NewSimpleClientset(test.startNodes...)
+			fakeLister := v1helpers.NewFakeNodeLister(kubeClient)
+			fakeStaticPodOperatorClient := v1helpers.NewFakeStaticPodOperatorClient(
+				&operatorv1.StaticPodOperatorSpec{
+					OperatorSpec: operatorv1.OperatorSpec{
+						ManagementState: operatorv1.Managed,
+					},
+				},
+				&operatorv1.StaticPodOperatorStatus{
+					OperatorStatus: operatorv1.OperatorStatus{
+						LatestAvailableRevision: 1,
+					},
+					NodeStatuses: test.startNodeStatus,
+				},
+				nil,
+				nil,
+			)
+
+			eventRecorder := events.NewRecorder(kubeClient.CoreV1().Events("test"), "test-operator", &corev1.ObjectReference{}, clocktesting.NewFakePassiveClock(time.Now()))
+
+			c := &NodeController{
+				operatorClient:      fakeStaticPodOperatorClient,
+				nodeLister:          fakeLister,
+				masterNodesSelector: masterNodesSelector(t),
+			}
+
+			// override the lister so we don't have to run the informer to list nodes
+			c.nodeLister = fakeLister
+			if err := c.sync(context.TODO(), factory.NewSyncContext("NodeController", eventRecorder)); err != nil {
+				t.Fatal(err)
+			}
+
+			_, status, _, _ := fakeStaticPodOperatorClient.GetStaticPodOperatorState()
+			if err := test.evaluateNodeStatus(status.NodeStatuses); err != nil {
+				t.Errorf("%s: failed to evaluate node status: %v", test.name, err)
+			}
+		})
+
+	}
+}

pkg/operator/v1helpers/test_helpers.go

@@ -470,6 +470,7 @@ func mergeStaticPodOperatorStatusApplyConfiguration(currentOperatorStatus *v1.Op
 	for _, nodeStatus := range applyConfiguration.NodeStatuses {
 		newNodeStatus := operatorv1.NodeStatus{
 			NodeName:                 ptr.Deref(nodeStatus.NodeName, ""),
+			NodeUID:                  ptr.Deref(nodeStatus.NodeUID, ""),
 			CurrentRevision:          ptr.Deref(nodeStatus.CurrentRevision, 0),
 			TargetRevision:           ptr.Deref(nodeStatus.TargetRevision, 0),
 			LastFailedRevision:       ptr.Deref(nodeStatus.LastFailedRevision, 0),