support managed control plane anti-affinity groups

Makefile

@@ -9,13 +9,13 @@ IMAGE_REPO ?= ghcr.io/oxidecomputer/cluster-api-provider-oxide
 HELM_OCI_REPO ?= $(IMAGE_REPO)/helm-charts
 IMAGE_TAG ?= dev
 IMG ?= $(IMAGE_REPO):$(IMAGE_TAG)
-KO_DOCKER_REPO ?= $(IMAGE_REPO)
+export KO_DOCKER_REPO ?= $(IMAGE_REPO)
 KOCACHE ?= ~/.ko
 MAKEFILE_PATH := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 TOOLS_MOD := $(MAKEFILE_PATH)/tools/go.mod
 GO_TOOL := go tool -modfile=$(TOOLS_MOD)
 NAMESPACE ?= capox-system
-HELM_VERSION ?= v4.2.2
+HELM_VERSION ?= v4.2.3
 # CAPI contract version implemented by the provider; stamped on every release
 # series entry when hack/gen-capi-metadata.sh generates metadata.yaml. Bump it
 # (and add a series boundary in the script) when migrating to a new contract.
@@ -168,7 +168,7 @@ export PATH := $(LOCALBIN):$(PATH)
 CONTROLLER_GEN ?= $(GO_TOOL) controller-gen
 KUBECTL ?= $(LOCALBIN)/kubectl
 KIND ?= $(GO_TOOL) kind
-KO ?= KO_CACHE=$(KO_CACHE) $(GO_TOOL) ko
+KO ?= KOCACHE=$(KOCACHE) $(GO_TOOL) ko
 KUSTOMIZE ?= $(GO_TOOL) kustomize
 GORELEASER ?= $(GO_TOOL) goreleaser
 ENVTEST ?= go tool setup-envtest # this tool is in the main go.mod so the version stays in-sync
@@ -219,8 +219,8 @@ golangci-lint:
 IMAGE_REF_FILE := $(shell echo "$${TMPDIR:-/tmp}")/capox-image-ref
 
 .PHONY: build
-build: generate ## Builds a container image using ko and pushes to $KO_DOCKER_REPO
-	$(KO) build ./cmd | tee $(IMAGE_REF_FILE)
+build: generate ## Builds a container image using ko and pushes it to $IMAGE_REPO:$IMAGE_TAG
+	$(KO) build --bare --tags $(IMAGE_TAG) ./cmd | tee $(IMAGE_REF_FILE)
 
 .PHONY: build-kind
 build-kind:
@@ -290,8 +290,8 @@ deploy: generate ## Deploy controller to the K8s cluster specified in ~/.kube/co
 
 # Internal: deploy whatever image ref a build target captured in
 # $(IMAGE_REF_FILE). The chart renders "repository:tag", so splitting the ref
-# on its last colon recomposes to the exact ref ko printed — a digest ref
-# (repo@sha256 + hex) for registry builds, a digest-hex tag for --local
+# on its last colon recomposes to the exact ref ko printed — a tag+digest ref
+# (repo:tag@sha256 + hex) for registry builds, a digest-hex tag for kind
 # builds. Either way the ref is unique per build, so every deploy rolls the
 # pods.
 .PHONY: helm-apply

api/v1alpha1/oxidecluster_types.go

@@ -57,6 +57,19 @@ type OxideClusterSpec struct {
 	// +optional
 	// +kubebuilder:validation:Enum=v4;v6
 	IPType string `json:"ipType,omitempty"`
+
+	// ControlPlaneAntiAffinityPolicy, when set, makes the controller manage an anti-affinity
+	// group for the cluster's control plane machines so that their instances are spread across
+	// sleds. The group is created in the cluster's project before the cluster is marked
+	// provisioned, joined by every control plane instance at creation, and deleted with the
+	// cluster. "allow" permits co-location when the spread cannot be satisfied; "fail" refuses
+	// to place the instance instead. Note that the policy is applied when the group is first
+	// created: the Oxide API doesn't support updating a group's policy, so changing this field
+	// doesn't affect an existing group, and instances only join the group as they are created.
+	// https://docs.oxide.computer/guides/deploying-workloads#_affinity_and_anti_affinity
+	// +optional
+	// +kubebuilder:validation:Enum=allow;fail
+	ControlPlaneAntiAffinityPolicy string `json:"controlPlaneAntiAffinityPolicy,omitempty"`
 }
 
 // SecretReference is a reference to a Secret containing Oxide credentials. Adapted from

api/v1alpha1/oxidemachine_types.go

@@ -43,8 +43,10 @@ type OxideMachineSpec struct {
 	ImageID string `json:"imageID"`
 
 	// AntiAffinityGroups is the list of anti-affinity group names or IDs that the Oxide instance
-	// should use. Anti-affinity groups should be created outside the operator, else instance
-	// creation will fail.
+	// should use. Anti-affinity groups specified should already be created, else instance
+	// creation will fail. A control plane anti-affinity group managed via the OxideCluster's
+	// ControlPlaneAntiAffinityPolicy field is joined automatically and doesn't need to be listed
+	// here.
 	// https://docs.oxide.computer/guides/deploying-workloads#_affinity_and_anti_affinity
 	// +optional
 	// +listType=set

charts/cluster-api-provider-oxide/crds/infrastructure.cluster.x-k8s.io_oxideclusters.yaml

@@ -52,6 +52,21 @@ spec:
           spec:
             description: spec defines the desired state of OxideCluster
             properties:
+              controlPlaneAntiAffinityPolicy:
+                description: |-
+                  ControlPlaneAntiAffinityPolicy, when set, makes the controller manage an anti-affinity
+                  group for the cluster's control plane machines so that their instances are spread across
+                  sleds. The group is created in the cluster's project before the cluster is marked
+                  provisioned, joined by every control plane instance at creation, and deleted with the
+                  cluster. "allow" permits co-location when the spread cannot be satisfied; "fail" refuses
+                  to place the instance instead. Note that the policy is applied when the group is first
+                  created: the Oxide API doesn't support updating a group's policy, so changing this field
+                  doesn't affect an existing group, and instances only join the group as they are created.
+                  https://docs.oxide.computer/guides/deploying-workloads#_affinity_and_anti_affinity
+                enum:
+                - allow
+                - fail
+                type: string
               controlPlaneEndpoint:
                 description: |-
                   ControlPlaneEndpoint represents the host and port of the cluster's control plane. If

charts/cluster-api-provider-oxide/crds/infrastructure.cluster.x-k8s.io_oxidemachines.yaml

@@ -61,8 +61,10 @@ spec:
               antiAffinityGroups:
                 description: |-
                   AntiAffinityGroups is the list of anti-affinity group names or IDs that the Oxide instance
-                  should use. Anti-affinity groups should be created outside the operator, else instance
-                  creation will fail.
+                  should use. Anti-affinity groups specified should already be created, else instance
+                  creation will fail. A control plane anti-affinity group managed via the OxideCluster's
+                  ControlPlaneAntiAffinityPolicy field is joined automatically and doesn't need to be listed
+                  here.
                   https://docs.oxide.computer/guides/deploying-workloads#_affinity_and_anti_affinity
                 items:
                   type: string

charts/cluster-api-provider-oxide/crds/infrastructure.cluster.x-k8s.io_oxidemachinetemplates.yaml

@@ -79,8 +79,10 @@ spec:
                       antiAffinityGroups:
                         description: |-
                           AntiAffinityGroups is the list of anti-affinity group names or IDs that the Oxide instance
-                          should use. Anti-affinity groups should be created outside the operator, else instance
-                          creation will fail.
+                          should use. Anti-affinity groups specified should already be created, else instance
+                          creation will fail. A control plane anti-affinity group managed via the OxideCluster's
+                          ControlPlaneAntiAffinityPolicy field is joined automatically and doesn't need to be listed
+                          here.
                           https://docs.oxide.computer/guides/deploying-workloads#_affinity_and_anti_affinity
                         items:
                           type: string

go.mod

@@ -1,6 +1,6 @@
 module github.com/oxidecomputer/cluster-api-provider-oxide
 
-go 1.26.4
+go 1.26.5
 
 require (
 	github.com/google/uuid v1.6.0

internal/cloud/oxide.go

@@ -40,6 +40,16 @@ type OxideClient interface {
 
 	DiskView(context.Context, oxide.DiskViewParams) (*oxide.Disk, error)
 	DiskDelete(context.Context, oxide.DiskDeleteParams) error
+
+	AntiAffinityGroupCreate(
+		context.Context,
+		oxide.AntiAffinityGroupCreateParams,
+	) (*oxide.AntiAffinityGroup, error)
+	AntiAffinityGroupView(
+		context.Context,
+		oxide.AntiAffinityGroupViewParams,
+	) (*oxide.AntiAffinityGroup, error)
+	AntiAffinityGroupDelete(context.Context, oxide.AntiAffinityGroupDeleteParams) error
 }
 
 const (

internal/controller/naming.go

@@ -11,11 +11,12 @@ import (
 const maxResourceLength = 63
 
 const (
-	instancePrefix   = "capi"
-	nicPrefix        = "capi"
-	bootDiskPrefix   = "capi-boot"
-	floatingIPPrefix = "capi-fip"
-	dataDiskPrefix   = "capi-data-%d"
+	instancePrefix          = "capi"
+	nicPrefix               = "capi"
+	bootDiskPrefix          = "capi-boot"
+	floatingIPPrefix        = "capi-fip"
+	dataDiskPrefix          = "capi-data-%d"
+	antiAffinityGroupPrefix = "capi-aag"
 )
 
 func hashTruncateName(name string, maxLength int) string {
@@ -56,3 +57,7 @@ func getFloatingIPName(obj metav1.Object) string {
 func getDataDiskName(obj metav1.Object, idx int) string {
 	return getResourceName(fmt.Sprintf(dataDiskPrefix, idx), obj)
 }
+
+func getAntiAffinityGroupName(obj metav1.Object) string {
+	return getResourceName(antiAffinityGroupPrefix, obj)
+}

internal/controller/oxidecluster_controller.go

@@ -113,11 +113,7 @@ func (r *OxideClusterReconciler) Reconcile(
 	ipName := getFloatingIPName(oxideCluster)
 
 	if !oxideCluster.DeletionTimestamp.IsZero() {
-		if err := r.ensureFloatingIPDeleted(ctx, oxideClient, projectName, ipName); err != nil {
-			return ctrl.Result{}, fmt.Errorf("deleting floating ip: %w", err)
-		}
-		controllerutil.RemoveFinalizer(oxideCluster, infrav1.ClusterFinalizer)
-		return ctrl.Result{}, retErr
+		return ctrl.Result{}, r.reconcileDelete(ctx, oxideClient, oxideCluster, projectName, ipName)
 	} else if !cluster.DeletionTimestamp.IsZero() {
 		log.Info(
 			"cluster is being deleted, aborting OxideCluster reconcile",
@@ -140,10 +136,23 @@ func (r *OxideClusterReconciler) Reconcile(
 		oxideCluster.Spec.ControlPlaneEndpoint.Port = 6443
 	}
 
+	// Reconcile the control plane anti-affinity group, if enabled. This must happen before the
+	// cluster is marked provisioned below: upstream CAPI controllers won't create Machines until
+	// then, and instance creation fails if the machine references a group that doesn't exist.
+	if err := r.reconcileAntiAffinityGroup(
+		ctx,
+		oxideClient,
+		oxideCluster,
+		projectName,
+	); err != nil {
+		return ctrl.Result{}, fmt.Errorf("ensuring anti-affinity group: %w", err)
+	}
+
 	// We consider the OxideCluster to be Ready when the Oxide infrastructure that it manages (i.e.
-	// the floating IP) is provisioned. Note that upstream CAPI controllers won't provision
-	// Machine/OxideMachine resources until the OxideCluster is Ready, so we must mark Ready here
-	// rather than waiting for OxideMachines to be provisioned and attached.
+	// the floating IP and control plane anti-affinity group) is provisioned. Note that upstream
+	// CAPI controllers won't provision Machine/OxideMachine resources until the OxideCluster is
+	// Ready, so we must mark Ready here rather than waiting for OxideMachines to be provisioned
+	// and attached.
 	oxideCluster.Status.Initialization.Provisioned = new(true)
 	conditions.Set(oxideCluster, metav1.Condition{
 		Type:   clusterv1.ReadyCondition,
@@ -345,6 +354,115 @@ func (r *OxideClusterReconciler) ensureFloatingIPExists(
 	return ip, nil
 }
 
+// reconcileDelete deletes the Oxide resources owned by the OxideCluster and removes the
+// finalizer.
+func (r *OxideClusterReconciler) reconcileDelete(
+	ctx context.Context,
+	oxideClient cloud.OxideClient,
+	oxideCluster *infrav1.OxideCluster,
+	projectName string,
+	ipName string,
+) error {
+	if err := r.ensureFloatingIPDeleted(ctx, oxideClient, projectName, ipName); err != nil {
+		return fmt.Errorf("deleting floating ip: %w", err)
+	}
+	// Delete the control plane anti-affinity group unconditionally rather than only when the
+	// policy is currently set, so a group created before the policy was unset is cleaned up too.
+	// CAPI deletes the cluster's Machines before its infrastructure, so the group has no members
+	// by the time this runs.
+	if err := r.ensureAntiAffinityGroupDeleted(
+		ctx,
+		oxideClient,
+		projectName,
+		getAntiAffinityGroupName(oxideCluster),
+	); err != nil {
+		return fmt.Errorf("deleting anti-affinity group: %w", err)
+	}
+	controllerutil.RemoveFinalizer(oxideCluster, infrav1.ClusterFinalizer)
+	return nil
+}
+
+// reconcileAntiAffinityGroup ensures the control plane anti-affinity group exists when the
+// cluster opts in via ControlPlaneAntiAffinityPolicy, and is a no-op otherwise.
+func (r *OxideClusterReconciler) reconcileAntiAffinityGroup(
+	ctx context.Context,
+	oxideClient cloud.OxideClient,
+	oxideCluster *infrav1.OxideCluster,
+	projectName string,
+) error {
+	if oxideCluster.Spec.ControlPlaneAntiAffinityPolicy == "" {
+		return nil
+	}
+	_, err := r.ensureAntiAffinityGroupExists(
+		ctx,
+		oxideClient,
+		oxideCluster,
+		projectName,
+		getAntiAffinityGroupName(oxideCluster),
+	)
+	return err
+}
+
+// ensureAntiAffinityGroupExists creates or views the control plane anti-affinity group. An
+// existing group is adopted as is: the Oxide API doesn't support updating a group's policy, so
+// drift between the group and ControlPlaneAntiAffinityPolicy isn't reconciled.
+func (r *OxideClusterReconciler) ensureAntiAffinityGroupExists(
+	ctx context.Context,
+	oxideClient cloud.OxideClient,
+	oxideCluster *infrav1.OxideCluster,
+	projectName string,
+	groupName string,
+) (*oxide.AntiAffinityGroup, error) {
+	group, err := oxideClient.AntiAffinityGroupView(ctx, oxide.AntiAffinityGroupViewParams{
+		Project:           oxide.NameOrId(projectName),
+		AntiAffinityGroup: oxide.NameOrId(groupName),
+	})
+	if err != nil {
+		if !errors.Is(err, oxide.ErrObjectNotFound) {
+			return nil, fmt.Errorf("fetching existing anti-affinity group: %w", err)
+		}
+
+		group, err = oxideClient.AntiAffinityGroupCreate(ctx, oxide.AntiAffinityGroupCreateParams{
+			Project: oxide.NameOrId(projectName),
+			Body: &oxide.AntiAffinityGroupCreate{
+				Name: oxide.Name(groupName),
+				Description: fmt.Sprintf(
+					"Control plane anti-affinity for CAPI cluster %s/%s",
+					oxideCluster.Namespace,
+					oxideCluster.Name,
+				),
+				FailureDomain: oxide.FailureDomainSled,
+				Policy: oxide.AffinityPolicy(
+					oxideCluster.Spec.ControlPlaneAntiAffinityPolicy,
+				),
+			},
+		})
+		if err != nil {
+			return nil, fmt.Errorf("creating anti-affinity group: %w", err)
+		}
+	}
+
+	return group, nil
+}
+
+// ensureAntiAffinityGroupDeleted deletes the control plane anti-affinity group if it exists.
+func (r *OxideClusterReconciler) ensureAntiAffinityGroupDeleted(
+	ctx context.Context,
+	oxideClient cloud.OxideClient,
+	projectName string,
+	groupName string,
+) error {
+	if err := oxideClient.AntiAffinityGroupDelete(ctx, oxide.AntiAffinityGroupDeleteParams{
+		Project:           oxide.NameOrId(projectName),
+		AntiAffinityGroup: oxide.NameOrId(groupName),
+	}); err != nil {
+		if !errors.Is(err, oxide.ErrObjectNotFound) {
+			return err
+		}
+	}
+	return nil
+}
+
 // ensureFloatingIPDeleted deletes the floating IP if it exists.
 func (r *OxideClusterReconciler) ensureFloatingIPDeleted(
 	ctx context.Context,