feat: add CRD cleanup Job for agents

Makefile

@@ -7,11 +7,13 @@ HUB_AGENT_IMAGE_VERSION ?= $(TAG)
 MEMBER_AGENT_IMAGE_VERSION ?= $(TAG)
 REFRESH_TOKEN_IMAGE_VERSION ?= $(TAG)
 CRD_INSTALLER_IMAGE_VERSION ?= $(TAG)
+CRD_CLEANUP_IMAGE_VERSION ?= $(TAG)
 
 HUB_AGENT_IMAGE_NAME ?= hub-agent
 MEMBER_AGENT_IMAGE_NAME ?= member-agent
 REFRESH_TOKEN_IMAGE_NAME ?= refresh-token
 CRD_INSTALLER_IMAGE_NAME ?= crd-installer
+CRD_CLEANUP_IMAGE_NAME ?= crd-cleanup
 ARC_MEMBER_AGENT_HELMCHART_NAME = arc-member-cluster-agents-helm-chart
 
 TARGET_OS ?= linux
@@ -256,6 +258,7 @@ build: generate fmt vet ## Build agent binaries
 	go build -o bin/hubagent cmd/hubagent/main.go
 	go build -o bin/memberagent cmd/memberagent/main.go
 	go build -o bin/crdinstaller cmd/crdinstaller/main.go
+	go build -o bin/crdcleanup cmd/crdcleanup/main.go
 
 .PHONY: run-hubagent
 run-hubagent: manifests generate fmt vet ## Run hub-agent from your host
@@ -280,7 +283,7 @@ BUILDKIT_VERSION ?= v0.18.1
 
 .PHONY: push
 push: ## Build and push all Docker images
-	$(MAKE) OUTPUT_TYPE="type=registry" docker-build-hub-agent docker-build-member-agent docker-build-refresh-token docker-build-crd-installer
+	$(MAKE) OUTPUT_TYPE="type=registry" docker-build-hub-agent docker-build-member-agent docker-build-refresh-token docker-build-crd-installer docker-build-crd-cleanup
 
 # By default, docker buildx create will pull image moby/buildkit:buildx-stable-1 and hit the too many requests error
 #
@@ -360,6 +363,18 @@ docker-build-crd-installer: docker-buildx-builder
 		--build-arg GOARCH=$(TARGET_ARCH) \
 		--build-arg GOOS=${TARGET_OS} .
 
+.PHONY: docker-build-crd-cleanup
+docker-build-crd-cleanup: docker-buildx-builder ## Build crd-cleanup image
+	docker buildx build \
+		--file docker/crd-cleanup.Dockerfile \
+		--output=$(OUTPUT_TYPE) \
+		--platform=$(TARGET_OS)/$(TARGET_ARCH) \
+		--pull \
+		--tag $(REGISTRY)/$(CRD_CLEANUP_IMAGE_NAME):$(CRD_CLEANUP_IMAGE_VERSION) \
+		--progress=$(BUILDKIT_PROGRESS_TYPE) \
+		--build-arg GOARCH=$(TARGET_ARCH) \
+		--build-arg GOOS=${TARGET_OS} .
+
 # Fleet Agents and Networking Agents are packaged and pushed to MCR for Arc Extension.
 .PHONY: helm-package-arc-member-cluster-agents
 helm-package-arc-member-cluster-agents:

charts/hub-agent/templates/crd-cleanup-job.yaml

@@ -0,0 +1,39 @@
+{{- if and .Values.crdCleanup.enabled .Values.enableV1Beta1APIs }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "hub-agent.fullname" . }}-crd-cleanup
+  namespace: {{ .Values.namespace }}
+  labels:
+    {{- include "hub-agent.labels" . | nindent 4 }}
+  annotations:
+    # Run this job before helm deletes other resources
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded,hook-failed
+spec:
+  ttlSecondsAfterFinished: 60
+  activeDeadlineSeconds: 300
+  backoffLimit: 2
+  template:
+    metadata:
+      labels:
+        {{- include "hub-agent.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "hub-agent.fullname" . }}-sa
+      restartPolicy: Never
+      containers:
+        - name: crd-cleanup
+          image: "{{ .Values.crdCleanup.image.repository }}:{{ .Values.crdCleanup.image.tag }}"
+          imagePullPolicy: {{ .Values.crdCleanup.image.pullPolicy }}
+          args:
+            - --mode=hub
+            - --v={{ .Values.crdCleanup.logVerbosity }}
+          resources:
+            limits:
+              cpu: 50m
+              memory: 64Mi
+            requests:
+              cpu: 10m
+              memory: 32Mi
+{{- end }}

charts/hub-agent/values.yaml

@@ -19,6 +19,16 @@ crdInstaller:
     tag: main
   logVerbosity: 2
 
+# CRD cleanup job configuration.
+# This job cleans up CRDs when the helm chart is uninstalled.
+crdCleanup:
+  enabled: false
+  image:
+    repository: ghcr.io/azure/fleet/crd-cleanup
+    pullPolicy: Always
+    tag: main
+  logVerbosity: 2
+
 logVerbosity: 5
 
 enableWebhook: true

charts/member-agent/templates/crd-cleanup-job.yaml

@@ -0,0 +1,39 @@
+{{- if and .Values.crdCleanup.enabled .Values.enableV1Beta1APIs }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "member-agent.fullname" . }}-crd-cleanup
+  namespace: {{ .Values.namespace }}
+  labels:
+    {{- include "member-agent.labels" . | nindent 4 }}
+  annotations:
+    # Run this job before helm deletes other resources
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded,hook-failed
+spec:
+  ttlSecondsAfterFinished: 60
+  activeDeadlineSeconds: 300
+  backoffLimit: 2
+  template:
+    metadata:
+      labels:
+        {{- include "member-agent.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "member-agent.fullname" . }}-sa
+      restartPolicy: Never
+      containers:
+        - name: crd-cleanup
+          image: "{{ .Values.crdCleanup.image.repository }}:{{ .Values.crdCleanup.image.tag }}"
+          imagePullPolicy: {{ .Values.crdCleanup.image.pullPolicy }}
+          args:
+            - --mode=member
+            - --v={{ .Values.crdCleanup.logVerbosity }}
+          resources:
+            limits:
+              cpu: 50m
+              memory: 64Mi
+            requests:
+              cpu: 10m
+              memory: 32Mi
+{{- end }}

charts/member-agent/values.yaml

@@ -14,6 +14,16 @@ crdInstaller:
     tag: main
   logVerbosity: 2
 
+# CRD cleanup job configuration.
+# This job cleans up CRDs when the helm chart is uninstalled.
+crdCleanup:
+  enabled: false
+  image:
+    repository: ghcr.io/azure/fleet/crd-cleanup
+    pullPolicy: Always
+    tag: main
+  logVerbosity: 2
+
 logVerbosity: 5
 
 refreshtoken:

cmd/crdcleanup/main.go

@@ -0,0 +1,107 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+*/
+
+// Package main contains the CRD cleanup job for KubeFleet.
+// This job cleans up all CRDs that were installed by the CRD installer
+// when the Fleet agents are uninstalled via Helm pre-delete hook.
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"go.goms.io/fleet/cmd/crdinstaller/utils"
+)
+
+var mode = flag.String("mode", "", "Mode to run in: 'hub' or 'member' (required)")
+
+func main() {
+	klog.InitFlags(nil)
+	flag.Parse()
+
+	// Validate required flags.
+	if *mode != "hub" && *mode != "member" {
+		klog.Fatal("--mode flag must be either 'hub' or 'member'")
+	}
+
+	klog.Infof("Starting CRD cleanup job in %s mode", *mode)
+
+	// Print all flags for debugging.
+	flag.VisitAll(func(f *flag.Flag) {
+		klog.V(2).InfoS("flag:", "name", f.Name, "value", f.Value)
+	})
+
+	// Get Kubernetes config using controller-runtime.
+	config := ctrl.GetConfigOrDie()
+
+	// Create a scheme that knows about CRD types.
+	scheme := runtime.NewScheme()
+	if err := apiextensionsv1.AddToScheme(scheme); err != nil {
+		klog.Fatalf("Failed to add apiextensions scheme: %v", err)
+	}
+
+	k8sClient, err := client.New(config, client.Options{
+		Scheme: scheme,
+	})
+	if err != nil {
+		klog.Fatalf("Failed to create Kubernetes client: %v", err)
+	}
+
+	// Create context for cleanup operations.
+	ctx := context.Background()
+
+	// Perform cleanup.
+	if err := cleanupCRDs(ctx, k8sClient, *mode); err != nil {
+		klog.Errorf("Failed to cleanup CRDs: %v", err)
+		os.Exit(1)
+	}
+
+	klog.Info("CRD cleanup completed successfully")
+}
+
+// cleanupCRDs deletes all CRDs that were installed by the CRD installer for the given mode.
+// It uses the mode label to identify which CRDs to delete.
+func cleanupCRDs(ctx context.Context, k8sClient client.Client, mode string) error {
+	// List all CRDs with both the managed label and the matching mode label.
+	crdList := &apiextensionsv1.CustomResourceDefinitionList{}
+	labelSelector := labels.SelectorFromSet(labels.Set{
+		utils.CRDInstallerLabelKey:  "true",
+		utils.CRDInstallerModeLabel: mode,
+	})
+
+	if err := k8sClient.List(ctx, crdList, &client.ListOptions{
+		LabelSelector: labelSelector,
+	}); err != nil {
+		return err
+	}
+
+	klog.Infof("Found %d CRDs to cleanup for mode %s", len(crdList.Items), mode)
+
+	// Delete all matching CRDs.
+	var deletedCount int
+	for i := range crdList.Items {
+		crd := &crdList.Items[i]
+
+		klog.Infof("Deleting CRD: %s", crd.Name)
+		if err := k8sClient.Delete(ctx, crd); err != nil {
+			klog.Errorf("Failed to delete CRD %s: %v", crd.Name, err)
+			// Continue with other CRDs even if one fails.
+			continue
+		}
+		deletedCount++
+		klog.Infof("Successfully deleted CRD: %s", crd.Name)
+	}
+
+	klog.Infof("Cleanup complete: deleted %d CRDs", deletedCount)
+	return nil
+}