feat: cleanup done exec / debug CR after graceperiod

cmd/main.go

@@ -151,10 +151,11 @@ func main() {
 		os.Exit(1)
 	}
 	if err = (&controller.StoreExecReconciler{
-		Client:   nsClient,
-		Logger:   logger,
-		Scheme:   mgr.GetScheme(),
-		Recorder: mgr.GetEventRecorderFor(fmt.Sprintf("shopware-controller-%s", cfg.Namespace)),
+		Client:             nsClient,
+		Logger:             logger,
+		Scheme:             mgr.GetScheme(),
+		Recorder:           mgr.GetEventRecorderFor(fmt.Sprintf("shopware-controller-%s", cfg.Namespace)),
+		CleanupGracePeriod: cfg.SuccessfulCRCleanupGracePeriod,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create exec controller", "controller", "StoreExec")
 		os.Exit(1)
@@ -184,10 +185,11 @@ func main() {
 		os.Exit(1)
 	}
 	if err = (&controller.StoreDebugInstanceReconciler{
-		Client:   nsClient,
-		Logger:   logger,
-		Scheme:   mgr.GetScheme(),
-		Recorder: mgr.GetEventRecorderFor(fmt.Sprintf("shopware-controller-%s", cfg.Namespace)),
+		Client:             nsClient,
+		Logger:             logger,
+		Scheme:             mgr.GetScheme(),
+		Recorder:           mgr.GetEventRecorderFor(fmt.Sprintf("shopware-controller-%s", cfg.Namespace)),
+		CleanupGracePeriod: cfg.SuccessfulCRCleanupGracePeriod,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create instance controller", "controller", "StoreDebugInstance")
 		os.Exit(1)

helm/templates/deployment.yaml

@@ -79,6 +79,8 @@ spec:
           value: "{{ .Values.logFormat | default "json" }}"
         - name: DISABLE_CHECKS
           value: "{{ .Values.disableChecks | default "false" }}"
+        - name: SUCCESSFUL_CR_CLEANUP_GRACE_PERIOD
+          value: "{{ .Values.successfulCRCleanupGracePeriod | default "1h" }}"
         {{- if and (hasKey .Values "events") (hasKey .Values.events "nats") (.Values.events.nats.enable) }}
         - name: NATS_ENABLE
           value: "true"

helm/values.yaml

@@ -94,3 +94,5 @@ logFormat: json
 # Disable check for s3/database/fastly and Opensearch checks. Useful if network access is not given for one of the services.
 # This is a global level. You can also control this per store.
 disableChecks: false
+# Grace period before successful StoreExec and StoreDebugInstance CRs are deleted. Set to "0" to disable cleanup.
+successfulCRCleanupGracePeriod: 1h

internal/config/config.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/sethvargo/go-envconfig"
 )
@@ -65,6 +66,8 @@ type StoreConfig struct {
 	EnableLeaderElection bool   `env:"LEADER_ELECT, default=true"`
 	DisableChecks        bool   `env:"DISABLE_CHECKS, default=false"`
 	Namespace            string `env:"NAMESPACE, default=default"`
+
+	SuccessfulCRCleanupGracePeriod time.Duration `env:"SUCCESSFUL_CR_CLEANUP_GRACE_PERIOD, default=1h"`
 }
 
 type Config struct {

internal/controller/storedebuginstance_controller.go

@@ -40,9 +40,10 @@ import (
 // StoreDebugInstanceReconciler reconciles a StoreDebugInstance object
 type StoreDebugInstanceReconciler struct {
 	client.Client
-	Scheme   *runtime.Scheme
-	Recorder record.EventRecorder
-	Logger   *zap.SugaredLogger
+	Scheme             *runtime.Scheme
+	Recorder           record.EventRecorder
+	Logger             *zap.SugaredLogger
+	CleanupGracePeriod time.Duration
 }
 
 // +kubebuilder:rbac:groups=shop.shopware.com,namespace=default,resources=storedebuginstances,verbs=get;list;watch;create;update;patch;delete
@@ -59,8 +60,12 @@ func (r *StoreDebugInstanceReconciler) Reconcile(ctx context.Context, req ctrl.R
 
 	var store *shopv1.Store
 	var storeDebugInstance *shopv1.StoreDebugInstance
+	var skipStatusUpdate bool
 	rr = ctrl.Result{RequeueAfter: 10 * time.Second}
 	defer func() {
+		if skipStatusUpdate {
+			return
+		}
 		if err := r.reconcileCRStatus(ctx, store, storeDebugInstance, err); err != nil {
 			log.Errorw("failed to update status", zap.Error(err))
 		}
@@ -80,6 +85,16 @@ func (r *StoreDebugInstanceReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return rr, fmt.Errorf("invalid duration: %w", err)
 	}
 
+	if result, deleted, cleanupErr := r.deleteSuccessfulStoreDebugInstanceIfCleanupDue(ctx, storeDebugInstance); deleted || cleanupErr != nil {
+		if cleanupErr != nil {
+			log.Errorw("failed to cleanup successful store debug instance", zap.Error(cleanupErr))
+			skipStatusUpdate = true
+			return rr, nil
+		}
+		skipStatusUpdate = true
+		return result, nil
+	}
+
 	store, err = k8s.GetStore(ctx, r.Client, types.NamespacedName{
 		Namespace: req.Namespace,
 		Name:      storeDebugInstance.Spec.StoreRef,
@@ -106,6 +121,15 @@ func (r *StoreDebugInstanceReconciler) Reconcile(ctx context.Context, req ctrl.R
 			return rr, nil
 		}
 
+		if result, handled, cleanupErr := r.reconcileSuccessfulStoreDebugInstanceCleanup(ctx, storeDebugInstance); handled || cleanupErr != nil {
+			if cleanupErr != nil {
+				log.Errorw("failed to cleanup successful store debug instance", zap.Error(cleanupErr))
+				return rr, nil
+			}
+			skipStatusUpdate = true
+			return result, nil
+		}
+
 		rr.Requeue = false
 		return rr, nil
 	}
@@ -181,6 +205,63 @@ func (r *StoreDebugInstanceReconciler) reconcilePod(ctx context.Context, store *
 	return nil
 }
 
+func (r *StoreDebugInstanceReconciler) deleteSuccessfulStoreDebugInstanceIfCleanupDue(
+	ctx context.Context,
+	storeDebugInstance *shopv1.StoreDebugInstance,
+) (ctrl.Result, bool, error) {
+	if !r.isStoreDebugInstanceCleanupEligible(storeDebugInstance) {
+		return ctrl.Result{}, false, nil
+	}
+
+	if remaining := r.storeDebugInstanceCleanupRemaining(storeDebugInstance); remaining > 0 {
+		return ctrl.Result{}, false, nil
+	}
+
+	return r.deleteSuccessfulStoreDebugInstance(ctx, storeDebugInstance)
+}
+
+func (r *StoreDebugInstanceReconciler) reconcileSuccessfulStoreDebugInstanceCleanup(
+	ctx context.Context,
+	storeDebugInstance *shopv1.StoreDebugInstance,
+) (ctrl.Result, bool, error) {
+	if !r.isStoreDebugInstanceCleanupEligible(storeDebugInstance) {
+		return ctrl.Result{}, false, nil
+	}
+
+	if remaining := r.storeDebugInstanceCleanupRemaining(storeDebugInstance); remaining > 0 {
+		return ctrl.Result{RequeueAfter: remaining}, true, nil
+	}
+
+	return r.deleteSuccessfulStoreDebugInstance(ctx, storeDebugInstance)
+}
+
+func (r *StoreDebugInstanceReconciler) isStoreDebugInstanceCleanupEligible(
+	storeDebugInstance *shopv1.StoreDebugInstance,
+) bool {
+	return r.CleanupGracePeriod > 0 &&
+		storeDebugInstance.DeletionTimestamp == nil &&
+		storeDebugInstance.IsState(shopv1.StoreDebugInstanceStateDone)
+}
+
+func (r *StoreDebugInstanceReconciler) storeDebugInstanceCleanupRemaining(
+	storeDebugInstance *shopv1.StoreDebugInstance,
+) time.Duration {
+	duration, _ := time.ParseDuration(storeDebugInstance.Spec.Duration)
+	deleteAfter := storeDebugInstance.CreationTimestamp.Add(duration).Add(r.CleanupGracePeriod)
+	return time.Until(deleteAfter)
+}
+
+func (r *StoreDebugInstanceReconciler) deleteSuccessfulStoreDebugInstance(
+	ctx context.Context,
+	storeDebugInstance *shopv1.StoreDebugInstance,
+) (ctrl.Result, bool, error) {
+	if err := r.Delete(ctx, storeDebugInstance); err != nil && !k8serrors.IsNotFound(err) {
+		return ctrl.Result{}, false, fmt.Errorf("delete successful StoreDebugInstance: %w", err)
+	}
+
+	return ctrl.Result{Requeue: false}, true, nil
+}
+
 func (r *StoreDebugInstanceReconciler) reconcileService(ctx context.Context, store *shopv1.Store, storeDebugInstance *shopv1.StoreDebugInstance) error {
 	svc := service.DebugService(*store, *storeDebugInstance)
 

internal/controller/storeexec_controller.go

@@ -20,9 +20,10 @@ import (
 
 type StoreExecReconciler struct {
 	client.Client
-	Scheme   *runtime.Scheme
-	Recorder record.EventRecorder
-	Logger   *zap.SugaredLogger
+	Scheme             *runtime.Scheme
+	Recorder           record.EventRecorder
+	Logger             *zap.SugaredLogger
+	CleanupGracePeriod time.Duration
 }
 
 // +kubebuilder:rbac:groups=shop.shopware.com,namespace=default,resources=storeexecs,verbs=get;list;watch;create;update;patch;delete
@@ -42,7 +43,11 @@ func (r *StoreExecReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 	var ex *v1.StoreExec
 	var store *v1.Store
+	var skipStatusUpdate bool
 	defer func() {
+		if skipStatusUpdate {
+			return
+		}
 		if err := r.reconcileCRStatus(ctx, store, ex, err); err != nil {
 			log.Errorw("failed to update status", zap.Error(err))
 		}
@@ -57,6 +62,16 @@ func (r *StoreExecReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return rr, nil
 	}
 
+	if result, handled, cleanupErr := r.reconcileSuccessfulStoreExecCleanup(ctx, ex); handled || cleanupErr != nil {
+		if cleanupErr != nil {
+			log.Errorw("failed to cleanup successful store-exec", zap.Error(cleanupErr))
+			skipStatusUpdate = true
+			return rr, nil
+		}
+		skipStatusUpdate = true
+		return result, nil
+	}
+
 	store, err = k8s.GetStore(ctx, r.Client, types.NamespacedName{
 		Namespace: req.Namespace,
 		Name:      ex.Spec.StoreRef,
@@ -131,6 +146,49 @@ func (r *StoreExecReconciler) reconcileJob(ctx context.Context, store *v1.Store,
 	return nil
 }
 
+func (r *StoreExecReconciler) reconcileSuccessfulStoreExecCleanup(
+	ctx context.Context,
+	ex *v1.StoreExec,
+) (ctrl.Result, bool, error) {
+	if r.CleanupGracePeriod <= 0 ||
+		ex.DeletionTimestamp != nil ||
+		ex.Spec.CronSchedule != "" ||
+		!ex.IsState(v1.ExecStateDone) {
+		return ctrl.Result{}, false, nil
+	}
+
+	deleteAfter := storeExecFinishedAt(ex).Add(r.CleanupGracePeriod)
+	if remaining := time.Until(deleteAfter); remaining > 0 {
+		return ctrl.Result{RequeueAfter: remaining}, true, nil
+	}
+
+	if err := r.Delete(ctx, ex); err != nil && !k8serrors.IsNotFound(err) {
+		return ctrl.Result{}, false, fmt.Errorf("delete successful StoreExec: %w", err)
+	}
+
+	return ctrl.Result{Requeue: false}, true, nil
+}
+
+func storeExecFinishedAt(ex *v1.StoreExec) time.Time {
+	for i := len(ex.Status.Conditions) - 1; i >= 0; i-- {
+		if !ex.Status.Conditions[i].LastTransitionTime.IsZero() {
+			return ex.Status.Conditions[i].LastTransitionTime.Time
+		}
+	}
+
+	for i := len(ex.Status.Conditions) - 1; i >= 0; i-- {
+		if !ex.Status.Conditions[i].LastUpdateTime.IsZero() {
+			return ex.Status.Conditions[i].LastUpdateTime.Time
+		}
+	}
+
+	if !ex.CreationTimestamp.IsZero() {
+		return ex.CreationTimestamp.Time
+	}
+
+	return time.Now()
+}
+
 func (r *StoreExecReconciler) reconcileCronJob(ctx context.Context, store *v1.Store, exec *v1.StoreExec) (err error) {
 	var changed bool
 	obj := job.CommandCronJob(*store, *exec)