From c20a974eaf9b5154465472bce0de97e28fec034e Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 28 Apr 2026 13:25:12 +0100 Subject: [PATCH 01/15] updated snapshotRep api spec --- api/v1alpha1/snapshotreplication_types.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/snapshotreplication_types.go b/api/v1alpha1/snapshotreplication_types.go index 45eaec3d..9a76bc96 100644 --- a/api/v1alpha1/snapshotreplication_types.go +++ b/api/v1alpha1/snapshotreplication_types.go @@ -95,15 +95,15 @@ type SnapshotReplicationSpec struct { // +kubebuilder:validation:Enum=failback Action string `json:"action,omitempty"` - // Optional: only these volumes are included in failback. + // Optional: only these PVCs are included in failback. // If empty, all volumes are candidates unless excluded below. - IncludeVolumeIDs []string `json:"includeVolumeIDs,omitempty"` + IncludePVCRefs []PersistentVolumeClaimRef `json:"includePVCRefs,omitempty"` - // Optional: volumes to exclude from failback. - ExcludeVolumeIDs []string `json:"excludeVolumeIDs,omitempty"` + // Optional: PVCs to exclude from failback. + ExcludePVCRefs []PersistentVolumeClaimRef `json:"excludePVCRefs,omitempty"` - // Optional: list of volumes to replicate. Empty means all volumes - VolumeIDs []string `json:"volumeIDs,omitempty"` + // Optional: list of PVCs to replicate. Empty means all volumes. + PVCRefs []PersistentVolumeClaimRef `json:"pvcRefs,omitempty"` } // SnapshotReplicationStatus defines the observed state of SnapshotReplication. @@ -126,8 +126,11 @@ type SnapshotReplicationStatus struct { // VolumeReplicationStatus tracks the replication state of an individual volume type VolumeReplicationStatus struct { - // Volume ID - VolumeID string `json:"volumeID"` + // PVCRef identifies the PVC being replicated + PVCRef PersistentVolumeClaimRef `json:"pvcRef"` + + // VolumeID is the resolved backend volume UUID + VolumeID string `json:"volumeID,omitempty"` // Phase is the current replication phase for this volume. // +kubebuilder:validation:Enum=Pending;Running;TriggeringTargetReplication;WaitingForTargetReplication;ReplicatingToSource;WaitingForTargetDeletion;Completed;Failed;Paused From e75221a6c5da0f1051cb2e45d2ef53c6ae9d66e7 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 28 Apr 2026 13:32:39 +0100 Subject: [PATCH 02/15] replace VolumeIDs with PVCRefs in SnapshotReplication spec and controller --- api/v1alpha1/zz_generated.deepcopy.go | 19 +++-- ...e.simplyblock.io_snapshotreplications.yaml | 71 ++++++++++++---- ...mplyblocksnapshotreplication_controller.go | 82 ++++++++++++++++++- ...napshotreplication_controller_unit_test.go | 25 +++++- 4 files changed, 168 insertions(+), 29 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 387b68a2..b2636953 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -861,19 +861,19 @@ func (in *SnapshotReplicationSpec) DeepCopyInto(out *SnapshotReplicationSpec) { *out = new(int32) **out = **in } - if in.IncludeVolumeIDs != nil { - in, out := &in.IncludeVolumeIDs, &out.IncludeVolumeIDs - *out = make([]string, len(*in)) + if in.IncludePVCRefs != nil { + in, out := &in.IncludePVCRefs, &out.IncludePVCRefs + *out = make([]PersistentVolumeClaimRef, len(*in)) copy(*out, *in) } - if in.ExcludeVolumeIDs != nil { - in, out := &in.ExcludeVolumeIDs, &out.ExcludeVolumeIDs - *out = make([]string, len(*in)) + if in.ExcludePVCRefs != nil { + in, out := &in.ExcludePVCRefs, &out.ExcludePVCRefs + *out = make([]PersistentVolumeClaimRef, len(*in)) copy(*out, *in) } - if in.VolumeIDs != nil { - in, out := &in.VolumeIDs, &out.VolumeIDs - *out = make([]string, len(*in)) + if in.PVCRefs != nil { + in, out := &in.PVCRefs, &out.PVCRefs + *out = make([]PersistentVolumeClaimRef, len(*in)) copy(*out, *in) } } @@ -1623,6 +1623,7 @@ func (in *TaskStatus) DeepCopy() *TaskStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeReplicationStatus) DeepCopyInto(out *VolumeReplicationStatus) { *out = *in + out.PVCRef = in.PVCRef if in.LastReplicationTime != nil { in, out := &in.LastReplicationTime, &out.LastReplicationTime *out = (*in).DeepCopy() diff --git a/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml b/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml index de3283f8..b4488376 100644 --- a/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml +++ b/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml @@ -57,22 +57,58 @@ spec: enum: - failback type: string - excludeVolumeIDs: - description: 'Optional: volumes to exclude from failback.' + excludePVCRefs: + description: 'Optional: PVCs to exclude from failback.' items: - type: string + properties: + name: + description: Name is the PVC name. + type: string + namespace: + description: Namespace overrides the backup resource namespace + for the PVC lookup. + type: string + required: + - name + type: object type: array - includeVolumeIDs: + includePVCRefs: description: |- - Optional: only these volumes are included in failback. + Optional: only these PVCs are included in failback. If empty, all volumes are candidates unless excluded below. items: - type: string + properties: + name: + description: Name is the PVC name. + type: string + namespace: + description: Namespace overrides the backup resource namespace + for the PVC lookup. + type: string + required: + - name + type: object type: array interval: description: 'snapshot replication interval in seconds (default: 300sec)' format: int32 type: integer + pvcRefs: + description: 'Optional: list of PVCs to replicate. Empty means all + volumes.' + items: + properties: + name: + description: Name is the PVC name. + type: string + namespace: + description: Namespace overrides the backup resource namespace + for the PVC lookup. + type: string + required: + - name + type: object + type: array sourceCluster: description: Source cluster for the snapshots type: string @@ -89,12 +125,6 @@ spec: description: snapshot replication timeout format: int32 type: integer - volumeIDs: - description: 'Optional: list of volumes to replicate. Empty means - all volumes' - items: - type: string - type: array required: - sourceCluster - targetCluster @@ -215,15 +245,28 @@ spec: - Failed - Paused type: string + pvcRef: + description: PVCRef identifies the PVC being replicated + properties: + name: + description: Name is the PVC name. + type: string + namespace: + description: Namespace overrides the backup resource namespace + for the PVC lookup. + type: string + required: + - name + type: object replicatedCount: description: Number of snapshots successfully replicated format: int32 type: integer volumeID: - description: Volume ID + description: VolumeID is the resolved backend volume UUID type: string required: - - volumeID + - pvcRef type: object type: array type: object diff --git a/internal/controller/simplyblocksnapshotreplication_controller.go b/internal/controller/simplyblocksnapshotreplication_controller.go index b4727bb9..9914cd9a 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller.go +++ b/internal/controller/simplyblocksnapshotreplication_controller.go @@ -25,6 +25,7 @@ import ( "strings" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -46,6 +47,7 @@ type SnapshotReplicationReconciler struct { // +kubebuilder:rbac:groups=storage.simplyblock.io,resources=snapshotreplications,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=storage.simplyblock.io,resources=snapshotreplications/status,verbs=get;update;patch // +kubebuilder:rbac:groups=storage.simplyblock.io,resources=snapshotreplications/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims;persistentvolumes,verbs=get;list;watch func (r *SnapshotReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := logf.FromContext(ctx) @@ -183,8 +185,13 @@ func (r *SnapshotReplicationReconciler) reconcileFailback( return 0, fmt.Errorf("failed to list target lvols for failback: %w", err) } - includeIDs := snapRepCR.Spec.IncludeVolumeIDs - excludeIDs := snapRepCR.Spec.ExcludeVolumeIDs + var includeIDs, excludeIDs []string + if len(snapRepCR.Spec.IncludePVCRefs) > 0 { + includeIDs = r.resolvePVCRefsToVolumeIDs(ctx, snapRepCR.Namespace, snapRepCR.Spec.IncludePVCRefs) + } + if len(snapRepCR.Spec.ExcludePVCRefs) > 0 { + excludeIDs = r.resolvePVCRefsToVolumeIDs(ctx, snapRepCR.Namespace, snapRepCR.Spec.ExcludePVCRefs) + } orig := snapRepCR.DeepCopy() r.setConditionOnCopy(snapRepCR, simplyblockv1alpha1.ConditionTypeFailback, metav1.ConditionFalse, "InProgress", "Failback in progress") @@ -296,6 +303,11 @@ func (r *SnapshotReplicationReconciler) reconcileNormalReplication( return nil } + var pvcVolumeIDs []string + if len(snapRepCR.Spec.PVCRefs) > 0 { + pvcVolumeIDs = r.resolvePVCRefsToVolumeIDs(ctx, snapRepCR.Namespace, snapRepCR.Spec.PVCRefs) + } + interval := utils.IntPtrOrDefault(snapRepCR.Spec.Interval, 300) now := time.Now().UTC() @@ -319,7 +331,7 @@ func (r *SnapshotReplicationReconciler) reconcileNormalReplication( continue } - if len(snapRepCR.Spec.VolumeIDs) > 0 && !slices.Contains(snapRepCR.Spec.VolumeIDs, lvolSummary.UUID) { + if len(pvcVolumeIDs) > 0 && !slices.Contains(pvcVolumeIDs, lvolSummary.UUID) { continue } @@ -687,6 +699,70 @@ func (r *SnapshotReplicationReconciler) setVolumeLastReplicationTime( } } +/* -------------------- PVC resolution helpers -------------------- */ + +func (r *SnapshotReplicationReconciler) resolvePVCRefToLvolID( + ctx context.Context, + namespace string, + pvcRef simplyblockv1alpha1.PersistentVolumeClaimRef, +) (string, error) { + pvcNamespace := pvcRef.Namespace + if pvcNamespace == "" { + pvcNamespace = namespace + } + + pvc := &corev1.PersistentVolumeClaim{} + if err := r.Get(ctx, client.ObjectKey{Name: pvcRef.Name, Namespace: pvcNamespace}, pvc); err != nil { + return "", fmt.Errorf("get PVC %s/%s: %w", pvcNamespace, pvcRef.Name, err) + } + + if pvc.Spec.VolumeName == "" { + return "", fmt.Errorf("PVC %s/%s is not bound", pvcNamespace, pvc.Name) + } + + pv := &corev1.PersistentVolume{} + if err := r.Get(ctx, client.ObjectKey{Name: pvc.Spec.VolumeName}, pv); err != nil { + return "", fmt.Errorf("get PV %s: %w", pvc.Spec.VolumeName, err) + } + + if pv.Spec.CSI == nil { + return "", fmt.Errorf("PV %s is not a CSI volume", pv.Name) + } + + _, _, lvolID, err := parseSimplyblockVolumeHandle(pv.Spec.CSI.VolumeHandle) + if err != nil { + return "", err + } + + if annotation := pvc.Annotations[pvcLvolIDAnnotation]; annotation != "" { + lvolID = annotation + } + + if lvolID == "" { + return "", fmt.Errorf("PVC %s/%s has no Simplyblock lvol ID", pvcNamespace, pvc.Name) + } + + return lvolID, nil +} + +func (r *SnapshotReplicationReconciler) resolvePVCRefsToVolumeIDs( + ctx context.Context, + namespace string, + refs []simplyblockv1alpha1.PersistentVolumeClaimRef, +) []string { + log := logf.FromContext(ctx) + ids := make([]string, 0, len(refs)) + for _, ref := range refs { + id, err := r.resolvePVCRefToLvolID(ctx, namespace, ref) + if err != nil { + log.Error(err, "Failed to resolve PVCRef to volume ID", "pvc", ref.Name) + continue + } + ids = append(ids, id) + } + return ids +} + /* -------------------- Condition helpers -------------------- */ // setCondition patches the named condition on the CR directly (issues its own Status patch). diff --git a/internal/controller/simplyblocksnapshotreplication_controller_unit_test.go b/internal/controller/simplyblocksnapshotreplication_controller_unit_test.go index a4b49cd8..7a5f0847 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller_unit_test.go +++ b/internal/controller/simplyblocksnapshotreplication_controller_unit_test.go @@ -331,7 +331,7 @@ func TestSnapshotReplicationNormalReplication(t *testing.T) { } }) - t.Run("skips lvol not in VolumeIDs allowlist", func(t *testing.T) { + t.Run("skips lvol not in PVCRefs allowlist", func(t *testing.T) { const ( srcUUID = "src-uuid-filter" poolUUID = "pool-uuid-filter" @@ -350,6 +350,23 @@ func TestSnapshotReplicationNormalReplication(t *testing.T) { )}) t.Setenv("SIMPLYBLOCK_WEBAPI_BASE_URL", mock.URL()) + // PVC + PV that resolve to allowedUUID + allowedPV := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "pv-allowed"}, + Spec: corev1.PersistentVolumeSpec{ + ClaimRef: &corev1.ObjectReference{Name: "pvc-allowed", Namespace: "default"}, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + VolumeHandle: srcUUID + ":" + poolUUID + ":" + allowedUUID, + }, + }, + }, + } + allowedPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "pvc-allowed", Namespace: "default"}, + Spec: corev1.PersistentVolumeClaimSpec{VolumeName: "pv-allowed"}, + } + cr := &simplyblockv1alpha1.SnapshotReplication{ ObjectMeta: metav1.ObjectMeta{ Name: "snap-filter", @@ -360,13 +377,15 @@ func TestSnapshotReplicationNormalReplication(t *testing.T) { SourceCluster: "cluster-src", TargetCluster: "tgt-uuid", TargetPool: "pool-uuid", - VolumeIDs: []string{allowedUUID}, // only allowedUUID — lvolUUID should be skipped + PVCRefs: []simplyblockv1alpha1.PersistentVolumeClaimRef{ + {Name: "pvc-allowed", Namespace: "default"}, + }, }, Status: simplyblockv1alpha1.SnapshotReplicationStatus{Configured: true}, } srcSecret := snapRepClusterSecret("cluster-src", srcUUID, "src-s") srcCluster := snapRepClusterCR("cluster-src", srcUUID) - r := newSnapRepTestReconciler(t, cr, srcSecret, srcCluster) + r := newSnapRepTestReconciler(t, cr, srcSecret, srcCluster, allowedPVC, allowedPV) _, _ = r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(cr)}) From ae86531367971a9582929c2dc06e3dd774281815 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 5 May 2026 14:33:26 +0100 Subject: [PATCH 03/15] fix(storagenode): use per-cluster EndpointSlice name to prevent multi-cluster endpoint overwrite --- internal/utils/storage_node_ds.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/utils/storage_node_ds.go b/internal/utils/storage_node_ds.go index e7b5eaf4..532edcd6 100644 --- a/internal/utils/storage_node_ds.go +++ b/internal/utils/storage_node_ds.go @@ -344,7 +344,7 @@ func BuildStorageNodeEndpointSlice(sn *simplyblockv1alpha1.StorageNode, nodeIPs return &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ - Name: "simplyblock-storage-node-api-endpoints", + Name: "simplyblock-storage-node-api-endpoints-" + sn.Spec.ClusterName, Namespace: sn.Namespace, Labels: map[string]string{ "kubernetes.io/service-name": "simplyblock-storage-node-api", From 07d26d494bba3fb2f70f981f3d14d7a88d4618c2 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 5 May 2026 16:39:35 +0100 Subject: [PATCH 04/15] replace failbackLvol blocking waits with non-blocking per-volume phase state machine --- api/v1alpha1/snapshotreplication_types.go | 45 +- api/v1alpha1/zz_generated.deepcopy.go | 20 + ...e.simplyblock.io_snapshotreplications.yaml | 21 + config/rbac/role.yaml | 16 +- ...mplyblocksnapshotreplication_controller.go | 446 +++++++----------- 5 files changed, 270 insertions(+), 278 deletions(-) diff --git a/api/v1alpha1/snapshotreplication_types.go b/api/v1alpha1/snapshotreplication_types.go index 3eee7784..5670e590 100644 --- a/api/v1alpha1/snapshotreplication_types.go +++ b/api/v1alpha1/snapshotreplication_types.go @@ -61,6 +61,36 @@ const ( // VolPhasePaused means replication is explicitly paused. VolPhasePaused = "Paused" + + // Failback sub-phases — each maps to a single non-blocking reconcile step. + // Source info is resolved once and stored in FailbackVolumeState so that + // every subsequent step can be re-entered safely after a crash or requeue. + + // VolPhaseFailbackTriggerFirst means the first replication trigger on the + // target cluster is pending (source info already resolved and persisted). + VolPhaseFailbackTriggerFirst = "FailbackTriggerFirst" + // VolPhaseFailbackWaitFirst means the controller is polling for the first + // target replication task to complete. + VolPhaseFailbackWaitFirst = "FailbackWaitFirst" + // VolPhaseFailbackSuspend means the target lvol should be suspended and the + // second replication triggered in the same step. + VolPhaseFailbackSuspend = "FailbackSuspend" + // VolPhaseFailbackWaitSecond means the controller is polling for the second + // target replication task to complete. + VolPhaseFailbackWaitSecond = "FailbackWaitSecond" + // VolPhaseFailbackDeleteSource means the source lvol should be deleted. + VolPhaseFailbackDeleteSource = "FailbackDeleteSource" + // VolPhaseFailbackWaitSourceDeleted means the controller is polling for the + // source lvol to reach the deleted state. + VolPhaseFailbackWaitSourceDeleted = "FailbackWaitSourceDeleted" + // VolPhaseFailbackReplicateOnSource means replicate_lvol_on_source_cluster + // should be called, then the target lvol deleted. + VolPhaseFailbackReplicateOnSource = "FailbackReplicateOnSource" + // VolPhaseFailbackDeleteTarget means the target lvol should be deleted. + VolPhaseFailbackDeleteTarget = "FailbackDeleteTarget" + // VolPhaseFailbackWaitTargetDeleted means the controller is polling for the + // target lvol to reach the deleted state. + VolPhaseFailbackWaitTargetDeleted = "FailbackWaitTargetDeleted" ) // Condition type constants used in SnapshotReplicationStatus.Conditions. @@ -132,6 +162,15 @@ type SnapshotReplicationStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` } +// FailbackVolumeState holds per-volume identifiers resolved once at the start +// of a failback and persisted so every subsequent phase can be re-entered +// safely after a crash or requeue without re-querying the source cluster. +type FailbackVolumeState struct { + SourcePoolUUID string `json:"sourcePoolUUID,omitempty"` + SourceLvolUUID string `json:"sourceLvolUUID,omitempty"` + IsFreshCluster bool `json:"isFreshCluster,omitempty"` +} + // VolumeReplicationStatus tracks the replication state of an individual volume type VolumeReplicationStatus struct { // PVCRef identifies the PVC being replicated @@ -141,7 +180,7 @@ type VolumeReplicationStatus struct { VolumeID string `json:"volumeID,omitempty"` // Phase is the current replication phase for this volume. - // +kubebuilder:validation:Enum=Pending;Running;TriggeringTargetReplication;WaitingForTargetReplication;ReplicatingToSource;WaitingForTargetDeletion;Completed;Failed;Paused + // +kubebuilder:validation:Enum=Pending;Running;TriggeringTargetReplication;WaitingForTargetReplication;ReplicatingToSource;WaitingForTargetDeletion;Completed;Failed;Paused;FailbackTriggerFirst;FailbackWaitFirst;FailbackSuspend;FailbackWaitSecond;FailbackDeleteSource;FailbackWaitSourceDeleted;FailbackReplicateOnSource;FailbackDeleteTarget;FailbackWaitTargetDeleted Phase string `json:"phase,omitempty"` // Last snapshot ID replicated for this volume @@ -155,6 +194,10 @@ type VolumeReplicationStatus struct { // Optional: list of errors encountered for this volume Errors []ReplicationError `json:"errors,omitempty"` + + // FailbackState stores identifiers resolved during failback initiation and + // used by all subsequent failback phases. + FailbackState *FailbackVolumeState `json:"failbackState,omitempty"` } // ReplicationError stores timestamped error messages diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f1e22e13..53c290eb 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -509,6 +509,21 @@ func (in *DeviceThroughputStats) DeepCopy() *DeviceThroughputStats { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FailbackVolumeState) DeepCopyInto(out *FailbackVolumeState) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FailbackVolumeState. +func (in *FailbackVolumeState) DeepCopy() *FailbackVolumeState { + if in == nil { + return nil + } + out := new(FailbackVolumeState) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *JournalManagerSpec) DeepCopyInto(out *JournalManagerSpec) { *out = *in @@ -1963,6 +1978,11 @@ func (in *VolumeReplicationStatus) DeepCopyInto(out *VolumeReplicationStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.FailbackState != nil { + in, out := &in.FailbackState, &out.FailbackState + *out = new(FailbackVolumeState) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeReplicationStatus. diff --git a/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml b/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml index b4488376..a4efc0f6 100644 --- a/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml +++ b/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml @@ -223,6 +223,18 @@ spec: - timestamp type: object type: array + failbackState: + description: |- + FailbackState stores identifiers resolved during failback initiation and + used by all subsequent failback phases. + properties: + isFreshCluster: + type: boolean + sourceLvolUUID: + type: string + sourcePoolUUID: + type: string + type: object lastReplicationTime: description: Timestamp of the last successful replication for this volume @@ -244,6 +256,15 @@ spec: - Completed - Failed - Paused + - FailbackTriggerFirst + - FailbackWaitFirst + - FailbackSuspend + - FailbackWaitSecond + - FailbackDeleteSource + - FailbackWaitSourceDeleted + - FailbackReplicateOnSource + - FailbackDeleteTarget + - FailbackWaitTargetDeleted type: string pvcRef: description: PVCRef identifies the PVC being replicated diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7792bd87..188f0d92 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -15,7 +15,6 @@ rules: - "" resources: - nodes - - pods verbs: - get - list @@ -35,11 +34,8 @@ rules: - apiGroups: - "" resources: - - secrets - - serviceaccounts - - services + - pods verbs: - - create - delete - get - list @@ -47,9 +43,11 @@ rules: - update - watch - apiGroups: - - apps + - "" resources: - - daemonsets + - secrets + - serviceaccounts + - services verbs: - create - delete @@ -63,8 +61,12 @@ rules: resources: - daemonsets verbs: + - create + - delete - get - list + - patch + - update - watch - apiGroups: - discovery.k8s.io diff --git a/internal/controller/simplyblocksnapshotreplication_controller.go b/internal/controller/simplyblocksnapshotreplication_controller.go index bea39f8e..428d32be 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller.go +++ b/internal/controller/simplyblocksnapshotreplication_controller.go @@ -142,9 +142,9 @@ func (r *SnapshotReplicationReconciler) SetupWithManager(mgr ctrl.Manager) error /* -------------------- Phase-driven failback reconciliation -------------------- */ -// reconcileFailback drives each volume through its failback phases. -// Returns a non-zero requeue duration if any volume is still in-progress, -// or 0 if all volumes are in a terminal phase (Completed or Failed). +// reconcileFailback advances each target volume through its failback phases. +// Each call does at most one non-blocking step per volume and returns a requeue +// duration if any volume is still in progress, or 0 when all are terminal. func (r *SnapshotReplicationReconciler) reconcileFailback( ctx context.Context, apiClient *webapi.Client, @@ -224,17 +224,20 @@ func (r *SnapshotReplicationReconciler) reconcileFailback( continue } - if advErr := r.advanceFailbackVolume( + inProgress, advErr := r.advanceFailbackVolume( ctx, apiClient, snapRepCR, sourceClusterUUID, sourceClusterSecret, targetClusterUUID, targetClusterSecret, targetPoolUUID, lvolDetail, currentPhase, - ); advErr != nil { + ) + if advErr != nil { log.Error(advErr, "Failed to advance failback volume phase", "lvolUUID", lvolDetail.UUID, "phase", currentPhase) r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailed, advErr.Error()) anyInProgress = true + } else if inProgress { + anyInProgress = true } } @@ -249,8 +252,15 @@ func (r *SnapshotReplicationReconciler) reconcileFailback( return 0, nil } -// advanceFailbackVolume runs the full failback sequence for a single volume. -// failbackLvol is a blocking call that handles all steps internally. +// advanceFailbackVolume advances a single volume one step through its failback +// phase machine without blocking. Returns (true, nil) if the volume is still in +// progress and needs a requeue, (false, nil) when it reaches a terminal phase, +// or (false, err) on a fatal error that should mark the volume as Failed. +// +// Polling steps (WaitFirst, WaitSecond, WaitSourceDeleted, WaitTargetDeleted) +// treat transient API errors as non-fatal and return (true, nil) so the +// reconciler retries on the next requeue rather than permanently failing the +// volume. func (r *SnapshotReplicationReconciler) advanceFailbackVolume( ctx context.Context, apiClient *webapi.Client, @@ -260,32 +270,140 @@ func (r *SnapshotReplicationReconciler) advanceFailbackVolume( targetPoolUUID string, lvolDetail *utils.Lvol, currentPhase string, -) error { +) (stillInProgress bool, err error) { + log := logf.FromContext(ctx) + switch currentPhase { - case simplyblockv1alpha1.VolPhaseCompleted, simplyblockv1alpha1.VolPhaseFailed: - return nil - } - r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseReplicatingToSource, "failback in progress") + // Initial entry point: resolve source info once and persist it before + // making any API calls so subsequent phases are crash-safe. + case "", simplyblockv1alpha1.VolPhaseReplicatingToSource: + sourcePoolUUID, sourceLvolUUID, isFreshCluster, resolveErr := r.resolveSourceFailbackTarget( + ctx, apiClient, snapRepCR, sourceClusterSecret, sourceClusterUUID, lvolDetail, + ) + if resolveErr != nil { + return false, fmt.Errorf("resolve source failback target: %w", resolveErr) + } + if isFreshCluster { + // startReplicationOnFreshSource is a known gap (separate issue): + // the source cluster does not yet have the volume, so that step is + // skipped. deleteLvol on source will receive a 404 and continue. + log.Info("Failback: fresh source cluster — skipping replication_start step", + "lvolUUID", lvolDetail.UUID) + } + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackTriggerFirst, "") + r.setFailbackState(snapRepCR, lvolDetail.UUID, sourcePoolUUID, sourceLvolUUID, isFreshCluster) + return true, nil - sourcePoolUUID, sourceLvolUUID, isFreshCluster, err := r.resolveSourceFailbackTarget( - ctx, apiClient, snapRepCR, sourceClusterSecret, sourceClusterUUID, lvolDetail, - ) - if err != nil { - return fmt.Errorf("resolve source failback target: %w", err) - } + case simplyblockv1alpha1.VolPhaseFailbackTriggerFirst: + if trigErr := triggerReplication(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID); trigErr != nil { + return false, fmt.Errorf("trigger first failback replication: %w", trigErr) + } + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackWaitFirst, "") + return true, nil - if err := failbackLvol( - ctx, apiClient, - sourceClusterSecret, sourceClusterUUID, sourcePoolUUID, sourceLvolUUID, - targetClusterSecret, targetClusterUUID, targetPoolUUID, - lvolDetail, isFreshCluster, - ); err != nil { - return fmt.Errorf("failback failed for lvol %s: %w", lvolDetail.UUID, err) - } + case simplyblockv1alpha1.VolPhaseFailbackWaitFirst: + done, _, pollErr := utils.GetLastSnapshotTaskDoneStatus(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID) + if pollErr != nil { + log.Error(pollErr, "Failback: poll first replication task failed, will retry", "lvolUUID", lvolDetail.UUID) + return true, nil + } + if !done { + log.Info("Failback: first replication task pending", "lvolUUID", lvolDetail.UUID) + return true, nil + } + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackSuspend, "") + return true, nil - r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseCompleted, "failback complete") - return nil + case simplyblockv1alpha1.VolPhaseFailbackSuspend: + if suspErr := suspendLvol(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID); suspErr != nil { + return false, fmt.Errorf("suspend target lvol: %w", suspErr) + } + if trigErr := triggerReplication(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID); trigErr != nil { + return false, fmt.Errorf("trigger second failback replication: %w", trigErr) + } + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackWaitSecond, "") + return true, nil + + case simplyblockv1alpha1.VolPhaseFailbackWaitSecond: + done, _, pollErr := utils.GetLastSnapshotTaskDoneStatus(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID) + if pollErr != nil { + log.Error(pollErr, "Failback: poll second replication task failed, will retry", "lvolUUID", lvolDetail.UUID) + return true, nil + } + if !done { + log.Info("Failback: second replication task pending", "lvolUUID", lvolDetail.UUID) + return true, nil + } + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackDeleteSource, "") + return true, nil + + case simplyblockv1alpha1.VolPhaseFailbackDeleteSource: + state := r.getFailbackState(snapRepCR, lvolDetail.UUID) + if state == nil { + return false, fmt.Errorf("missing failback state for lvol %s at DeleteSource", lvolDetail.UUID) + } + if delErr := deleteLvol(ctx, apiClient, sourceClusterSecret, sourceClusterUUID, state.SourcePoolUUID, state.SourceLvolUUID); delErr != nil { + return false, fmt.Errorf("delete source lvol: %w", delErr) + } + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackWaitSourceDeleted, "") + return true, nil + + case simplyblockv1alpha1.VolPhaseFailbackWaitSourceDeleted: + state := r.getFailbackState(snapRepCR, lvolDetail.UUID) + if state == nil { + return false, fmt.Errorf("missing failback state for lvol %s at WaitSourceDeleted", lvolDetail.UUID) + } + srcLvol, getErr := utils.GetLvol(ctx, apiClient, sourceClusterSecret, sourceClusterUUID, state.SourcePoolUUID, state.SourceLvolUUID) + if getErr != nil { + if errors.Is(getErr, utils.ErrLvolNotFound) { + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackReplicateOnSource, "") + return true, nil + } + log.Error(getErr, "Failback: poll source lvol deletion failed, will retry", "lvolUUID", state.SourceLvolUUID) + return true, nil + } + if strings.EqualFold(strings.TrimSpace(srcLvol.Status), "deleted") { + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackReplicateOnSource, "") + return true, nil + } + log.Info("Failback: source lvol pending deletion", "sourceLvolUUID", state.SourceLvolUUID) + return true, nil + + case simplyblockv1alpha1.VolPhaseFailbackReplicateOnSource: + state := r.getFailbackState(snapRepCR, lvolDetail.UUID) + if state == nil { + return false, fmt.Errorf("missing failback state for lvol %s at ReplicateOnSource", lvolDetail.UUID) + } + if repErr := replicateLvolOnSourceCluster(ctx, apiClient, sourceClusterSecret, sourceClusterUUID, state.SourcePoolUUID, state.SourceLvolUUID); repErr != nil { + return false, fmt.Errorf("replicate lvol on source cluster: %w", repErr) + } + if delErr := deleteLvol(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID); delErr != nil { + return false, fmt.Errorf("delete target lvol: %w", delErr) + } + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackWaitTargetDeleted, "") + return true, nil + + case simplyblockv1alpha1.VolPhaseFailbackWaitTargetDeleted: + tgtLvol, getErr := utils.GetLvol(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID) + if getErr != nil { + if errors.Is(getErr, utils.ErrLvolNotFound) { + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseCompleted, "failback complete") + return false, nil + } + log.Error(getErr, "Failback: poll target lvol deletion failed, will retry", "lvolUUID", lvolDetail.UUID) + return true, nil + } + if strings.EqualFold(strings.TrimSpace(tgtLvol.Status), "deleted") { + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseCompleted, "failback complete") + return false, nil + } + log.Info("Failback: target lvol pending deletion", "lvolUUID", lvolDetail.UUID) + return true, nil + + default: + return false, nil + } } /* -------------------- Normal periodic replication -------------------- */ @@ -689,6 +807,36 @@ func (r *SnapshotReplicationReconciler) setVolumeRepInfo( } } +func (r *SnapshotReplicationReconciler) setFailbackState( + snapRepCR *simplyblockv1alpha1.SnapshotReplication, + volumeID, sourcePoolUUID, sourceLvolUUID string, + isFreshCluster bool, +) { + state := &simplyblockv1alpha1.FailbackVolumeState{ + SourcePoolUUID: sourcePoolUUID, + SourceLvolUUID: sourceLvolUUID, + IsFreshCluster: isFreshCluster, + } + for i := range snapRepCR.Status.Volumes { + if snapRepCR.Status.Volumes[i].VolumeID == volumeID { + snapRepCR.Status.Volumes[i].FailbackState = state + return + } + } +} + +func (r *SnapshotReplicationReconciler) getFailbackState( + snapRepCR *simplyblockv1alpha1.SnapshotReplication, + volumeID string, +) *simplyblockv1alpha1.FailbackVolumeState { + for _, v := range snapRepCR.Status.Volumes { + if v.VolumeID == volumeID { + return v.FailbackState + } + } + return nil +} + func (r *SnapshotReplicationReconciler) setVolumeLastReplicationTime( snapRepCR *simplyblockv1alpha1.SnapshotReplication, volumeID string, @@ -961,157 +1109,6 @@ func failbackFilterID(lvol *utils.Lvol) string { return lvol.UUID } -func failbackLvol( - ctx context.Context, - apiClient *webapi.Client, - sourceClusterSecret string, - sourceClusterUUID string, - sourcePoolUUID string, - sourceLvolUUID string, - targetClusterSecret string, - targetClusterUUID string, - targetPoolUUID string, - targetLvol *utils.Lvol, - isFreshCluster bool, -) error { - if isFreshCluster { - if err := startReplicationOnFreshSource( - ctx, - apiClient, - sourceClusterSecret, - sourceClusterUUID, - sourcePoolUUID, - targetLvol.UUID, - 10*time.Minute, - 5*time.Second, - ); err != nil { - return fmt.Errorf("start replication on fresh source cluster failed for target lvol %s: %w", targetLvol.UUID, err) - } - } - - if err := triggerReplication(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, targetLvol.UUID); err != nil { - return fmt.Errorf("target trigger replication failed for lvol %s: %w", targetLvol.UUID, err) - } - - if err := waitForReplicationTaskCompletion( - ctx, - apiClient, - targetClusterSecret, - targetClusterUUID, - targetPoolUUID, - targetLvol.UUID, - 10*time.Minute, - 5*time.Second, - ); err != nil { - return fmt.Errorf("waiting for first target replication task failed for lvol %s: %w", targetLvol.UUID, err) - } - - if err := suspendLvol(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, targetLvol.UUID); err != nil { - return fmt.Errorf("suspend target lvol failed for lvol %s: %w", targetLvol.UUID, err) - } - - if err := triggerReplication(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, targetLvol.UUID); err != nil { - return fmt.Errorf("second target trigger replication failed for lvol %s: %w", targetLvol.UUID, err) - } - - if err := waitForReplicationTaskCompletion( - ctx, - apiClient, - targetClusterSecret, - targetClusterUUID, - targetPoolUUID, - targetLvol.UUID, - 10*time.Minute, - 5*time.Second, - ); err != nil { - return fmt.Errorf("waiting for second target replication task failed for lvol %s: %w", targetLvol.UUID, err) - } - - if err := deleteLvol(ctx, apiClient, sourceClusterSecret, sourceClusterUUID, sourcePoolUUID, sourceLvolUUID); err != nil { - return fmt.Errorf("delete source lvol failed for lvol %s: %w", sourceLvolUUID, err) - } - - if err := waitForLvolDeleted( - ctx, - apiClient, - sourceClusterSecret, - sourceClusterUUID, - sourcePoolUUID, - sourceLvolUUID, - 10*time.Minute, - 5*time.Second, - ); err != nil { - return fmt.Errorf("waiting for source lvol %s to reach deleted state failed: %w", sourceLvolUUID, err) - } - - if err := replicateLvolOnSourceCluster( - ctx, - apiClient, - sourceClusterSecret, - sourceClusterUUID, - sourcePoolUUID, - sourceLvolUUID, - ); err != nil { - return fmt.Errorf("replicate lvol on source cluster failed for source lvol %s: %w", sourceLvolUUID, err) - } - - if err := deleteLvol(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, targetLvol.UUID); err != nil { - return fmt.Errorf("delete target lvol failed for lvol %s: %w", targetLvol.UUID, err) - } - - if err := waitForLvolDeleted( - ctx, - apiClient, - targetClusterSecret, - targetClusterUUID, - targetPoolUUID, - targetLvol.UUID, - 10*time.Minute, - 5*time.Second, - ); err != nil { - return fmt.Errorf("waiting for target lvol %s to reach deleted state failed: %w", targetLvol.UUID, err) - } - - return nil -} - -func startReplicationOnFreshSource( - ctx context.Context, - apiClient *webapi.Client, - sourceClusterSecret string, - sourceClusterUUID string, - sourcePoolUUID string, - targetLvolUUID string, - timeout time.Duration, - pollInterval time.Duration, -) error { - endpoint := fmt.Sprintf( - "/api/v2/clusters/%s/storage-pools/%s/volumes/%s/replication_start", - sourceClusterUUID, - sourcePoolUUID, - targetLvolUUID, - ) - body, status, err := apiClient.Do(ctx, sourceClusterSecret, http.MethodPost, endpoint, nil) - if err != nil || status >= 300 { - return fmt.Errorf("replication_start failed for target lvol %s, status %d: %v, body: %s", targetLvolUUID, status, err, string(body)) - } - - if err := waitForReplicationTaskCompletion( - ctx, - apiClient, - sourceClusterSecret, - sourceClusterUUID, - sourcePoolUUID, - targetLvolUUID, - timeout, - pollInterval, - ); err != nil { - return fmt.Errorf("waiting for replication task on fresh source failed for lvol %s: %w", targetLvolUUID, err) - } - - return nil -} - func suspendLvol( ctx context.Context, apiClient *webapi.Client, @@ -1186,94 +1183,3 @@ func replicateLvolOnSourceCluster( } return nil } - -func waitForReplicationTaskCompletion( - ctx context.Context, - apiClient *webapi.Client, - clusterSecret string, - clusterUUID string, - poolUUID string, - lvolUUID string, - timeout time.Duration, - pollInterval time.Duration, -) error { - timeoutTimer := time.NewTimer(timeout) - defer timeoutTimer.Stop() - - ticker := time.NewTicker(pollInterval) - defer ticker.Stop() - - for { - done, task, err := utils.GetLastSnapshotTaskDoneStatus( - ctx, - apiClient, - clusterSecret, - clusterUUID, - poolUUID, - lvolUUID, - ) - if err != nil { - return fmt.Errorf("failed to get replication task status for lvol %s: %w", lvolUUID, err) - } - if done { - return nil - } - - select { - case <-ctx.Done(): - return ctx.Err() - case <-timeoutTimer.C: - return fmt.Errorf( - "timed out waiting for replication task completion for lvol %s (taskID=%s status=%s)", - lvolUUID, - task.UUID, - task.Status, - ) - case <-ticker.C: - } - } -} - -func waitForLvolDeleted( - ctx context.Context, - apiClient *webapi.Client, - clusterSecret string, - clusterUUID string, - poolUUID string, - lvolUUID string, - timeout time.Duration, - pollInterval time.Duration, -) error { - timeoutTimer := time.NewTimer(timeout) - defer timeoutTimer.Stop() - - ticker := time.NewTicker(pollInterval) - defer ticker.Stop() - - var lastErr error - - for { - lvol, err := utils.GetLvol(ctx, apiClient, clusterSecret, clusterUUID, poolUUID, lvolUUID) - if err == nil { - if strings.EqualFold(strings.TrimSpace(lvol.Status), "deleted") { - return nil - } - } else { - lastErr = err - if errors.Is(err, utils.ErrLvolNotFound) { - return nil - } - } - - select { - case <-ctx.Done(): - return ctx.Err() - case <-timeoutTimer.C: - if lastErr != nil { - return fmt.Errorf("timed out waiting for lvol %s deletion state; last get error: %w", lvolUUID, lastErr) - } - return fmt.Errorf("timed out waiting for lvol %s to reach deleted status", lvolUUID) - case <-ticker.C: - } - } -} From 95c5e3a33920d10ca0ba1cd6d1186d6b4a27d9d1 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 5 May 2026 16:46:04 +0100 Subject: [PATCH 05/15] split advanceFailbackVolume into replication and deletion phase helpers to fix gocyclo lint --- ...mplyblocksnapshotreplication_controller.go | 68 ++++++++++++++++--- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/internal/controller/simplyblocksnapshotreplication_controller.go b/internal/controller/simplyblocksnapshotreplication_controller.go index 428d32be..2190e9e3 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller.go +++ b/internal/controller/simplyblocksnapshotreplication_controller.go @@ -257,10 +257,10 @@ func (r *SnapshotReplicationReconciler) reconcileFailback( // progress and needs a requeue, (false, nil) when it reaches a terminal phase, // or (false, err) on a fatal error that should mark the volume as Failed. // -// Polling steps (WaitFirst, WaitSecond, WaitSourceDeleted, WaitTargetDeleted) -// treat transient API errors as non-fatal and return (true, nil) so the -// reconciler retries on the next requeue rather than permanently failing the -// volume. +// The implementation is split into two helpers to stay within cyclomatic +// complexity limits: advanceFailbackReplicationPhases handles the initial +// resolution and first-replication steps; advanceFailbackDeletionPhases handles +// the source/target cleanup steps. func (r *SnapshotReplicationReconciler) advanceFailbackVolume( ctx context.Context, apiClient *webapi.Client, @@ -270,7 +270,39 @@ func (r *SnapshotReplicationReconciler) advanceFailbackVolume( targetPoolUUID string, lvolDetail *utils.Lvol, currentPhase string, -) (stillInProgress bool, err error) { +) (bool, error) { + switch currentPhase { + case "", + simplyblockv1alpha1.VolPhaseReplicatingToSource, + simplyblockv1alpha1.VolPhaseFailbackTriggerFirst, + simplyblockv1alpha1.VolPhaseFailbackWaitFirst, + simplyblockv1alpha1.VolPhaseFailbackSuspend, + simplyblockv1alpha1.VolPhaseFailbackWaitSecond: + return r.advanceFailbackReplicationPhases(ctx, apiClient, snapRepCR, + sourceClusterUUID, sourceClusterSecret, + targetClusterUUID, targetClusterSecret, + targetPoolUUID, lvolDetail, currentPhase) + default: + return r.advanceFailbackDeletionPhases(ctx, apiClient, snapRepCR, + sourceClusterUUID, sourceClusterSecret, + targetClusterUUID, targetClusterSecret, + targetPoolUUID, lvolDetail, currentPhase) + } +} + +// advanceFailbackReplicationPhases handles the early failback steps: resolving +// source info, triggering and waiting for the two target replication rounds, and +// suspending the target lvol between them. +func (r *SnapshotReplicationReconciler) advanceFailbackReplicationPhases( + ctx context.Context, + apiClient *webapi.Client, + snapRepCR *simplyblockv1alpha1.SnapshotReplication, + sourceClusterUUID, sourceClusterSecret string, + targetClusterUUID, targetClusterSecret string, + targetPoolUUID string, + lvolDetail *utils.Lvol, + currentPhase string, +) (bool, error) { log := logf.FromContext(ctx) switch currentPhase { @@ -337,6 +369,27 @@ func (r *SnapshotReplicationReconciler) advanceFailbackVolume( } r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackDeleteSource, "") return true, nil + } + + return false, nil +} + +// advanceFailbackDeletionPhases handles the later failback steps: deleting the +// source lvol, calling replicate_lvol_on_source_cluster, deleting the target +// lvol, and polling for each deletion to complete. +func (r *SnapshotReplicationReconciler) advanceFailbackDeletionPhases( + ctx context.Context, + apiClient *webapi.Client, + snapRepCR *simplyblockv1alpha1.SnapshotReplication, + sourceClusterUUID, sourceClusterSecret string, + targetClusterUUID, targetClusterSecret string, + targetPoolUUID string, + lvolDetail *utils.Lvol, + currentPhase string, +) (bool, error) { + log := logf.FromContext(ctx) + + switch currentPhase { case simplyblockv1alpha1.VolPhaseFailbackDeleteSource: state := r.getFailbackState(snapRepCR, lvolDetail.UUID) @@ -400,10 +453,9 @@ func (r *SnapshotReplicationReconciler) advanceFailbackVolume( } log.Info("Failback: target lvol pending deletion", "lvolUUID", lvolDetail.UUID) return true, nil - - default: - return false, nil } + + return false, nil } /* -------------------- Normal periodic replication -------------------- */ From 9e8fe61e151f40e00248acd1ca675f02c6278bd4 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 6 May 2026 13:15:31 +0100 Subject: [PATCH 06/15] fix: split VolPhaseFailbackReplicateOnSource into separate DeleteTarget phase to prevent duplicate volume creation on retry --- .../simplyblocksnapshotreplication_controller.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/internal/controller/simplyblocksnapshotreplication_controller.go b/internal/controller/simplyblocksnapshotreplication_controller.go index 2190e9e3..37d3d686 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller.go +++ b/internal/controller/simplyblocksnapshotreplication_controller.go @@ -317,11 +317,10 @@ func (r *SnapshotReplicationReconciler) advanceFailbackReplicationPhases( return false, fmt.Errorf("resolve source failback target: %w", resolveErr) } if isFreshCluster { - // startReplicationOnFreshSource is a known gap (separate issue): - // the source cluster does not yet have the volume, so that step is - // skipped. deleteLvol on source will receive a 404 and continue. - log.Info("Failback: fresh source cluster — skipping replication_start step", - "lvolUUID", lvolDetail.UUID) + // Fresh source cluster: the volume does not exist there yet. + // deleteLvol on source will receive a 404 (handled gracefully) and + // replicate_lvol_on_source_cluster will create it from the target snapshot. + log.Info("Failback: fresh source cluster detected", "lvolUUID", lvolDetail.UUID) } r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackTriggerFirst, "") r.setFailbackState(snapRepCR, lvolDetail.UUID, sourcePoolUUID, sourceLvolUUID, isFreshCluster) @@ -431,6 +430,10 @@ func (r *SnapshotReplicationReconciler) advanceFailbackDeletionPhases( if repErr := replicateLvolOnSourceCluster(ctx, apiClient, sourceClusterSecret, sourceClusterUUID, state.SourcePoolUUID, state.SourceLvolUUID); repErr != nil { return false, fmt.Errorf("replicate lvol on source cluster: %w", repErr) } + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackDeleteTarget, "") + return true, nil + + case simplyblockv1alpha1.VolPhaseFailbackDeleteTarget: if delErr := deleteLvol(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID); delErr != nil { return false, fmt.Errorf("delete target lvol: %w", delErr) } From 95478aed2656eee3c6adba1e3785812e96b1f02b Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 6 May 2026 16:42:48 +0100 Subject: [PATCH 07/15] fix: use NQN-extracted source lvol UUID for fresh cluster failback to prevent delete hitting target node --- .../simplyblocksnapshotreplication_controller.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/controller/simplyblocksnapshotreplication_controller.go b/internal/controller/simplyblocksnapshotreplication_controller.go index 37d3d686..f1e6cc34 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller.go +++ b/internal/controller/simplyblocksnapshotreplication_controller.go @@ -1057,7 +1057,14 @@ func (r *SnapshotReplicationReconciler) resolveSourceFailbackTarget( if err != nil { return "", "", true, err } - return sourcePoolUUID, targetLvol.UUID, true, nil + // The target lvol's NQN encodes the original source lvol UUID (e.g. ...lvol:). + // Using targetLvol.UUID would point sbcli at the target's FDB record, causing + // DeleteSource to delete from the target node instead of the source. + sourceLvolUUID, ok := lvolIDFromNQN(targetLvol.NQN) + if !ok { + return "", "", true, fmt.Errorf("cannot extract source lvol UUID from target NQN %q", targetLvol.NQN) + } + return sourcePoolUUID, sourceLvolUUID, true, nil } /* -------------------- Pure functions -------------------- */ From 3f651b41333d74aa1fee65a4824abfa5c62434e9 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 7 May 2026 14:30:24 +0100 Subject: [PATCH 08/15] refactor(failback): replace first replication_trigger with replication_start using source cluster id --- api/v1alpha1/snapshotreplication_types.go | 18 +++++----- ...mplyblocksnapshotreplication_controller.go | 34 +++++++++++++------ 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/api/v1alpha1/snapshotreplication_types.go b/api/v1alpha1/snapshotreplication_types.go index 5670e590..897844be 100644 --- a/api/v1alpha1/snapshotreplication_types.go +++ b/api/v1alpha1/snapshotreplication_types.go @@ -66,17 +66,17 @@ const ( // Source info is resolved once and stored in FailbackVolumeState so that // every subsequent step can be re-entered safely after a crash or requeue. - // VolPhaseFailbackTriggerFirst means the first replication trigger on the - // target cluster is pending (source info already resolved and persisted). - VolPhaseFailbackTriggerFirst = "FailbackTriggerFirst" - // VolPhaseFailbackWaitFirst means the controller is polling for the first - // target replication task to complete. - VolPhaseFailbackWaitFirst = "FailbackWaitFirst" - // VolPhaseFailbackSuspend means the target lvol should be suspended and the - // second replication triggered in the same step. + // VolPhaseFailbackStartReplication means replication_start should be called + // on the target lvol with the source cluster as the replication destination. + VolPhaseFailbackStartReplication = "FailbackStartReplication" + // VolPhaseFailbackWaitReplication means the controller is polling for the + // first replication task (started via replication_start) to complete. + VolPhaseFailbackWaitReplication = "FailbackWaitReplication" + // VolPhaseFailbackSuspend means the target lvol should be suspended and + // replication_trigger called to capture the final delta snapshot. VolPhaseFailbackSuspend = "FailbackSuspend" // VolPhaseFailbackWaitSecond means the controller is polling for the second - // target replication task to complete. + // (post-suspend) replication task to complete. VolPhaseFailbackWaitSecond = "FailbackWaitSecond" // VolPhaseFailbackDeleteSource means the source lvol should be deleted. VolPhaseFailbackDeleteSource = "FailbackDeleteSource" diff --git a/internal/controller/simplyblocksnapshotreplication_controller.go b/internal/controller/simplyblocksnapshotreplication_controller.go index f1e6cc34..420b3156 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller.go +++ b/internal/controller/simplyblocksnapshotreplication_controller.go @@ -274,8 +274,8 @@ func (r *SnapshotReplicationReconciler) advanceFailbackVolume( switch currentPhase { case "", simplyblockv1alpha1.VolPhaseReplicatingToSource, - simplyblockv1alpha1.VolPhaseFailbackTriggerFirst, - simplyblockv1alpha1.VolPhaseFailbackWaitFirst, + simplyblockv1alpha1.VolPhaseFailbackStartReplication, + simplyblockv1alpha1.VolPhaseFailbackWaitReplication, simplyblockv1alpha1.VolPhaseFailbackSuspend, simplyblockv1alpha1.VolPhaseFailbackWaitSecond: return r.advanceFailbackReplicationPhases(ctx, apiClient, snapRepCR, @@ -322,25 +322,25 @@ func (r *SnapshotReplicationReconciler) advanceFailbackReplicationPhases( // replicate_lvol_on_source_cluster will create it from the target snapshot. log.Info("Failback: fresh source cluster detected", "lvolUUID", lvolDetail.UUID) } - r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackTriggerFirst, "") + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackStartReplication, "") r.setFailbackState(snapRepCR, lvolDetail.UUID, sourcePoolUUID, sourceLvolUUID, isFreshCluster) return true, nil - case simplyblockv1alpha1.VolPhaseFailbackTriggerFirst: - if trigErr := triggerReplication(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID); trigErr != nil { - return false, fmt.Errorf("trigger first failback replication: %w", trigErr) + case simplyblockv1alpha1.VolPhaseFailbackStartReplication: + if startErr := replicationStart(ctx, apiClient, targetClusterSecret, sourceClusterUUID, targetPoolUUID, lvolDetail.UUID); startErr != nil { + return false, fmt.Errorf("failback replication_start: %w", startErr) } - r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackWaitFirst, "") + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackWaitReplication, "") return true, nil - case simplyblockv1alpha1.VolPhaseFailbackWaitFirst: + case simplyblockv1alpha1.VolPhaseFailbackWaitReplication: done, _, pollErr := utils.GetLastSnapshotTaskDoneStatus(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID) if pollErr != nil { - log.Error(pollErr, "Failback: poll first replication task failed, will retry", "lvolUUID", lvolDetail.UUID) + log.Error(pollErr, "Failback: poll replication task failed, will retry", "lvolUUID", lvolDetail.UUID) return true, nil } if !done { - log.Info("Failback: first replication task pending", "lvolUUID", lvolDetail.UUID) + log.Info("Failback: replication task pending", "lvolUUID", lvolDetail.UUID) return true, nil } r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackSuspend, "") @@ -351,7 +351,7 @@ func (r *SnapshotReplicationReconciler) advanceFailbackReplicationPhases( return false, fmt.Errorf("suspend target lvol: %w", suspErr) } if trigErr := triggerReplication(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolDetail.UUID); trigErr != nil { - return false, fmt.Errorf("trigger second failback replication: %w", trigErr) + return false, fmt.Errorf("trigger failback replication: %w", trigErr) } r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackWaitSecond, "") return true, nil @@ -1069,6 +1069,18 @@ func (r *SnapshotReplicationReconciler) resolveSourceFailbackTarget( /* -------------------- Pure functions -------------------- */ +func replicationStart(ctx context.Context, apiClient *webapi.Client, targetClusterSecret, sourceClusterUUID, targetPoolUUID, targetLvolUUID string) error { + endpoint := fmt.Sprintf( + "/api/v2/clusters/%s/storage-pools/%s/volumes/%s/replication_start", + sourceClusterUUID, targetPoolUUID, targetLvolUUID, + ) + body, status, err := apiClient.Do(ctx, targetClusterSecret, http.MethodPost, endpoint, nil) + if err != nil || status >= 300 { + return fmt.Errorf("replication_start for lvol %s, status %d: %v, body: %s", targetLvolUUID, status, err, string(body)) + } + return nil +} + func triggerReplication(ctx context.Context, apiClient *webapi.Client, clusterSecret, clusterUUID, poolUUID, lvolUUID string) error { endpoint := fmt.Sprintf( "/api/v2/clusters/%s/storage-pools/%s/volumes/%s/replication_trigger/", From b4e369cb138fabe0f7582d30491cd1f42789d58d Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 7 May 2026 16:10:01 +0100 Subject: [PATCH 09/15] fix: add FailbackStartReplication and FailbackWaitReplication to CRD phase enum --- api/v1alpha1/snapshotreplication_types.go | 2 +- .../bases/storage.simplyblock.io_snapshotreplications.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/v1alpha1/snapshotreplication_types.go b/api/v1alpha1/snapshotreplication_types.go index 897844be..3009d280 100644 --- a/api/v1alpha1/snapshotreplication_types.go +++ b/api/v1alpha1/snapshotreplication_types.go @@ -180,7 +180,7 @@ type VolumeReplicationStatus struct { VolumeID string `json:"volumeID,omitempty"` // Phase is the current replication phase for this volume. - // +kubebuilder:validation:Enum=Pending;Running;TriggeringTargetReplication;WaitingForTargetReplication;ReplicatingToSource;WaitingForTargetDeletion;Completed;Failed;Paused;FailbackTriggerFirst;FailbackWaitFirst;FailbackSuspend;FailbackWaitSecond;FailbackDeleteSource;FailbackWaitSourceDeleted;FailbackReplicateOnSource;FailbackDeleteTarget;FailbackWaitTargetDeleted + // +kubebuilder:validation:Enum=Pending;Running;TriggeringTargetReplication;WaitingForTargetReplication;ReplicatingToSource;WaitingForTargetDeletion;Completed;Failed;Paused;FailbackStartReplication;FailbackWaitReplication;FailbackSuspend;FailbackWaitSecond;FailbackDeleteSource;FailbackWaitSourceDeleted;FailbackReplicateOnSource;FailbackDeleteTarget;FailbackWaitTargetDeleted Phase string `json:"phase,omitempty"` // Last snapshot ID replicated for this volume diff --git a/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml b/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml index a4efc0f6..6f51182f 100644 --- a/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml +++ b/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml @@ -256,8 +256,8 @@ spec: - Completed - Failed - Paused - - FailbackTriggerFirst - - FailbackWaitFirst + - FailbackStartReplication + - FailbackWaitReplication - FailbackSuspend - FailbackWaitSecond - FailbackDeleteSource From 6f196a996e11c5f3390f04aead6fc51422cf9cce Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 7 May 2026 16:17:23 +0100 Subject: [PATCH 10/15] fix: use sourceClusterSecret for replication_start auth to match sourceClusterUUID in path --- .../controller/simplyblocksnapshotreplication_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/simplyblocksnapshotreplication_controller.go b/internal/controller/simplyblocksnapshotreplication_controller.go index 420b3156..83dad39f 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller.go +++ b/internal/controller/simplyblocksnapshotreplication_controller.go @@ -327,7 +327,7 @@ func (r *SnapshotReplicationReconciler) advanceFailbackReplicationPhases( return true, nil case simplyblockv1alpha1.VolPhaseFailbackStartReplication: - if startErr := replicationStart(ctx, apiClient, targetClusterSecret, sourceClusterUUID, targetPoolUUID, lvolDetail.UUID); startErr != nil { + if startErr := replicationStart(ctx, apiClient, sourceClusterSecret, sourceClusterUUID, targetPoolUUID, lvolDetail.UUID); startErr != nil { return false, fmt.Errorf("failback replication_start: %w", startErr) } r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailbackWaitReplication, "") From b57b890f7240bf3c2d88f19e5f23d6b0cc40da96 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 7 May 2026 19:28:11 +0100 Subject: [PATCH 11/15] updated snapshot replication crd --- api/v1alpha1/snapshotreplication_types.go | 2 +- .../bases/storage.simplyblock.io_snapshotreplications.yaml | 2 +- config/rbac/role.yaml | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/snapshotreplication_types.go b/api/v1alpha1/snapshotreplication_types.go index 9b718357..2c34bc98 100644 --- a/api/v1alpha1/snapshotreplication_types.go +++ b/api/v1alpha1/snapshotreplication_types.go @@ -137,7 +137,7 @@ type SnapshotReplicationSpec struct { // If empty, all volumes are candidates unless excluded below. IncludePVCRefs []PersistentVolumeClaimRef `json:"includePVCRefs,omitempty"` - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Exclude Volume IDs" + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Exclude Volume IDs" // Optional: PVCs to exclude from failback. ExcludePVCRefs []PersistentVolumeClaimRef `json:"excludePVCRefs,omitempty"` diff --git a/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml b/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml index 6f51182f..7edc607e 100644 --- a/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml +++ b/config/crd/bases/storage.simplyblock.io_snapshotreplications.yaml @@ -74,7 +74,7 @@ spec: type: array includePVCRefs: description: |- - Optional: only these PVCs are included in failback. + Optional: only these volumes are included in failback. If empty, all volumes are candidates unless excluded below. items: properties: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 5c05c532..3ca677d5 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -49,6 +49,7 @@ rules: - serviceaccounts - services verbs: + - create - delete - get - list @@ -56,11 +57,9 @@ rules: - update - watch - apiGroups: - - "" + - apps resources: - - secrets - - serviceaccounts - - services + - daemonsets verbs: - create - delete From 98f2e1a1027f0234cf84422fccec6bb38ca634c6 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 8 May 2026 13:23:54 +0100 Subject: [PATCH 12/15] fix failback loop processing REP_ transfer lvols as failback candidates by skipping volumes with REP_ name prefix --- .../controller/simplyblocksnapshotreplication_controller.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/controller/simplyblocksnapshotreplication_controller.go b/internal/controller/simplyblocksnapshotreplication_controller.go index 83dad39f..df7805d0 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller.go +++ b/internal/controller/simplyblocksnapshotreplication_controller.go @@ -202,6 +202,12 @@ func (r *SnapshotReplicationReconciler) reconcileFailback( anyInProgress := false for _, lvolSummary := range lvols { + // Skip REP_ transfer lvols created by the snapshot replication service; + // only user volumes (created during failover) are failback candidates. + if strings.HasPrefix(lvolSummary.Name, "REP_") { + continue + } + lvolDetail, err := utils.GetLvol(ctx, apiClient, targetClusterSecret, targetClusterUUID, targetPoolUUID, lvolSummary.UUID) if err != nil { log.Error(err, "Failed to get target lvol", "lvolUUID", lvolSummary.UUID) From 5e22a63122056f4e1d08a53bc4434f01234ff008 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 8 May 2026 15:42:10 +0100 Subject: [PATCH 13/15] fix: mark volume Failed when replicate_lvol_on_target_cluster has no snapshot to clone --- .../controller/simplyblocksnapshotreplication_controller.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/controller/simplyblocksnapshotreplication_controller.go b/internal/controller/simplyblocksnapshotreplication_controller.go index df7805d0..01279264 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller.go +++ b/internal/controller/simplyblocksnapshotreplication_controller.go @@ -731,7 +731,8 @@ func (r *SnapshotReplicationReconciler) handleFailoverReplication( } if err := replicateLvol(ctx, apiClient, clusterSecret, clusterUUID, poolUUID, lvolDetail.UUID); err != nil { - log.Error(err, "Failed to trigger replicate_lvol", "lvolUUID", lvolDetail.UUID) + log.Error(err, "Failed to trigger replicate_lvol — no replicated snapshot available yet, skipping failover for this volume", "lvolUUID", lvolDetail.UUID) + r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseFailed, err.Error()) return false } From d73c191542dd24b1d4b4abfe104f4d57cf1f09ab Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 11 May 2026 10:49:47 +0100 Subject: [PATCH 14/15] feat: populate pvcRef in volume status from lvol pvc_name when pvcRefs not specified --- ...mplyblocksnapshotreplication_controller.go | 35 ++++++++++++++++++- internal/utils/objects.go | 1 + 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/internal/controller/simplyblocksnapshotreplication_controller.go b/internal/controller/simplyblocksnapshotreplication_controller.go index 01279264..539748c0 100644 --- a/internal/controller/simplyblocksnapshotreplication_controller.go +++ b/internal/controller/simplyblocksnapshotreplication_controller.go @@ -486,8 +486,17 @@ func (r *SnapshotReplicationReconciler) reconcileNormalReplication( } var pvcVolumeIDs []string + pvcRefByVolumeID := map[string]simplyblockv1alpha1.PersistentVolumeClaimRef{} if len(snapRepCR.Spec.PVCRefs) > 0 { - pvcVolumeIDs = r.resolvePVCRefsToVolumeIDs(ctx, snapRepCR.Namespace, snapRepCR.Spec.PVCRefs) + for _, ref := range snapRepCR.Spec.PVCRefs { + id, err := r.resolvePVCRefToLvolID(ctx, snapRepCR.Namespace, ref) + if err != nil { + log.Error(err, "Failed to resolve PVCRef to volume ID", "pvc", ref.Name) + continue + } + pvcVolumeIDs = append(pvcVolumeIDs, id) + pvcRefByVolumeID[id] = ref + } } interval := utils.IntPtrOrDefault(snapRepCR.Spec.Interval, 300) @@ -525,10 +534,18 @@ func (r *SnapshotReplicationReconciler) reconcileNormalReplication( continue } + pvcRef := pvcRefByVolumeID[lvolDetail.UUID] + if pvcRef.Name == "" && lvolDetail.PvcName != "" { + pvcRef = simplyblockv1alpha1.PersistentVolumeClaimRef{Name: lvolDetail.PvcName} + } + if failover { triggered := r.handleFailoverReplication(ctx, apiClient, snapRepCR, sourceClusterUUID, sourceClusterSecret, poolUUID, lvolDetail, targetIDs) if triggered { r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseTriggeringTargetReplication, "failover replicate_lvol dispatched") + if pvcRef.Name != "" { + r.setVolumePVCRef(snapRepCR, lvolDetail.UUID, pvcRef) + } changed = true } continue @@ -537,6 +554,9 @@ func (r *SnapshotReplicationReconciler) reconcileNormalReplication( triggered := r.handleNormalReplication(ctx, apiClient, sourceClusterUUID, sourceClusterSecret, poolUUID, lvolDetail, interval, now) if triggered { r.setVolumePhase(snapRepCR, lvolDetail.UUID, simplyblockv1alpha1.VolPhaseRunning, "replication triggered") + if pvcRef.Name != "" { + r.setVolumePVCRef(snapRepCR, lvolDetail.UUID, pvcRef) + } now2 := metav1.Now() r.setVolumeLastReplicationTime(snapRepCR, lvolDetail.UUID, &now2) r.setVolumeRepInfo(snapRepCR, lvolDetail) @@ -852,6 +872,19 @@ func (r *SnapshotReplicationReconciler) setVolumePhase( snapRepCR.Status.Volumes = append(snapRepCR.Status.Volumes, entry) } +func (r *SnapshotReplicationReconciler) setVolumePVCRef( + snapRepCR *simplyblockv1alpha1.SnapshotReplication, + volumeID string, + pvcRef simplyblockv1alpha1.PersistentVolumeClaimRef, +) { + for i := range snapRepCR.Status.Volumes { + if snapRepCR.Status.Volumes[i].VolumeID == volumeID { + snapRepCR.Status.Volumes[i].PVCRef = pvcRef + return + } + } +} + func (r *SnapshotReplicationReconciler) setVolumeRepInfo( snapRepCR *simplyblockv1alpha1.SnapshotReplication, lvol *utils.Lvol, diff --git a/internal/utils/objects.go b/internal/utils/objects.go index 814aec50..38af84f2 100644 --- a/internal/utils/objects.go +++ b/internal/utils/objects.go @@ -35,6 +35,7 @@ type Lvol struct { DoReplicate bool `json:"do_replicate"` NQN string `json:"nqn"` Status string `json:"status"` + PvcName string `json:"pvc_name,omitempty"` RepInfo *ReplicationInfo `json:"rep_info,omitempty"` } From 1eb937c7e09181e164c214de0d34f7e83f34d711 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 22 May 2026 10:31:41 +0100 Subject: [PATCH 15/15] ran make generate --- api/v1alpha1/zz_generated.deepcopy.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 4c162ba3..fc99c379 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -532,31 +532,31 @@ func (in *ControlPlaneStatus) DeepCopy() *ControlPlaneStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HashicorpVaultSettings) DeepCopyInto(out *HashicorpVaultSettings) { +func (in *FailbackVolumeState) DeepCopyInto(out *FailbackVolumeState) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HashicorpVaultSettings. -func (in *HashicorpVaultSettings) DeepCopy() *HashicorpVaultSettings { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FailbackVolumeState. +func (in *FailbackVolumeState) DeepCopy() *FailbackVolumeState { if in == nil { return nil } - out := new(HashicorpVaultSettings) + out := new(FailbackVolumeState) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FailbackVolumeState) DeepCopyInto(out *FailbackVolumeState) { +func (in *HashicorpVaultSettings) DeepCopyInto(out *HashicorpVaultSettings) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FailbackVolumeState. -func (in *FailbackVolumeState) DeepCopy() *FailbackVolumeState { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HashicorpVaultSettings. +func (in *HashicorpVaultSettings) DeepCopy() *HashicorpVaultSettings { if in == nil { return nil } - out := new(FailbackVolumeState) + out := new(HashicorpVaultSettings) in.DeepCopyInto(out) return out }