diff --git a/images/controller/cmd/main.go b/images/controller/cmd/main.go index 349f4ad..15b67a3 100644 --- a/images/controller/cmd/main.go +++ b/images/controller/cmd/main.go @@ -72,6 +72,14 @@ func main() { if err != nil { log.Error(err, "[main] unable to KubernetesDefaultConfigCreate") } + // Raise the shared manager client rate limit from the client-go default (QPS 5 / Burst 10). That + // default paces the whole VolumeCaptureRequest capture leg: under a concurrent multi-tree snapshot + // burst the VCR/VolumeSnapshotContent creates and status patches queue behind the 5 QPS limiter, so + // capture-done time grows ~linearly with tree count (measured ~13s for 10 trees, a ~0.77 tree/s + // staircase) regardless of MaxConcurrentReconciles. This mirrors the same fix already applied to the + // state-snapshotter controller client. + kConfig.QPS = 50 + kConfig.Burst = 100 log.Info("[main] kubernetes config has been successfully created.") scheme := apiruntime.NewScheme() diff --git a/images/controller/internal/controllers/constants.go b/images/controller/internal/controllers/constants.go index 0c3717d..622ac07 100644 --- a/images/controller/internal/controllers/constants.go +++ b/images/controller/internal/controllers/constants.go @@ -40,6 +40,7 @@ const ( LabelKeyVCRName = "vcr-name" LabelKeyCreatedBy = "storage.deckhouse.io/created-by" LabelKeyVCRNameFull = "storage.deckhouse.io/vcr-name" + LabelKeyVCRNamespaceFull = "storage.deckhouse.io/vcr-namespace" LabelKeyVCRUIDFull = "storage.deckhouse.io/vcr-uid" LabelKeySourcePVCName = "storage.deckhouse.io/source-pvc-name" LabelKeySourcePVCNamespace = "storage.deckhouse.io/source-pvc-namespace" diff --git a/images/controller/internal/controllers/volumecapturerequest_controller.go b/images/controller/internal/controllers/volumecapturerequest_controller.go index ae8a9b4..dbcf784 100644 --- a/images/controller/internal/controllers/volumecapturerequest_controller.go +++ b/images/controller/internal/controllers/volumecapturerequest_controller.go @@ -28,10 +28,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/util/retry" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + deckhousev1alpha1 "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" storagev1alpha1 "github.com/deckhouse/storage-foundation/api/v1alpha1" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" @@ -39,6 +44,25 @@ import ( "github.com/deckhouse/storage-foundation/images/controller/pkg/config" ) +// mapVolumeSnapshotContentToVCR maps a CSI VolumeSnapshotContent event to the owning +// VolumeCaptureRequest using the coordinate labels stamped at VSC creation (snapshot Snapshot mode). +// A VSC without those labels (e.g. created before this label existed, or unrelated to a VCR) maps to +// nothing and is ignored — the VCR's 5s requeue still covers it. +func mapVolumeSnapshotContentToVCR(_ context.Context, obj client.Object) []reconcile.Request { + labels := obj.GetLabels() + if labels == nil { + return nil + } + name := labels[LabelKeyVCRNameFull] + namespace := labels[LabelKeyVCRNamespaceFull] + if name == "" || namespace == "" { + return nil + } + return []reconcile.Request{{ + NamespacedName: client.ObjectKey{Namespace: namespace, Name: name}, + }} +} + // Invariants (architectural guarantees): // // 1. Ownership model: @@ -647,6 +671,27 @@ func (r *VolumeCaptureRequestController) finalizeVCR( func (r *VolumeCaptureRequestController) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&storagev1alpha1.VolumeCaptureRequest{}). + // L1 latency fix: wake the owning VCR the moment the CSI VolumeSnapshotContent flips + // readyToUse/error, instead of waiting for the 5s requeue. Mapping is by the VCR-coordinate + // labels stamped at VSC creation (mapVolumeSnapshotContentToVCR); the 5s requeue remains a + // safety net (e.g. VSCs created before the label existed). + Watches( + &snapshotv1.VolumeSnapshotContent{}, + handler.EnqueueRequestsFromMapFunc(mapVolumeSnapshotContentToVCR), + ). + WithOptions(controller.Options{ + // L2b-foundation: process independent VolumeCaptureRequests in parallel. Each VCR owns its + // own UID-scoped ObjectKeeper / VSC names (objectKeeperNameForVCR, snapshotVSCName) with no + // cross-VCR collisions, holds no shared mutable state (Config is read-only), and status + // writes go through RetryOnConflict; controller-runtime still serializes reconciles of the + // same VCR. This parallelizes the capture leg that gated concurrent tree snapshots once the + // state-snapshotter manifest path (L2b-ssc) was no longer the bottleneck. + MaxConcurrentReconciles: 4, + // Bound the per-item retry backoff (200ms floor -> 10s ceiling) so transient + // apiserver/network requeues re-run quickly instead of backing off to the controller-runtime + // default (~16min), mirroring the state-snapshotter controllers. + RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[ctrl.Request](200*time.Millisecond, 10*time.Second), + }). Complete(r) } diff --git a/images/controller/internal/controllers/volumecapturerequest_snapshot_bulk.go b/images/controller/internal/controllers/volumecapturerequest_snapshot_bulk.go index 41ef86c..7de3a7d 100644 --- a/images/controller/internal/controllers/volumecapturerequest_snapshot_bulk.go +++ b/images/controller/internal/controllers/volumecapturerequest_snapshot_bulk.go @@ -288,6 +288,15 @@ func (r *VolumeCaptureRequestController) processSnapshotTarget( csiVSC = &snapshotv1.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: csiVSCName, + // Stamp the owning VCR coordinates so the VSC watch (SetupWithManager) can map a + // readyToUse/error transition straight back to this VCR and reconcile it immediately, + // instead of noticing it only on the 5s requeue tick. This is the L1 latency fix; the + // requeue stays as a safety net for VSCs created before this label existed. + Labels: map[string]string{ + LabelKeyVCRNameFull: vcr.Name, + LabelKeyVCRNamespaceFull: vcr.Namespace, + LabelKeyVCRUIDFull: string(vcr.UID), + }, OwnerReferences: []metav1.OwnerReference{{ APIVersion: APIGroupDeckhouse, Kind: KindObjectKeeper, diff --git a/images/controller/internal/controllers/volumecapturerequest_vsc_watch_test.go b/images/controller/internal/controllers/volumecapturerequest_vsc_watch_test.go new file mode 100644 index 0000000..3f4f191 --- /dev/null +++ b/images/controller/internal/controllers/volumecapturerequest_vsc_watch_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "testing" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestMapVolumeSnapshotContentToVCR(t *testing.T) { + tests := []struct { + name string + labels map[string]string + wantLen int + wantName string + wantNsName string + }{ + { + name: "labeled VSC maps to owning VCR", + labels: map[string]string{ + LabelKeyVCRNameFull: "test-vcr", + LabelKeyVCRNamespaceFull: "ns1", + LabelKeyVCRUIDFull: "uid-1", + }, + wantLen: 1, + wantName: "test-vcr", + wantNsName: "ns1", + }, + { + name: "VSC without labels maps to nothing", + labels: nil, + wantLen: 0, + }, + { + name: "VSC missing namespace label maps to nothing", + labels: map[string]string{LabelKeyVCRNameFull: "test-vcr"}, + wantLen: 0, + }, + { + name: "VSC missing name label maps to nothing", + labels: map[string]string{LabelKeyVCRNamespaceFull: "ns1"}, + wantLen: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + vsc := &snapshotv1.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snapshot-uid-1-abc", + Labels: tc.labels, + }, + } + reqs := mapVolumeSnapshotContentToVCR(context.Background(), vsc) + if len(reqs) != tc.wantLen { + t.Fatalf("expected %d requests, got %d", tc.wantLen, len(reqs)) + } + if tc.wantLen == 1 { + if reqs[0].Name != tc.wantName || reqs[0].Namespace != tc.wantNsName { + t.Fatalf("expected %s/%s, got %s/%s", tc.wantNsName, tc.wantName, reqs[0].Namespace, reqs[0].Name) + } + } + }) + } +}