From e8153ea0cfef5d4dbc2a53eb1b0498f27ad482c9 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Tue, 23 Jun 2026 11:42:04 +0300 Subject: [PATCH 01/32] feat(agent): add LVM on file-backed loop devices (spec.fileDevices) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agents: Claude Opus 4.6 (ведущий, авто-режим) --- api/v1alpha1/lvm_volume_group.go | 30 +++-- api/v1alpha1/lvm_volume_group_set.go | 11 +- api/v1alpha1/zz_generated.deepcopy.go | 53 +++++++++ crds/lvmvolumegroup.yaml | 52 ++++++++- crds/lvmvolumegroupset.yaml | 29 ++++- docs/FAQ.md | 19 ++++ docs/RESOURCES.md | 52 +++++++++ images/agent/cmd/main.go | 45 ++++++++ images/agent/internal/const.go | 21 +++- .../internal/controller/lvg/discoverer.go | 103 +++++++++++++++--- .../controller/lvg/discoverer_test.go | 8 +- .../internal/controller/lvg/reconciler.go | 80 ++++++++++++++ .../controller/lvg/reconciler_test.go | 55 ++++++++++ images/agent/internal/mock_utils/commands.go | 77 +++++++++++++ images/agent/internal/type.go | 8 ++ images/agent/internal/utils/activation.go | 44 ++++++++ images/agent/internal/utils/commands.go | 79 ++++++++++++++ images/agent/internal/utils/commands_test.go | 4 +- .../agent/internal/utils/devicefilter_test.go | 14 ++- ...pconfiguration-blacklist-loop-devices.yaml | 26 +---- 20 files changed, 743 insertions(+), 67 deletions(-) diff --git a/api/v1alpha1/lvm_volume_group.go b/api/v1alpha1/lvm_volume_group.go index 54f1b66f9..48fdc9c4d 100644 --- a/api/v1alpha1/lvm_volume_group.go +++ b/api/v1alpha1/lvm_volume_group.go @@ -40,11 +40,18 @@ type LVMVolumeGroup struct { // +k8s:deepcopy-gen=true type LVMVolumeGroupSpec struct { - ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` - BlockDeviceSelector *metav1.LabelSelector `json:"blockDeviceSelector"` - ThinPools []LVMVolumeGroupThinPoolSpec `json:"thinPools"` - Type string `json:"type"` - Local LVMVolumeGroupLocalSpec `json:"local"` + ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` + BlockDeviceSelector *metav1.LabelSelector `json:"blockDeviceSelector,omitempty"` + ThinPools []LVMVolumeGroupThinPoolSpec `json:"thinPools"` + Type string `json:"type"` + Local LVMVolumeGroupLocalSpec `json:"local"` + FileDevices []LVMVolumeGroupFileDeviceSpec `json:"fileDevices,omitempty"` +} + +// +k8s:deepcopy-gen=true +type LVMVolumeGroupFileDeviceSpec struct { + Directory string `json:"directory"` + Size resource.Quantity `json:"size"` } // +k8s:deepcopy-gen=true @@ -73,8 +80,17 @@ type LVMVolumeGroupDevice struct { // +k8s:deepcopy-gen=true type LVMVolumeGroupNode struct { - Devices []LVMVolumeGroupDevice `json:"devices"` - Name string `json:"name"` + Devices []LVMVolumeGroupDevice `json:"devices"` + FileDevices []LVMVolumeGroupFileDevice `json:"fileDevices,omitempty"` + Name string `json:"name"` +} + +// +k8s:deepcopy-gen=true +type LVMVolumeGroupFileDevice struct { + FilePath string `json:"filePath"` + LoopDevice string `json:"loopDevice"` + Size resource.Quantity `json:"size"` + PVUuid string `json:"pvUUID"` } // +k8s:deepcopy-gen=true diff --git a/api/v1alpha1/lvm_volume_group_set.go b/api/v1alpha1/lvm_volume_group_set.go index 0a4a9c951..30885346a 100644 --- a/api/v1alpha1/lvm_volume_group_set.go +++ b/api/v1alpha1/lvm_volume_group_set.go @@ -48,11 +48,12 @@ type LVMVolumeGroupSetSpec struct { // +k8s:deepcopy-gen=true type LVMVolumeGroupTemplate struct { - Metadata LVMVolumeGroupTemplateMeta `json:"metadata"` - BlockDeviceSelector *metav1.LabelSelector `json:"blockDeviceSelector"` - ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` - ThinPools []LVMVolumeGroupThinPoolSpec `json:"thinPools"` - Type string `json:"type"` + Metadata LVMVolumeGroupTemplateMeta `json:"metadata"` + BlockDeviceSelector *metav1.LabelSelector `json:"blockDeviceSelector"` + ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` + ThinPools []LVMVolumeGroupThinPoolSpec `json:"thinPools"` + Type string `json:"type"` + FileDevices []LVMVolumeGroupFileDeviceSpec `json:"fileDevices,omitempty"` } // +k8s:deepcopy-gen=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 7821a2bee..8a5cbaaa9 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -532,6 +532,38 @@ func (in *LVMVolumeGroupLocalSpec) DeepCopy() *LVMVolumeGroupLocalSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LVMVolumeGroupFileDevice) DeepCopyInto(out *LVMVolumeGroupFileDevice) { + *out = *in + out.Size = in.Size.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMVolumeGroupFileDevice. +func (in *LVMVolumeGroupFileDevice) DeepCopy() *LVMVolumeGroupFileDevice { + if in == nil { + return nil + } + out := new(LVMVolumeGroupFileDevice) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LVMVolumeGroupFileDeviceSpec) DeepCopyInto(out *LVMVolumeGroupFileDeviceSpec) { + *out = *in + out.Size = in.Size.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMVolumeGroupFileDeviceSpec. +func (in *LVMVolumeGroupFileDeviceSpec) DeepCopy() *LVMVolumeGroupFileDeviceSpec { + if in == nil { + return nil + } + out := new(LVMVolumeGroupFileDeviceSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LVMVolumeGroupNode) DeepCopyInto(out *LVMVolumeGroupNode) { *out = *in @@ -542,6 +574,13 @@ func (in *LVMVolumeGroupNode) DeepCopyInto(out *LVMVolumeGroupNode) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.FileDevices != nil { + in, out := &in.FileDevices, &out.FileDevices + *out = make([]LVMVolumeGroupFileDevice, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMVolumeGroupNode. @@ -683,6 +722,13 @@ func (in *LVMVolumeGroupSpec) DeepCopyInto(out *LVMVolumeGroupSpec) { copy(*out, *in) } out.Local = in.Local + if in.FileDevices != nil { + in, out := &in.FileDevices, &out.FileDevices + *out = make([]LVMVolumeGroupFileDeviceSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMVolumeGroupSpec. @@ -749,6 +795,13 @@ func (in *LVMVolumeGroupTemplate) DeepCopyInto(out *LVMVolumeGroupTemplate) { *out = make([]LVMVolumeGroupThinPoolSpec, len(*in)) copy(*out, *in) } + if in.FileDevices != nil { + in, out := &in.FileDevices, &out.FileDevices + *out = make([]LVMVolumeGroupFileDeviceSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMVolumeGroupTemplate. diff --git a/crds/lvmvolumegroup.yaml b/crds/lvmvolumegroup.yaml index cb3e58718..c21fa27e4 100644 --- a/crds/lvmvolumegroup.yaml +++ b/crds/lvmvolumegroup.yaml @@ -38,9 +38,11 @@ spec: - rule: | (self.type == "Local" && has(self.local)) || self.type != "Local" message: "The 'local' field is required when the 'type' field is 'Local'." + - rule: | + has(self.blockDeviceSelector) || (has(self.fileDevices) && size(self.fileDevices) > 0) + message: "At least one of 'blockDeviceSelector' or 'fileDevices' must be specified." required: - type - - blockDeviceSelector - actualVGNameOnTheNode properties: type: @@ -120,6 +122,31 @@ spec: x-kubernetes-validations: - rule: self == oldSelf message: "The actualVGNameOnTheNode field is immutable." + fileDevices: + type: array + description: | + File-backed block devices to create as LVM PVs for this Volume Group. + + The agent will create a preallocated file in the specified directory, + attach it as a loop device, and use it as a Physical Volume. + File devices and block devices can be used together in the same VG. + items: + type: object + required: + - directory + - size + properties: + directory: + type: string + description: | + Directory on the node's filesystem where the backing file will be created. + + The directory must already exist and be writable. + size: + type: string + pattern: '^[0-9]+(\.[0-9]+)?(E|P|T|G|M|k|Ei|Pi|Ti|Gi|Mi|Ki)?$' + description: | + Size of the preallocated backing file. Must be at least 1Gi. thinPools: type: array description: | @@ -312,6 +339,29 @@ spec: type: string description: | Name of the corresponding block device resource. + fileDevices: + type: array + description: | + Information about the file-backed devices used in the Volume Group on the current node. + items: + type: object + properties: + filePath: + type: string + description: | + Path to the backing file on the node. + loopDevice: + type: string + description: | + Loop device path (e.g., `/dev/loop0`). + size: + type: string + description: | + Backing file size. + pvUUID: + type: string + description: | + LVM Physical Volume UUID. subresources: status: {} additionalPrinterColumns: diff --git a/crds/lvmvolumegroupset.yaml b/crds/lvmvolumegroupset.yaml index 902cff710..23d2b7957 100644 --- a/crds/lvmvolumegroupset.yaml +++ b/crds/lvmvolumegroupset.yaml @@ -86,10 +86,13 @@ spec: type: object description: | Common template for LVMVolumeGroup resources provided by the set. + x-kubernetes-validations: + - rule: | + has(self.blockDeviceSelector) || (has(self.fileDevices) && size(self.fileDevices) > 0) + message: "At least one of 'blockDeviceSelector' or 'fileDevices' must be specified." required: - type - actualVGNameOnTheNode - - blockDeviceSelector properties: blockDeviceSelector: type: object @@ -162,6 +165,30 @@ spec: x-kubernetes-validations: - rule: self == oldSelf message: "The actualVGNameOnTheNode field is immutable." + fileDevices: + type: array + description: | + File-backed block devices to create as LVM PVs in LVMVolumeGroups. + + The agent will create a preallocated file in the specified directory, + attach it as a loop device, and use it as a Physical Volume. + + > Note that the configuration will be common for every LVMVolumeGroup created by the set. + items: + type: object + required: + - directory + - size + properties: + directory: + type: string + description: | + Directory on the node's filesystem where the backing file will be created. + size: + type: string + pattern: '^[0-9]+(\.[0-9]+)?(E|P|T|G|M|k|Ei|Pi|Ti|Gi|Mi|Ki)?$' + description: | + Size of the preallocated backing file. Must be at least 1Gi. thinPools: type: array description: | diff --git a/docs/FAQ.md b/docs/FAQ.md index dd37be7ec..bf49f9a7c 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -151,6 +151,25 @@ If the Volume Group has active Logical Volumes, perform the following steps: If necessary, the command can be added to the `cloud-init` script for automatic execution when creating virtual machines. +## How do file-backed devices (fileDevices) work? + +File-backed devices allow you to allocate part of an existing filesystem for LVM without dedicated block devices. The agent creates a preallocated file in the specified directory, attaches it as a loop device via `losetup`, and uses it as an LVM Physical Volume. + +### Limitations + +- **No resize**: File device size cannot be changed after creation. To increase capacity, add a new `fileDevices` entry. +- **Preallocated only**: Files are created with `fallocate`, which preallocates space on the filesystem. The directory must have enough free space. +- **Minimum size**: Each file device must be at least 1Gi. +- **Performance overhead**: LVM on a loop device over a filesystem adds double indirection. Use file-backed devices only when dedicated disks are not available. + +### Reboot recovery + +After a node reboot, the agent automatically re-establishes loop device mappings before activating Volume Groups. The backing file path is stored in `status.nodes[].fileDevices[].filePath`. + +### Deletion + +When an LVMVolumeGroup with file-backed devices is deleted, the agent detaches the loop devices and removes the backing files. + ## What labels are added by the controller to BlockDevice resources? - `status.blockdevice.storage.deckhouse.io/type`: LVM type. diff --git a/docs/RESOURCES.md b/docs/RESOURCES.md index 9472e16ee..185eca97f 100644 --- a/docs/RESOURCES.md +++ b/docs/RESOURCES.md @@ -195,6 +195,58 @@ The `spec.local` field is mandatory for the `Local` type. If there's a discrepan All selected block devices must belong to the same node for [LVMVolumeGroup](./cr.html#lvmvolumegroup) with type `Local`. {{< /alert >}} +#### Creating an LVMVolumeGroup with file-backed devices + +When dedicated block devices are not available, you can allocate part of an existing filesystem for LVM by using file-backed devices (`spec.fileDevices`). The agent will create a preallocated file, attach it as a loop device, and use it as an LVM Physical Volume. + +Creating a Volume Group backed by a 50Gi file on an existing filesystem: + +```yaml +apiVersion: storage.deckhouse.io/v1alpha1 +kind: LVMVolumeGroup +metadata: + name: "vg-file-on-node-0" +spec: + type: Local + local: + nodeName: "node-0" + actualVGNameOnTheNode: "vg-file" + fileDevices: + - directory: /data/lvm-backing + size: 50Gi +``` + +Combining block devices and file-backed devices in a single Volume Group: + +```yaml +apiVersion: storage.deckhouse.io/v1alpha1 +kind: LVMVolumeGroup +metadata: + name: "vg-mixed-on-node-0" +spec: + type: Local + local: + nodeName: "node-0" + blockDeviceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - dev-07ad52cef2348996b72db262011f1b5f896bb68f + actualVGNameOnTheNode: "vg-mixed" + fileDevices: + - directory: /data/lvm-backing + size: 100Gi +``` + +{{< alert level="warning" >}} +File-backed devices add overhead due to double indirection (LVM on loop device on filesystem). Use them only when dedicated block devices are not available. +{{< /alert >}} + +{{< alert level="info" >}} +The agent automatically recovers loop device mappings after a node reboot. File device resize is not supported in the current version. +{{< /alert >}} + ### Updating an LVMVolumeGroup resource To change the desired state of a Volume Group or thin pool on nodes, modify the `spec` field of the corresponding [LVMVolumeGroup](./cr.html#lvmvolumegroup) resource. The controller automatically validates the new data and, if correct, makes the necessary changes to entities on the node. diff --git a/images/agent/cmd/main.go b/images/agent/cmd/main.go index 00e3b827b..2cbe77993 100644 --- a/images/agent/cmd/main.go +++ b/images/agent/cmd/main.go @@ -146,6 +146,10 @@ func run() int { log.Error(err, "[main] unable to run ReTag") } + log.Info("[main] ReattachFileDevices starts") + reattachFileDevicesAtStartup(ctx, log, mgr, commands, cfgParams.NodeName) + log.Info("[main] ReattachFileDevices completed") + log.Info("[main] ActivateVGs starts") if err := utils.ActivateAllManagedVGs(ctx, log, commands, metrics, cfgParams.CmdDeadlineDuration); err != nil { log.Error(err, "[main] unable to activate managed VGs") @@ -319,3 +323,44 @@ func run() int { log.Info("[main] successfully starts the manager") return 0 } + +func reattachFileDevicesAtStartup(ctx context.Context, log logger.Logger, mgr manager.Manager, commands utils.Commands, nodeName string) { + var lvgList v1alpha1.LVMVolumeGroupList + if err := mgr.GetAPIReader().List(ctx, &lvgList); err != nil { + log.Error(err, "[reattachFileDevicesAtStartup] unable to list LVMVolumeGroups") + return + } + + var items []utils.LVGWithFileDevices + for _, lvg := range lvgList.Items { + if lvg.Spec.Local.NodeName != nodeName { + continue + } + for _, node := range lvg.Status.Nodes { + if node.Name != nodeName { + continue + } + if len(node.FileDevices) == 0 { + continue + } + fds := make([]utils.FileDeviceStatus, 0, len(node.FileDevices)) + for _, fd := range node.FileDevices { + fds = append(fds, utils.FileDeviceStatus{ + FilePath: fd.FilePath, + LoopDevice: fd.LoopDevice, + }) + } + items = append(items, utils.LVGWithFileDevices{ + LVGName: lvg.Name, + FileDevices: fds, + }) + } + } + + if len(items) == 0 { + log.Debug("[reattachFileDevicesAtStartup] no file devices to reattach") + return + } + + utils.ReattachFileDevices(ctx, log, commands, items) +} diff --git a/images/agent/internal/const.go b/images/agent/internal/const.go index 5d5d24814..57be6039c 100644 --- a/images/agent/internal/const.go +++ b/images/agent/internal/const.go @@ -43,8 +43,13 @@ const ( // LVMGlobalFilter is passed via `lvm --config` for every LVM // subcommand the agent runs. It rejects canonical names of block // devices that always belong to a foreign storage layer (Ceph RBD, - // DRBD, NBD, loopback) so lvm does not even read PV labels - // from them when udev integration is unavailable. + // DRBD, NBD) so lvm does not even read PV labels from them + // when udev integration is unavailable. + // + // Loop devices are intentionally NOT rejected: the agent manages + // file-backed loop devices as LVM PVs (spec.fileDevices). Unmanaged + // loop-backed VGs are safely ignored because the discoverer only + // picks up VGs tagged with storage.deckhouse.io/enabled=true. // // There is intentionally no blanket "a|.*|" accept rule. When a // device matches none of the reject patterns, LVM accepts it by @@ -57,7 +62,7 @@ const ( // The authoritative foreign-PV filter (FilterForeignPVs) still runs // after lvm returns and catches any PVs that slip through // via /dev/block/MAJ:MIN or /dev/disk/by-id/... aliases. - LVMGlobalFilter = `devices/global_filter=["r|^/dev/rbd|","r|^/dev/drbd|","r|^/dev/nbd|","r|^/dev/loop|"]` + LVMGlobalFilter = `devices/global_filter=["r|^/dev/rbd|","r|^/dev/drbd|","r|^/dev/nbd|"]` // LVMArchiveRetention caps the size of /etc/lvm/archive: keep at // most the last 10 metadata snapshots and at most 7 days of history. @@ -94,6 +99,8 @@ const ( DeletionProtectionAnnotation = "storage.deckhouse.io/deletion-protection" LVMVolumeGroupTag = "storage.deckhouse.io/lvmVolumeGroupName" LVGMetadataNameLabelKey = "kubernetes.io/metadata.name" + + FileDeviceImageSuffix = ".img" ) var ( @@ -110,11 +117,15 @@ var ( // rbd - Ceph RBD (kernel rbd module, major 251) // drbd - DRBD (sds-replicated-volume, major 147) // nbd - network block device (major 43) - // loop - loopback (major 7) — typically backs QEMU/file-based VM disks + // + // Loop devices (major 7) are NOT in this list: the agent manages + // file-backed loop devices as LVM PVs via spec.fileDevices. + // Unmanaged loop PVs are safe because the discoverer only acts + // on VGs tagged with storage.deckhouse.io/enabled=true. // // Used after lvm returns the PV list, against the canonical // path resolved via readlink -f in the host mount namespace. - ForeignDeviceBasePrefixes = []string{"rbd", "drbd", "nbd", "loop"} + ForeignDeviceBasePrefixes = []string{"rbd", "drbd", "nbd"} ) const ( diff --git a/images/agent/internal/controller/lvg/discoverer.go b/images/agent/internal/controller/lvg/discoverer.go index 008ea1013..246629dda 100644 --- a/images/agent/internal/controller/lvg/discoverer.go +++ b/images/agent/internal/controller/lvg/discoverer.go @@ -17,10 +17,12 @@ limitations under the License. package lvg import ( + "bytes" "context" "errors" "fmt" "maps" + "os/exec" "strings" "time" @@ -427,8 +429,10 @@ func (d *Discoverer) GetLVMVolumeGroupCandidates(bds map[string]v1alpha1.BlockDe VGFree: *resource.NewQuantity(vg.VGFree.Value(), resource.BinarySI), VGUUID: vg.VGUUID, ExtentSize: *resource.NewQuantity(vg.VGExtentSize.Value(), resource.BinarySI), - Nodes: d.configureCandidateNodeDevices(sortedPVs, sortedBDs, vg, d.cfg.NodeName), + Nodes: nil, + FileDeviceNodes: nil, } + candidate.Nodes, candidate.FileDeviceNodes = d.configureCandidateNodeDevices(sortedPVs, sortedBDs, vg, d.cfg.NodeName) candidates = append(candidates, candidate) } @@ -460,7 +464,7 @@ func (d *Discoverer) CreateLVMVolumeGroupByCandidate( }, Status: v1alpha1.LVMVolumeGroupStatus{ AllocatedSize: candidate.AllocatedSize, - Nodes: convertLVMVGNodes(candidate.Nodes), + Nodes: convertLVMVGNodes(candidate.Nodes, candidate.FileDeviceNodes), ThinPools: thinPools, VGSize: candidate.VGSize, VGUuid: candidate.VGUUID, @@ -545,7 +549,7 @@ func (d *Discoverer) UpdateLVMVolumeGroupByCandidate( } lvg.Status.AllocatedSize = candidate.AllocatedSize - lvg.Status.Nodes = convertLVMVGNodes(candidate.Nodes) + lvg.Status.Nodes = convertLVMVGNodes(candidate.Nodes, candidate.FileDeviceNodes) lvg.Status.ThinPools = thinPools lvg.Status.VGSize = candidate.VGSize lvg.Status.VGFree = candidate.VGFree @@ -569,21 +573,27 @@ func (d *Discoverer) UpdateLVMVolumeGroupByCandidate( return err } -func (d *Discoverer) configureCandidateNodeDevices(pvs map[string][]internal.PVData, bds map[string][]v1alpha1.BlockDevice, vg internal.VGData, currentNode string) map[string][]internal.LVMVGDevice { +func (d *Discoverer) configureCandidateNodeDevices(pvs map[string][]internal.PVData, bds map[string][]v1alpha1.BlockDevice, vg internal.VGData, currentNode string) (map[string][]internal.LVMVGDevice, map[string][]internal.LVMVGFileDevice) { filteredPV := pvs[vg.VGName+vg.VGUUID] filteredBds := bds[vg.VGName+vg.VGUUID] bdPathStatus := make(map[string]v1alpha1.BlockDevice, len(bds)) result := make(map[string][]internal.LVMVGDevice, len(filteredPV)) + fileResult := make(map[string][]internal.LVMVGFileDevice) for _, blockDevice := range filteredBds { bdPathStatus[blockDevice.Status.Path] = blockDevice } for _, pv := range filteredPV { + if strings.HasPrefix(pv.PVName, "/dev/loop") { + fileDev := d.buildFileDeviceFromLoopPV(pv) + if fileDev != nil { + fileResult[currentNode] = append(fileResult[currentNode], *fileDev) + } + continue + } + bd, exist := bdPathStatus[pv.PVName] - // this is very rare case which might occurred while VG extend operation goes. In this case, in the cache the controller - // sees a new PV included in the VG, but BlockDeviceDiscover did not update the corresponding BlockDevice resource on time, - // so the BlockDevice resource does not have any info, that it is in the VG. if !exist { d.log.Warning(fmt.Sprintf("[configureCandidateNodeDevices] no BlockDevice resource is yet configured for PV %s in VG %s, retry on the next iteration", pv.PVName, vg.VGName)) continue @@ -601,7 +611,44 @@ func (d *Discoverer) configureCandidateNodeDevices(pvs map[string][]internal.PVD result[currentNode] = append(result[currentNode], device) } - return result + return result, fileResult +} + +func (d *Discoverer) buildFileDeviceFromLoopPV(pv internal.PVData) *internal.LVMVGFileDevice { + loopBase := strings.TrimPrefix(pv.PVName, "/dev/") + sysPath := fmt.Sprintf("/sys/block/%s/loop/backing_file", loopBase) + + cmd, backingFile, err := d.readSysFile(sysPath) + d.log.Debug(cmd) + if err != nil { + d.log.Warning(fmt.Sprintf("[buildFileDeviceFromLoopPV] unable to read backing_file for %s: %v", pv.PVName, err)) + return nil + } + + if backingFile == "" { + return nil + } + + return &internal.LVMVGFileDevice{ + FilePath: backingFile, + LoopDevice: pv.PVName, + Size: *resource.NewQuantity(pv.PVSize.Value(), resource.BinarySI), + PVUUID: pv.PVUuid, + } +} + +func (d *Discoverer) readSysFile(path string) (string, string, error) { + args := []string{"-t", "1", "-m", "-u", "-i", "-n", "-p", "--", "/bin/cat", path} + cmd := exec.Command(internal.NSENTERCmd, args...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return cmd.String(), "", fmt.Errorf("unable to read %s: %w, stderr: %s", path, err, stderr.String()) + } + return cmd.String(), strings.TrimSpace(stdout.String()), nil } func checkVGHealth(vgIssues map[string]string, pvIssues map[string][]string, lvIssues map[string]map[string]string, vg internal.VGData) (health, message string) { @@ -879,7 +926,7 @@ func hasLVMVolumeGroupDiff(log logger.Logger, lvg v1alpha1.LVMVolumeGroup, candi } log.Trace(fmt.Sprintf(`VGSize, candidate: %s, lvg: %s`, candidate.VGSize.String(), lvg.Status.VGSize.String())) log.Trace(fmt.Sprintf(`VGUUID, candidate: %s, lvg: %s`, candidate.VGUUID, lvg.Status.VGUuid)) - log.Trace(fmt.Sprintf(`Nodes, candidate: %+v, lvg: %+v`, convertLVMVGNodes(candidate.Nodes), lvg.Status.Nodes)) + log.Trace(fmt.Sprintf(`Nodes, candidate: %+v, lvg: %+v`, convertLVMVGNodes(candidate.Nodes, candidate.FileDeviceNodes), lvg.Status.Nodes)) return candidate.AllocatedSize.Value() != lvg.Status.AllocatedSize.Value() || hasStatusPoolDiff(convertedStatusPools, lvg.Status.ThinPools) || @@ -887,7 +934,7 @@ func hasLVMVolumeGroupDiff(log logger.Logger, lvg v1alpha1.LVMVolumeGroup, candi candidate.VGFree.Value() != lvg.Status.VGFree.Value() || candidate.VGUUID != lvg.Status.VGUuid || candidate.ExtentSize.Value() != lvg.Status.ExtentSize.Value() || - hasStatusNodesDiff(log, convertLVMVGNodes(candidate.Nodes), lvg.Status.Nodes) + hasStatusNodesDiff(log, convertLVMVGNodes(candidate.Nodes, candidate.FileDeviceNodes), lvg.Status.Nodes) } func hasStatusNodesDiff(log logger.Logger, first, second []v1alpha1.LVMVolumeGroupNode) bool { @@ -964,19 +1011,43 @@ func configureBlockDeviceSelector(candidate internal.LVMVolumeGroupCandidate) *m } } -func convertLVMVGNodes(nodes map[string][]internal.LVMVGDevice) []v1alpha1.LVMVolumeGroupNode { - lvmvgNodes := make([]v1alpha1.LVMVolumeGroupNode, 0, len(nodes)) +func convertLVMVGNodes(nodes map[string][]internal.LVMVGDevice, fileNodes map[string][]internal.LVMVGFileDevice) []v1alpha1.LVMVolumeGroupNode { + allNodeNames := make(map[string]struct{}, len(nodes)) + for n := range nodes { + allNodeNames[n] = struct{}{} + } + for n := range fileNodes { + allNodeNames[n] = struct{}{} + } - for nodeName, nodeDevices := range nodes { - lvmvgNodes = append(lvmvgNodes, v1alpha1.LVMVolumeGroupNode{ - Devices: convertLVMVGDevices(nodeDevices), + lvmvgNodes := make([]v1alpha1.LVMVolumeGroupNode, 0, len(allNodeNames)) + for nodeName := range allNodeNames { + node := v1alpha1.LVMVolumeGroupNode{ + Devices: convertLVMVGDevices(nodes[nodeName]), Name: nodeName, - }) + } + if fds := fileNodes[nodeName]; len(fds) > 0 { + node.FileDevices = convertLVMVGFileDevices(fds) + } + lvmvgNodes = append(lvmvgNodes, node) } return lvmvgNodes } +func convertLVMVGFileDevices(devices []internal.LVMVGFileDevice) []v1alpha1.LVMVolumeGroupFileDevice { + result := make([]v1alpha1.LVMVolumeGroupFileDevice, 0, len(devices)) + for _, dev := range devices { + result = append(result, v1alpha1.LVMVolumeGroupFileDevice{ + FilePath: dev.FilePath, + LoopDevice: dev.LoopDevice, + Size: dev.Size, + PVUuid: dev.PVUUID, + }) + } + return result +} + func convertLVMVGDevices(devices []internal.LVMVGDevice) []v1alpha1.LVMVolumeGroupDevice { convertedDevices := make([]v1alpha1.LVMVolumeGroupDevice, 0, len(devices)) diff --git a/images/agent/internal/controller/lvg/discoverer_test.go b/images/agent/internal/controller/lvg/discoverer_test.go index 81a4b4c1e..f230e1103 100644 --- a/images/agent/internal/controller/lvg/discoverer_test.go +++ b/images/agent/internal/controller/lvg/discoverer_test.go @@ -334,7 +334,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { mp := map[string][]v1alpha1.BlockDevice{vgName + vgUUID: bds} ar := map[string][]internal.PVData{vgName + vgUUID: pvs} - actual := setupDiscoverer(nil).configureCandidateNodeDevices(ar, mp, vg, nodeName) + actual, _ := setupDiscoverer(nil).configureCandidateNodeDevices(ar, mp, vg, nodeName) assert.Equal(t, expected, actual) }) @@ -501,7 +501,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { }, Status: v1alpha1.LVMVolumeGroupStatus{ AllocatedSize: size10G, - Nodes: convertLVMVGNodes(nodes), + Nodes: convertLVMVGNodes(nodes, nil), ThinPools: thinPools, VGSize: size10G, VGUuid: VGUUID, @@ -857,7 +857,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { }, Status: v1alpha1.LVMVolumeGroupStatus{ AllocatedSize: resource.MustParse("9765625Ki"), - Nodes: convertLVMVGNodes(nodes), + Nodes: convertLVMVGNodes(nodes, nil), ThinPools: thinPools, VGSize: resource.MustParse("9765625Ki"), }, @@ -922,7 +922,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { lvmVolumeGroup := v1alpha1.LVMVolumeGroup{ Status: v1alpha1.LVMVolumeGroupStatus{ AllocatedSize: allocatedSize, - Nodes: convertLVMVGNodes(nodes), + Nodes: convertLVMVGNodes(nodes, nil), ThinPools: thinPools, VGSize: vgSize, VGFree: *resource.NewQuantity(vgFree.Value()+10000, resource.BinarySI), diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 8da45ab30..a47efd788 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -361,6 +361,8 @@ func (r *Reconciler) reconcileLVGDeleteFunc(ctx context.Context, lvg *v1alpha1.L return true, err } + r.cleanupFileDevices(lvg) + removed, err := r.removeLVGFinalizerIfExist(ctx, lvg) if err != nil { r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to remove a finalizer %s from the LVMVolumeGroup %s", internal.SdsNodeConfiguratorFinalizer, lvg.Name)) @@ -771,6 +773,10 @@ func (r *Reconciler) validateLVGForCreateFunc( r.log.Debug(fmt.Sprintf("[validateLVGForCreateFunc] all BlockDevices of the LVMVolumeGroup %s are consumable", lvg.Name)) } + for i, fd := range lvg.Spec.FileDevices { + r.validateFileDevice(fd, i, &reason, &totalVGSize) + } + if lvg.Spec.ThinPools != nil { r.log.Debug(fmt.Sprintf("[validateLVGForCreateFunc] the LVMVolumeGroup %s has thin-pools. Validate if VG size has enough space for the thin-pools", lvg.Name)) r.log.Trace(fmt.Sprintf("[validateLVGForCreateFunc] the LVMVolumeGroup %s has thin-pools %v", lvg.Name, lvg.Spec.ThinPools)) @@ -818,6 +824,21 @@ func (r *Reconciler) validateLVGForCreateFunc( return true, "" } +var minFileDeviceSize = resource.MustParse("1Gi") + +func (r *Reconciler) validateFileDevice(fd v1alpha1.LVMVolumeGroupFileDeviceSpec, index int, reason *strings.Builder, totalVGSize *resource.Quantity) { + if fd.Directory == "" { + reason.WriteString(fmt.Sprintf("fileDevices[%d].directory is empty. ", index)) + } + + if fd.Size.Value() < minFileDeviceSize.Value() { + reason.WriteString(fmt.Sprintf("fileDevices[%d].size %s is less than minimum %s. ", index, fd.Size.String(), minFileDeviceSize.String())) + return + } + + totalVGSize.Add(fd.Size) +} + func (r *Reconciler) validateLVGForUpdateFunc( lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice, @@ -1351,9 +1372,68 @@ func (r *Reconciler) triggerUdevForPaths(parent context.Context, paths []string) } } +func (r *Reconciler) provisionFileDevices(lvg *v1alpha1.LVMVolumeGroup) ([]string, error) { + if len(lvg.Spec.FileDevices) == 0 { + return nil, nil + } + + loopPaths := make([]string, 0, len(lvg.Spec.FileDevices)) + for i, fd := range lvg.Spec.FileDevices { + filePath := fmt.Sprintf("%s/sds-%s-%d%s", fd.Directory, lvg.Name, i, internal.FileDeviceImageSuffix) + sizeBytes := fd.Size.Value() + + r.log.Info(fmt.Sprintf("[provisionFileDevices] creating file device %s (%d bytes) for LVMVolumeGroup %s", filePath, sizeBytes, lvg.Name)) + + cmd, err := r.commands.CreateFileDevice(filePath, sizeBytes) + r.log.Debug(cmd) + if err != nil { + r.log.Error(err, fmt.Sprintf("[provisionFileDevices] unable to create file %s", filePath)) + return nil, err + } + + cmd, loopDev, err := r.commands.SetupLoopDevice(filePath) + r.log.Debug(cmd) + if err != nil { + r.log.Error(err, fmt.Sprintf("[provisionFileDevices] unable to setup loop device for %s", filePath)) + return nil, err + } + + r.log.Info(fmt.Sprintf("[provisionFileDevices] file %s attached to %s", filePath, loopDev)) + loopPaths = append(loopPaths, loopDev) + } + return loopPaths, nil +} + +func (r *Reconciler) cleanupFileDevices(lvg *v1alpha1.LVMVolumeGroup) { + for _, node := range lvg.Status.Nodes { + for _, fd := range node.FileDevices { + if fd.LoopDevice != "" { + cmd, err := r.commands.DetachLoopDevice(fd.LoopDevice) + r.log.Debug(cmd) + if err != nil { + r.log.Warning(fmt.Sprintf("[cleanupFileDevices] unable to detach loop %s: %v", fd.LoopDevice, err)) + } + } + if fd.FilePath != "" { + cmd, err := r.commands.RemoveFileDevice(fd.FilePath) + r.log.Debug(cmd) + if err != nil { + r.log.Warning(fmt.Sprintf("[cleanupFileDevices] unable to remove file %s: %v", fd.FilePath, err)) + } + } + } + } +} + func (r *Reconciler) createVGComplex(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) error { paths := extractPathsFromBlockDevices(nil, blockDevices) + loopPaths, err := r.provisionFileDevices(lvg) + if err != nil { + return fmt.Errorf("file device provisioning failed: %w", err) + } + paths = append(paths, loopPaths...) + r.log.Trace(fmt.Sprintf("[CreateVGComplex] LVMVolumeGroup %s devices paths %v", lvg.Name, paths)) for _, path := range paths { start := time.Now() diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index c85807442..103b3a091 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -19,6 +19,7 @@ package lvg import ( "bytes" "context" + "strings" "testing" "time" @@ -1482,3 +1483,57 @@ func setupReconciler() *Reconciler { utils.NewCommands(), ReconcilerConfig{NodeName: "test_node"}) } + +func TestValidateFileDevice(t *testing.T) { + r := setupReconciler() + + t.Run("valid_file_device", func(t *testing.T) { + var reason strings.Builder + totalSize := resource.MustParse("0") + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ + Directory: "/data", + Size: resource.MustParse("10Gi"), + } + r.validateFileDevice(fd, 0, &reason, &totalSize) + assert.Empty(t, reason.String()) + want := resource.MustParse("10Gi") + assert.Equal(t, want.Value(), totalSize.Value()) + }) + + t.Run("empty_directory", func(t *testing.T) { + var reason strings.Builder + totalSize := resource.MustParse("0") + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ + Directory: "", + Size: resource.MustParse("10Gi"), + } + r.validateFileDevice(fd, 0, &reason, &totalSize) + assert.Contains(t, reason.String(), "directory is empty") + }) + + t.Run("size_too_small", func(t *testing.T) { + var reason strings.Builder + totalSize := resource.MustParse("0") + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ + Directory: "/data", + Size: resource.MustParse("500Mi"), + } + r.validateFileDevice(fd, 0, &reason, &totalSize) + assert.Contains(t, reason.String(), "less than minimum") + }) + + t.Run("size_adds_to_total_vg_size", func(t *testing.T) { + var reason strings.Builder + totalSize := resource.MustParse("5Gi") + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ + Directory: "/data", + Size: resource.MustParse("10Gi"), + } + r.validateFileDevice(fd, 0, &reason, &totalSize) + assert.Empty(t, reason.String()) + expected := resource.MustParse("5Gi") + add := resource.MustParse("10Gi") + expected.Add(add) + assert.Equal(t, expected.Value(), totalSize.Value()) + }) +} diff --git a/images/agent/internal/mock_utils/commands.go b/images/agent/internal/mock_utils/commands.go index 469e3c550..63300047c 100644 --- a/images/agent/internal/mock_utils/commands.go +++ b/images/agent/internal/mock_utils/commands.go @@ -552,3 +552,80 @@ func (mr *MockCommandsMockRecorder) VGScan(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VGScan", reflect.TypeOf((*MockCommands)(nil).VGScan), ctx) } + +// CreateFileDevice mocks base method. +func (m *MockCommands) CreateFileDevice(path string, sizeBytes int64) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFileDevice", path, sizeBytes) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFileDevice indicates an expected call of CreateFileDevice. +func (mr *MockCommandsMockRecorder) CreateFileDevice(path, sizeBytes any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFileDevice", reflect.TypeOf((*MockCommands)(nil).CreateFileDevice), path, sizeBytes) +} + +// SetupLoopDevice mocks base method. +func (m *MockCommands) SetupLoopDevice(filePath string) (string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetupLoopDevice", filePath) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// SetupLoopDevice indicates an expected call of SetupLoopDevice. +func (mr *MockCommandsMockRecorder) SetupLoopDevice(filePath any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupLoopDevice", reflect.TypeOf((*MockCommands)(nil).SetupLoopDevice), filePath) +} + +// DetachLoopDevice mocks base method. +func (m *MockCommands) DetachLoopDevice(loopDev string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DetachLoopDevice", loopDev) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DetachLoopDevice indicates an expected call of DetachLoopDevice. +func (mr *MockCommandsMockRecorder) DetachLoopDevice(loopDev any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DetachLoopDevice", reflect.TypeOf((*MockCommands)(nil).DetachLoopDevice), loopDev) +} + +// FindLoopDeviceByFile mocks base method. +func (m *MockCommands) FindLoopDeviceByFile(filePath string) (string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindLoopDeviceByFile", filePath) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// FindLoopDeviceByFile indicates an expected call of FindLoopDeviceByFile. +func (mr *MockCommandsMockRecorder) FindLoopDeviceByFile(filePath any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindLoopDeviceByFile", reflect.TypeOf((*MockCommands)(nil).FindLoopDeviceByFile), filePath) +} + +// RemoveFileDevice mocks base method. +func (m *MockCommands) RemoveFileDevice(path string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveFileDevice", path) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveFileDevice indicates an expected call of RemoveFileDevice. +func (mr *MockCommandsMockRecorder) RemoveFileDevice(path any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveFileDevice", reflect.TypeOf((*MockCommands)(nil).RemoveFileDevice), path) +} diff --git a/images/agent/internal/type.go b/images/agent/internal/type.go index fbbed3bb0..56da0b33f 100644 --- a/images/agent/internal/type.go +++ b/images/agent/internal/type.go @@ -38,6 +38,7 @@ type LVMVolumeGroupCandidate struct { VGUUID string ExtentSize resource.Quantity Nodes map[string][]LVMVGDevice + FileDeviceNodes map[string][]LVMVGFileDevice } type LVMVGStatusThinPool struct { @@ -57,6 +58,13 @@ type LVMVGDevice struct { BlockDevice string } +type LVMVGFileDevice struct { + FilePath string + LoopDevice string + Size resource.Quantity + PVUUID string +} + type Devices struct { BlockDevices []Device `json:"blockdevices"` } diff --git a/images/agent/internal/utils/activation.go b/images/agent/internal/utils/activation.go index 376a85aeb..567b2fa2f 100644 --- a/images/agent/internal/utils/activation.go +++ b/images/agent/internal/utils/activation.go @@ -74,6 +74,50 @@ func runWithTimeout[T any](ctx context.Context, timeout time.Duration, fn func(c return fn(ctx) } +// ReattachFileDevices re-establishes loop device mappings for file-backed +// PVs after a node reboot. It iterates status.nodes[].fileDevices of every +// LVMVolumeGroup belonging to this node and runs `losetup --find --show` +// for files whose loop device is not attached yet. +// Must be called BEFORE ActivateAllManagedVGs so that LVM can see the PVs. +func ReattachFileDevices(ctx context.Context, log logger.Logger, commands Commands, lvgs []LVGWithFileDevices) { + for _, item := range lvgs { + for _, fd := range item.FileDevices { + if fd.FilePath == "" { + continue + } + cmd, existing, err := commands.FindLoopDeviceByFile(fd.FilePath) + log.Debug(cmd) + if err != nil { + log.Warning(fmt.Sprintf("[ReattachFileDevices] unable to query loop for %s: %v", fd.FilePath, err)) + continue + } + if existing != "" { + log.Debug(fmt.Sprintf("[ReattachFileDevices] %s already attached to %s", fd.FilePath, existing)) + continue + } + cmd, loopDev, err := commands.SetupLoopDevice(fd.FilePath) + log.Debug(cmd) + if err != nil { + log.Error(err, fmt.Sprintf("[ReattachFileDevices] unable to reattach %s", fd.FilePath)) + continue + } + log.Info(fmt.Sprintf("[ReattachFileDevices] reattached %s → %s (LVG %s)", fd.FilePath, loopDev, item.LVGName)) + } + } +} + +// LVGWithFileDevices carries the minimal data needed by ReattachFileDevices. +type LVGWithFileDevices struct { + LVGName string + FileDevices []FileDeviceStatus +} + +// FileDeviceStatus is the status-level record of a single file device. +type FileDeviceStatus struct { + FilePath string + LoopDevice string +} + func ActivateAllManagedVGs(ctx context.Context, log logger.Logger, commands Commands, metrics *monitoring.Metrics, cmdTimeout time.Duration) error { log.Info("[ActivateVGs] refreshing LVM metadata cache") if cmd, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { diff --git a/images/agent/internal/utils/commands.go b/images/agent/internal/utils/commands.go index e90a6bbab..265cdf5cc 100644 --- a/images/agent/internal/utils/commands.go +++ b/images/agent/internal/utils/commands.go @@ -70,6 +70,12 @@ type Commands interface { UdevadmTrigger(ctx context.Context, paths []string) (string, error) UnmarshalDevices(out []byte) ([]internal.Device, error) ReTag(ctx context.Context, log logger.Logger, metrics *monitoring.Metrics, ctrlName string, cmdTimeout time.Duration) error + + CreateFileDevice(path string, sizeBytes int64) (string, error) + SetupLoopDevice(filePath string) (string, string, error) + DetachLoopDevice(loopDev string) (string, error) + FindLoopDeviceByFile(filePath string) (string, string, error) + RemoveFileDevice(path string) (string, error) } type commands struct { @@ -784,6 +790,79 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito return nil } +func (commands) CreateFileDevice(path string, sizeBytes int64) (string, error) { + args := nsentrerExpendedArgs("/usr/bin/fallocate", "-l", fmt.Sprintf("%d", sizeBytes), path) + cmd := exec.Command(internal.NSENTERCmd, args...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return cmd.String(), fmt.Errorf("unable to create file device %s: %w, stderr: %s", path, err, stderr.String()) + } + return cmd.String(), nil +} + +func (commands) SetupLoopDevice(filePath string) (string, string, error) { + args := nsentrerExpendedArgs("/sbin/losetup", "--find", "--show", filePath) + cmd := exec.Command(internal.NSENTERCmd, args...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return cmd.String(), "", fmt.Errorf("unable to setup loop device for %s: %w, stderr: %s", filePath, err, stderr.String()) + } + return cmd.String(), strings.TrimSpace(stdout.String()), nil +} + +func (commands) DetachLoopDevice(loopDev string) (string, error) { + args := nsentrerExpendedArgs("/sbin/losetup", "-d", loopDev) + cmd := exec.Command(internal.NSENTERCmd, args...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return cmd.String(), fmt.Errorf("unable to detach loop device %s: %w, stderr: %s", loopDev, err, stderr.String()) + } + return cmd.String(), nil +} + +func (commands) FindLoopDeviceByFile(filePath string) (string, string, error) { + args := nsentrerExpendedArgs("/sbin/losetup", "-j", filePath) + cmd := exec.Command(internal.NSENTERCmd, args...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return cmd.String(), "", fmt.Errorf("unable to find loop device for %s: %w, stderr: %s", filePath, err, stderr.String()) + } + + output := strings.TrimSpace(stdout.String()) + if output == "" { + return cmd.String(), "", nil + } + parts := strings.SplitN(output, ":", 2) + return cmd.String(), strings.TrimSpace(parts[0]), nil +} + +func (commands) RemoveFileDevice(path string) (string, error) { + args := nsentrerExpendedArgs("/bin/rm", "-f", path) + cmd := exec.Command(internal.NSENTERCmd, args...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return cmd.String(), fmt.Errorf("unable to remove file device %s: %w, stderr: %s", path, err, stderr.String()) + } + return cmd.String(), nil +} + func unmarshalPVs(out []byte) ([]internal.PVData, error) { var pvR internal.PVReport diff --git a/images/agent/internal/utils/commands_test.go b/images/agent/internal/utils/commands_test.go index 5123580ba..a2ddb19eb 100644 --- a/images/agent/internal/utils/commands_test.go +++ b/images/agent/internal/utils/commands_test.go @@ -402,8 +402,8 @@ func TestLvmStaticExtendedArgs(t *testing.T) { } t.Run("config_value_contains_all_foreign_prefixes_and_retention", func(t *testing.T) { - // Defensive cross-check: --config must reject all four foreign - // device prefixes the post-filter knows about (rbd/drbd/nbd/loop) + // Defensive cross-check: --config must reject all foreign + // device prefixes the post-filter knows about (rbd/drbd/nbd) // and must cap /etc/lvm/archive growth. If either drops out due // to a refactor, this assertion catches it before the next scan // loop silently regresses. diff --git a/images/agent/internal/utils/devicefilter_test.go b/images/agent/internal/utils/devicefilter_test.go index b54167ff5..3feca585e 100644 --- a/images/agent/internal/utils/devicefilter_test.go +++ b/images/agent/internal/utils/devicefilter_test.go @@ -38,9 +38,9 @@ func TestIsForeignDeviceBase(t *testing.T) { {"rbd with partition", "rbd14p1", true}, {"drbd canonical", "drbd0", true}, {"nbd canonical", "nbd5", true}, - {"loop canonical", "loop970", true}, - {"loop with high index", "loop1234", true}, + {"loop canonical", "loop970", false}, + {"loop with high index", "loop1234", false}, {"nvme canonical", "nvme4n1p1", false}, {"sda canonical", "sda1", false}, {"md raid", "md1", false}, @@ -70,8 +70,8 @@ func TestFilterForeignPVs(t *testing.T) { {PVName: "/dev/md1", VGName: "vg0", VGUuid: "IZMRUl"}, {PVName: "/dev/block/251:144", VGName: "vg-1", VGUuid: "Czf0Sf"}, // -> rbd9 {PVName: "/dev/block/251:16", VGName: "vg-thin-data", VGUuid: "zR5ouf"}, // -> rbd1 - {PVName: "/dev/disk/by-id/lvm-pv-uuid-9Uuprg-IFQM-5c2y", VGName: "vg-1", VGUuid: "kHmPc6"}, // -> loop808 - {PVName: "/dev/disk/by-id/lvm-pv-uuid-xl1soD-zQZM-BRAt", VGName: "vg-thin-data", VGUuid: "He4J"}, // -> loop485 + {PVName: "/dev/disk/by-id/lvm-pv-uuid-9Uuprg-IFQM-5c2y", VGName: "vg-1", VGUuid: "kHmPc6"}, // -> loop808 (kept — loops are managed) + {PVName: "/dev/disk/by-id/lvm-pv-uuid-xl1soD-zQZM-BRAt", VGName: "vg-thin-data", VGUuid: "He4J"}, // -> loop485 (kept — loops are managed) } // Resolver mimics the readlink -f result observed on d8-virt-node-0 @@ -94,7 +94,11 @@ func TestFilterForeignPVs(t *testing.T) { } got := FilterForeignPVs(context.Background(), log, resolver, pvs, 0) - wantNames := []string{"/dev/nvme4n1p1", "/dev/nvme4n1p2", "/dev/md1"} + wantNames := []string{ + "/dev/nvme4n1p1", "/dev/nvme4n1p2", "/dev/md1", + "/dev/disk/by-id/lvm-pv-uuid-9Uuprg-IFQM-5c2y", + "/dev/disk/by-id/lvm-pv-uuid-xl1soD-zQZM-BRAt", + } gotNames := make([]string, 0, len(got)) for _, pv := range got { gotNames = append(gotNames, pv.PVName) diff --git a/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml b/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml index 77f785c29..9d3cf6017 100644 --- a/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml +++ b/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml @@ -22,9 +22,11 @@ spec: # See the License for the specific language governing permissions and # limitations under the License. - # Loop devices should not be queried by the LVM and multipath commands. - # So we add loop devices into blacklist for multipath and configure - # global_filter in lvm.conf for them + # Loop devices should not be queried by multipath — they are never + # multipath targets. LVM filtering for loop devices is intentionally + # NOT configured here: the agent manages file-backed loop devices as + # LVM PVs (spec.fileDevices) and passes its own --config global_filter + # for every LVM command. bb-event-on 'bb-sync-file-changed' '_on_multipath_config_changed' _on_multipath_config_changed() { @@ -33,23 +35,6 @@ spec: fi } - configure_lvm() { - command -V lvmconfig >/dev/null 2>&1 || return 0 - test -f /etc/lvm/lvm.conf || return 0 - current_global_filter=$(lvmconfig devices/global_filter 2>/dev/null || true) - - case "${current_global_filter}" in - '' ) new_global_filter='["r|^/dev/loop[0-9]+|"]' ;; - */dev/loop*) return 0 ;; - 'global_filter="'*) new_global_filter='["r|^/dev/loop[0-9]+|",'${current_global_filter#*=}] ;; - 'global_filter=['*) new_global_filter='["r|^/dev/loop[0-9]+|",'${current_global_filter#*[} ;; - *) echo error parsing global_filter >&2; return 1 ;; - esac - - lvmconfig --config "devices/global_filter=$new_global_filter" --withcomments --merge > /etc/lvm/lvm.conf.$$ - mv /etc/lvm/lvm.conf.$$ /etc/lvm/lvm.conf - } - configure_multipath() { mkdir -p /etc/multipath/conf.d bb-sync-file /etc/multipath/conf.d/loop-blacklist.conf - < Date: Tue, 23 Jun 2026 18:21:34 +0300 Subject: [PATCH 02/32] fix(agent): idempotent fileDevices provisioning with rollback and owner marker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit provisionFileDevices used to allocate a fresh loop minor on every retry because `losetup --find --show` does not deduplicate, leaking loop devices until the system-wide limit. It also left half-attached files on the disk whenever any later step (CreatePV, vgcreate) failed because there was no compensating cleanup. cleanupFileDevices walked status.nodes[].fileDevices only, so a crash between fallocate and the next discoverer pass would leak the backing files forever. It also swallowed losetup errors, allowing the finalizer to be removed while a busy loop device was still alive. The basename was tied to the slice index (`sds--.img`), so any reorder/insert/delete in `spec.fileDevices` silently retargeted an existing entry to a different file. This commit rewrites both routines on top of two helpers (BuildFileDevicePath / IsManagedFileDevicePath) that derive the basename from a SHA-256 prefix of (directory, size). The pattern doubles as the owner marker: the cleanup refuses to rm any path whose basename does not match, which protects unrelated host files (libvirt qcow2, snap loops, …) from being clobbered if a foreign loop PV ever slips into status. Concrete changes: - provisionFileDevices now queries FindLoopDeviceByFile first and reuses the existing loop device if there is one; otherwise it tracks every file it creates and every loop it attaches, and rolls them back in LIFO order when any step in the loop fails. - cleanupFileDevices walks the union of spec.fileDevices and status.nodes[].fileDevices, returns an error when any losetup -d fails (so the finalizer stays in place and the reconcile retries), and refuses to act on paths that do not match the managed pattern. - validateFileDevice rejects relative directories and `..` segments — defense in depth against runaway fallocate in arbitrary locations. - reconcileLVGDeleteFunc now propagates the cleanup error, sets TypeVGConfigurationApplied=False/Terminating, and keeps the finalizer until the next pass succeeds. Agents: Claude Opus 4.7 (ведущий, авто-режим) --- images/agent/internal/const.go | 12 ++ .../internal/controller/lvg/reconciler.go | 181 ++++++++++++++++-- images/agent/internal/utils/filedevices.go | 101 ++++++++++ 3 files changed, 275 insertions(+), 19 deletions(-) create mode 100644 images/agent/internal/utils/filedevices.go diff --git a/images/agent/internal/const.go b/images/agent/internal/const.go index 57be6039c..f078b1f54 100644 --- a/images/agent/internal/const.go +++ b/images/agent/internal/const.go @@ -100,7 +100,19 @@ const ( LVMVolumeGroupTag = "storage.deckhouse.io/lvmVolumeGroupName" LVGMetadataNameLabelKey = "kubernetes.io/metadata.name" + // FileDeviceImageSuffix is the trailing component the agent appends + // to every backing file it creates for spec.fileDevices entries. FileDeviceImageSuffix = ".img" + + // FileDevicePrefix anchors the basename of every backing file the + // agent creates. The full pattern is `sds--.img` and + // is the sole owner marker: the discoverer treats a loop PV as a + // managed file device only when its backing file's basename matches + // this pattern (see utils.IsManagedFileDevicePath). Reusing the LVG + // name in the basename means a foreign loop device backed by an + // unrelated file (a libvirt qcow2, a snap, …) is never misidentified + // as ours during discovery or cleanup. + FileDevicePrefix = "sds-" ) var ( diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index a47efd788..7ead63725 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "reflect" "slices" "strings" @@ -361,7 +362,14 @@ func (r *Reconciler) reconcileLVGDeleteFunc(ctx context.Context, lvg *v1alpha1.L return true, err } - r.cleanupFileDevices(lvg) + if err := r.cleanupFileDevices(lvg); err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to clean up file devices for the LVMVolumeGroup %s", lvg.Name)) + condErr := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, err.Error()) + if condErr != nil { + r.log.Error(condErr, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + return true, err + } removed, err := r.removeLVGFinalizerIfExist(ctx, lvg) if err != nil { @@ -829,6 +837,21 @@ var minFileDeviceSize = resource.MustParse("1Gi") func (r *Reconciler) validateFileDevice(fd v1alpha1.LVMVolumeGroupFileDeviceSpec, index int, reason *strings.Builder, totalVGSize *resource.Quantity) { if fd.Directory == "" { reason.WriteString(fmt.Sprintf("fileDevices[%d].directory is empty. ", index)) + return + } + + // The agent runs in PID 1's mount namespace; an absolute path with + // no `..` segment is the minimum sanity check that keeps `fallocate` + // from creating runaway files outside whatever directory the cluster + // admin intended. + cleaned := filepath.Clean(fd.Directory) + if !filepath.IsAbs(cleaned) { + reason.WriteString(fmt.Sprintf("fileDevices[%d].directory %q must be an absolute path. ", index, fd.Directory)) + return + } + if cleaned != fd.Directory && strings.Contains(fd.Directory, "..") { + reason.WriteString(fmt.Sprintf("fileDevices[%d].directory %q must not contain '..' segments. ", index, fd.Directory)) + return } if fd.Size.Value() < minFileDeviceSize.Value() { @@ -1372,31 +1395,94 @@ func (r *Reconciler) triggerUdevForPaths(parent context.Context, paths []string) } } -func (r *Reconciler) provisionFileDevices(lvg *v1alpha1.LVMVolumeGroup) ([]string, error) { +// provisionFileDevices creates one preallocated backing file per +// spec.fileDevices entry and attaches each as a loop device. It is +// idempotent across reconcile retries: if a loop device is already +// attached to the target file, it reuses that loop device instead of +// creating a fresh one (`losetup --find --show` would otherwise hand +// out a new minor on every call, slowly leaking up to the system-wide +// loop limit). +// +// On any error mid-way the function rolls back everything it created +// in *this* invocation — detaches loop devices it just attached and +// removes files it just created — so a transient failure does not +// leave dangling files in the data directory. Loop devices and files +// that pre-existed (e.g. left behind by an earlier successful step) +// are preserved on purpose: they will be picked up by the next +// reconcile as "already attached". +func (r *Reconciler) provisionFileDevices(lvg *v1alpha1.LVMVolumeGroup) (loopPaths []string, retErr error) { if len(lvg.Spec.FileDevices) == 0 { return nil, nil } - loopPaths := make([]string, 0, len(lvg.Spec.FileDevices)) - for i, fd := range lvg.Spec.FileDevices { - filePath := fmt.Sprintf("%s/sds-%s-%d%s", fd.Directory, lvg.Name, i, internal.FileDeviceImageSuffix) + // Track resources we (and only we) just created so we can undo them + // on failure. We deliberately do NOT roll back pre-existing artifacts. + type rollback struct { + filePath string + createdFile bool + loopDev string + attachedLoopOK bool + } + created := make([]rollback, 0, len(lvg.Spec.FileDevices)) + defer func() { + if retErr == nil { + return + } + for i := len(created) - 1; i >= 0; i-- { + rb := created[i] + if rb.attachedLoopOK && rb.loopDev != "" { + if cmd, err := r.commands.DetachLoopDevice(rb.loopDev); err != nil { + r.log.Warning(fmt.Sprintf("[provisionFileDevices][rollback] unable to detach %s: %v (cmd: %s)", rb.loopDev, err, cmd)) + } + } + if rb.createdFile && rb.filePath != "" { + if cmd, err := r.commands.RemoveFileDevice(rb.filePath); err != nil { + r.log.Warning(fmt.Sprintf("[provisionFileDevices][rollback] unable to remove %s: %v (cmd: %s)", rb.filePath, err, cmd)) + } + } + } + }() + + loopPaths = make([]string, 0, len(lvg.Spec.FileDevices)) + for _, fd := range lvg.Spec.FileDevices { + filePath := utils.BuildFileDevicePath(fd.Directory, lvg.Name, fd.Size) sizeBytes := fd.Size.Value() + // Already attached? Reuse — calling losetup --find --show again + // would allocate a fresh loop minor and orphan the previous one. + cmd, existing, err := r.commands.FindLoopDeviceByFile(filePath) + r.log.Debug(cmd) + if err != nil { + return nil, fmt.Errorf("query loop for %s: %w", filePath, err) + } + if existing != "" { + r.log.Info(fmt.Sprintf("[provisionFileDevices] %s already attached to %s; reusing", filePath, existing)) + loopPaths = append(loopPaths, existing) + continue + } + r.log.Info(fmt.Sprintf("[provisionFileDevices] creating file device %s (%d bytes) for LVMVolumeGroup %s", filePath, sizeBytes, lvg.Name)) - cmd, err := r.commands.CreateFileDevice(filePath, sizeBytes) + rb := rollback{filePath: filePath} + cmd, err = r.commands.CreateFileDevice(filePath, sizeBytes) r.log.Debug(cmd) if err != nil { r.log.Error(err, fmt.Sprintf("[provisionFileDevices] unable to create file %s", filePath)) + created = append(created, rb) return nil, err } + rb.createdFile = true cmd, loopDev, err := r.commands.SetupLoopDevice(filePath) r.log.Debug(cmd) if err != nil { r.log.Error(err, fmt.Sprintf("[provisionFileDevices] unable to setup loop device for %s", filePath)) + created = append(created, rb) return nil, err } + rb.loopDev = loopDev + rb.attachedLoopOK = true + created = append(created, rb) r.log.Info(fmt.Sprintf("[provisionFileDevices] file %s attached to %s", filePath, loopDev)) loopPaths = append(loopPaths, loopDev) @@ -1404,25 +1490,82 @@ func (r *Reconciler) provisionFileDevices(lvg *v1alpha1.LVMVolumeGroup) ([]strin return loopPaths, nil } -func (r *Reconciler) cleanupFileDevices(lvg *v1alpha1.LVMVolumeGroup) { +// cleanupFileDevices detaches loop devices and removes backing files +// recorded for this LVG. It walks the union of status.nodes[].fileDevices +// (what the discoverer last observed) and spec.fileDevices (what the +// user asked for), so it cannot leak files that were created but never +// reflected in status — e.g. when the agent crashed mid-provision. +// +// `rm` errors are logged as warnings and do not abort the cleanup +// (a stale ENOENT after manual cleanup is harmless), but every +// `losetup -d` failure is reported back as an error: a busy loop +// device means there is still a live reference to the file (a mount, +// an LV, a sidecar) and removing the LVG resource at this point would +// strand state on the node. +func (r *Reconciler) cleanupFileDevices(lvg *v1alpha1.LVMVolumeGroup) error { + type target struct { + filePath string + loopDevice string + } + seen := make(map[string]target) + add := func(t target) { + if t.filePath == "" && t.loopDevice == "" { + return + } + key := t.filePath + if key == "" { + key = "loop:" + t.loopDevice + } + // Prefer the entry that carries both fields. + if prev, ok := seen[key]; ok { + if prev.loopDevice == "" && t.loopDevice != "" { + prev.loopDevice = t.loopDevice + seen[key] = prev + } + return + } + seen[key] = t + } for _, node := range lvg.Status.Nodes { for _, fd := range node.FileDevices { - if fd.LoopDevice != "" { - cmd, err := r.commands.DetachLoopDevice(fd.LoopDevice) - r.log.Debug(cmd) - if err != nil { - r.log.Warning(fmt.Sprintf("[cleanupFileDevices] unable to detach loop %s: %v", fd.LoopDevice, err)) - } + add(target{filePath: fd.FilePath, loopDevice: fd.LoopDevice}) + } + } + for _, fd := range lvg.Spec.FileDevices { + add(target{filePath: utils.BuildFileDevicePath(fd.Directory, lvg.Name, fd.Size)}) + } + + var detachErrs []error + for _, t := range seen { + // Defense in depth: never act on a path whose basename does not + // match the agent's managed pattern. If it slipped into status + // via a foreign loop PV (or a bug), refuse to rm and warn. + if t.filePath != "" && !utils.IsManagedFileDevicePath(t.filePath, lvg.Name) { + r.log.Warning(fmt.Sprintf("[cleanupFileDevices] refusing to act on unmanaged path %q for LVG %s", t.filePath, lvg.Name)) + continue + } + + if t.loopDevice != "" { + cmd, err := r.commands.DetachLoopDevice(t.loopDevice) + r.log.Debug(cmd) + if err != nil { + detachErrs = append(detachErrs, fmt.Errorf("detach %s: %w", t.loopDevice, err)) + r.log.Error(err, fmt.Sprintf("[cleanupFileDevices] unable to detach loop %s", t.loopDevice)) + continue } - if fd.FilePath != "" { - cmd, err := r.commands.RemoveFileDevice(fd.FilePath) - r.log.Debug(cmd) - if err != nil { - r.log.Warning(fmt.Sprintf("[cleanupFileDevices] unable to remove file %s: %v", fd.FilePath, err)) - } + } + if t.filePath != "" { + cmd, err := r.commands.RemoveFileDevice(t.filePath) + r.log.Debug(cmd) + if err != nil { + r.log.Warning(fmt.Sprintf("[cleanupFileDevices] unable to remove file %s: %v", t.filePath, err)) } } } + if len(detachErrs) > 0 { + return errors.Join(detachErrs...) + } + return nil } func (r *Reconciler) createVGComplex(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) error { diff --git a/images/agent/internal/utils/filedevices.go b/images/agent/internal/utils/filedevices.go new file mode 100644 index 000000000..af2411673 --- /dev/null +++ b/images/agent/internal/utils/filedevices.go @@ -0,0 +1,101 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "path/filepath" + "strings" + + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/deckhouse/sds-node-configurator/images/agent/internal" +) + +// fileDeviceBasenameHashLen is the prefix length of the lower-case hex +// SHA-256 digest embedded in a managed backing-file basename. 12 hex +// chars carry 48 bits — collision-resistant enough to discriminate two +// fileDevices entries that share the same directory while staying short +// enough to fit comfortably into status fields. +const fileDeviceBasenameHashLen = 12 + +// BuildFileDevicePath returns the deterministic absolute path the agent +// uses for the backing file of a single spec.fileDevices entry. +// +// The basename is `-` +// where is a truncated SHA-256 of (directory, size). The name is +// fully determined by the spec entry's contents — not its position in +// the slice — so reordering, deleting, or inserting entries in +// `spec.fileDevices` does not silently rename an existing file. +// +// Together with IsManagedFileDevicePath this is the single owner marker +// the agent relies on: the discoverer never claims a loop PV as a +// managed file device unless its backing file's basename matches this +// pattern with the same lvgName. +func BuildFileDevicePath(directory, lvgName string, size resource.Quantity) string { + digest := sha256.Sum256(fmt.Appendf(nil, "%s|%d", directory, size.Value())) + hash := hex.EncodeToString(digest[:])[:fileDeviceBasenameHashLen] + basename := internal.FileDevicePrefix + lvgName + "-" + hash + internal.FileDeviceImageSuffix + return filepath.Join(directory, basename) +} + +// IsManagedFileDevicePath reports whether path looks like a backing file +// the agent itself created for the given lvgName. The check is purely +// structural (basename pattern) — it does not stat the file. It is used +// by the discoverer to gate "is this loop PV one of ours?" and by +// cleanup to refuse rm-ing unrelated paths even if status was somehow +// corrupted. +// +// An empty lvgName matches any LVG-owned file, which is the right +// behaviour during cluster-wide cleanup where the caller does not know +// the owning LVG name yet. +func IsManagedFileDevicePath(path, lvgName string) bool { + if path == "" { + return false + } + base := filepath.Base(path) + if !strings.HasPrefix(base, internal.FileDevicePrefix) { + return false + } + if !strings.HasSuffix(base, internal.FileDeviceImageSuffix) { + return false + } + middle := strings.TrimSuffix(strings.TrimPrefix(base, internal.FileDevicePrefix), internal.FileDeviceImageSuffix) + dashIdx := strings.LastIndex(middle, "-") + if dashIdx <= 0 || dashIdx >= len(middle)-1 { + return false + } + name := middle[:dashIdx] + hash := middle[dashIdx+1:] + if len(hash) != fileDeviceBasenameHashLen { + return false + } + for _, c := range hash { + switch { + case c >= '0' && c <= '9': + case c >= 'a' && c <= 'f': + default: + return false + } + } + if lvgName == "" { + return name != "" + } + return name == lvgName +} From 7c51bb8d6afa33e3282d81c668a074375178b46a Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Tue, 23 Jun 2026 18:21:43 +0300 Subject: [PATCH 03/32] fix(agent): ReattachFileDevices returns error and aborts VG activation on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReattachFileDevices used to be fire-and-forget: a missing backing file or an EIO on losetup --find --show was logged at Warning level and the function returned cleanly. ActivateAllManagedVGs then proceeded to bring up VGs whose PVs were still missing, which left the affected volume groups in a degraded state with no user-visible signal — the first symptom was usually a confusing "device busy" or "no such file" when a PVC tried to mount much later. The function now collects every per-device failure, joins them with errors.Join, and returns a non-nil error if anything failed. The startup runnable in cmd/main.go skips ActivateAllManagedVGs for this pass when the reattach failed and logs the error so it is visible in pod logs and pickable by alerting on agent error rate. Agents: Claude Opus 4.7 (ведущий, авто-режим) --- images/agent/cmd/main.go | 18 +++++++++++++----- images/agent/internal/utils/activation.go | 23 ++++++++++++++++++++--- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/images/agent/cmd/main.go b/images/agent/cmd/main.go index 2cbe77993..9c49d4187 100644 --- a/images/agent/cmd/main.go +++ b/images/agent/cmd/main.go @@ -147,7 +147,15 @@ func run() int { } log.Info("[main] ReattachFileDevices starts") - reattachFileDevicesAtStartup(ctx, log, mgr, commands, cfgParams.NodeName) + // Activation must not proceed if any backing file failed to + // reattach: the affected VG would otherwise come up degraded + // with no obvious user-visible signal (PVC mount errors much + // later). Reconcilers will retry; surfacing the failure here + // makes the missing PV visible in pod logs and metrics. + if err := reattachFileDevicesAtStartup(ctx, log, mgr, commands, cfgParams.NodeName); err != nil { + log.Error(err, "[main] file device reattach failed; skipping ActivateVGs this pass") + return nil + } log.Info("[main] ReattachFileDevices completed") log.Info("[main] ActivateVGs starts") @@ -324,11 +332,11 @@ func run() int { return 0 } -func reattachFileDevicesAtStartup(ctx context.Context, log logger.Logger, mgr manager.Manager, commands utils.Commands, nodeName string) { +func reattachFileDevicesAtStartup(ctx context.Context, log logger.Logger, mgr manager.Manager, commands utils.Commands, nodeName string) error { var lvgList v1alpha1.LVMVolumeGroupList if err := mgr.GetAPIReader().List(ctx, &lvgList); err != nil { log.Error(err, "[reattachFileDevicesAtStartup] unable to list LVMVolumeGroups") - return + return err } var items []utils.LVGWithFileDevices @@ -359,8 +367,8 @@ func reattachFileDevicesAtStartup(ctx context.Context, log logger.Logger, mgr ma if len(items) == 0 { log.Debug("[reattachFileDevicesAtStartup] no file devices to reattach") - return + return nil } - utils.ReattachFileDevices(ctx, log, commands, items) + return utils.ReattachFileDevices(ctx, log, commands, items) } diff --git a/images/agent/internal/utils/activation.go b/images/agent/internal/utils/activation.go index 567b2fa2f..3d91ef514 100644 --- a/images/agent/internal/utils/activation.go +++ b/images/agent/internal/utils/activation.go @@ -18,6 +18,7 @@ package utils import ( "context" + "errors" "fmt" "strings" "time" @@ -77,9 +78,19 @@ func runWithTimeout[T any](ctx context.Context, timeout time.Duration, fn func(c // ReattachFileDevices re-establishes loop device mappings for file-backed // PVs after a node reboot. It iterates status.nodes[].fileDevices of every // LVMVolumeGroup belonging to this node and runs `losetup --find --show` -// for files whose loop device is not attached yet. -// Must be called BEFORE ActivateAllManagedVGs so that LVM can see the PVs. -func ReattachFileDevices(ctx context.Context, log logger.Logger, commands Commands, lvgs []LVGWithFileDevices) { +// for files whose loop device is not attached yet. Must be called BEFORE +// ActivateAllManagedVGs so that LVM can see the PVs. +// +// On any partial failure the function still tries every remaining entry +// (best-effort) but returns a non-nil error joining all per-device +// failures. The caller is expected to abort `ActivateAllManagedVGs` on +// non-nil return: activating a VG whose backing file silently failed to +// reattach would leave the VG in a degraded state with no obvious +// user-visible signal. +func ReattachFileDevices(ctx context.Context, log logger.Logger, commands Commands, lvgs []LVGWithFileDevices) error { + _ = ctx // future-proof: a context-aware losetup wrapper will use this. + + var failures []error for _, item := range lvgs { for _, fd := range item.FileDevices { if fd.FilePath == "" { @@ -88,6 +99,7 @@ func ReattachFileDevices(ctx context.Context, log logger.Logger, commands Comman cmd, existing, err := commands.FindLoopDeviceByFile(fd.FilePath) log.Debug(cmd) if err != nil { + failures = append(failures, fmt.Errorf("LVG %s: query loop for %s: %w", item.LVGName, fd.FilePath, err)) log.Warning(fmt.Sprintf("[ReattachFileDevices] unable to query loop for %s: %v", fd.FilePath, err)) continue } @@ -98,12 +110,17 @@ func ReattachFileDevices(ctx context.Context, log logger.Logger, commands Comman cmd, loopDev, err := commands.SetupLoopDevice(fd.FilePath) log.Debug(cmd) if err != nil { + failures = append(failures, fmt.Errorf("LVG %s: setup loop for %s: %w", item.LVGName, fd.FilePath, err)) log.Error(err, fmt.Sprintf("[ReattachFileDevices] unable to reattach %s", fd.FilePath)) continue } log.Info(fmt.Sprintf("[ReattachFileDevices] reattached %s → %s (LVG %s)", fd.FilePath, loopDev, item.LVGName)) } } + if len(failures) > 0 { + return errors.Join(failures...) + } + return nil } // LVGWithFileDevices carries the minimal data needed by ReattachFileDevices. From d28be9886f7742ecc4feb7a59c850569c47f2f97 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Tue, 23 Jun 2026 18:21:54 +0300 Subject: [PATCH 04/32] refactor(agent): read loop backing file via Commands.GetLoopBackingFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The discoverer used to run a private `nsenter -t 1 -m -- /bin/cat /sys/block//loop/backing_file` per loop PV per discovery cycle. This duplicated the nsenter argv that already lives in nsentrerExpendedArgs, bypassed exec.CommandContext (no timeout, the reconciler would hang on a stuck cat), and ran an external process from the discoverer's domain layer — outside the Commands interface that all other host commands flow through and that tests mock. This commit folds the read into a new Commands method, GetLoopBackingFile, implemented on top of `losetup --noheadings --output BACK-FILE `. The discoverer now calls it with a 10s context.WithTimeout, the gomock recorder is regenerated, and fileDevice discovery is wired through the owner marker added in the previous commit: any loop PV whose basename does not match the managed `sds--.img` pattern (or whose VG carries no storage.deckhouse.io/lvmVolumeGroupName tag) is skipped with a warning instead of being written into status. While here, drop the redundant `Nodes: nil, FileDeviceNodes: nil` literal that was assigned and then immediately overwritten, and the now-unused bytes/os/exec imports in discoverer.go. Agents: Claude Opus 4.7 (ведущий, авто-режим) --- .../internal/controller/lvg/discoverer.go | 61 +++++++++++-------- images/agent/internal/mock_utils/commands.go | 16 +++++ images/agent/internal/utils/commands.go | 26 +++++++- 3 files changed, 77 insertions(+), 26 deletions(-) diff --git a/images/agent/internal/controller/lvg/discoverer.go b/images/agent/internal/controller/lvg/discoverer.go index 246629dda..bbe009571 100644 --- a/images/agent/internal/controller/lvg/discoverer.go +++ b/images/agent/internal/controller/lvg/discoverer.go @@ -17,12 +17,10 @@ limitations under the License. package lvg import ( - "bytes" "context" "errors" "fmt" "maps" - "os/exec" "strings" "time" @@ -429,8 +427,6 @@ func (d *Discoverer) GetLVMVolumeGroupCandidates(bds map[string]v1alpha1.BlockDe VGFree: *resource.NewQuantity(vg.VGFree.Value(), resource.BinarySI), VGUUID: vg.VGUUID, ExtentSize: *resource.NewQuantity(vg.VGExtentSize.Value(), resource.BinarySI), - Nodes: nil, - FileDeviceNodes: nil, } candidate.Nodes, candidate.FileDeviceNodes = d.configureCandidateNodeDevices(sortedPVs, sortedBDs, vg, d.cfg.NodeName) @@ -584,9 +580,17 @@ func (d *Discoverer) configureCandidateNodeDevices(pvs map[string][]internal.PVD bdPathStatus[blockDevice.Status.Path] = blockDevice } + // The agent stamps the owning LVMVolumeGroup name onto its VGs via + // the storage.deckhouse.io/lvmVolumeGroupName tag — use it as the + // authoritative owner for any loop PV that lives in this VG. An + // untagged VG (foreign LVM imported by hand and then tagged enabled) + // must never be claimed; in that case we leave file-device discovery + // off entirely for safety. + _, ownerLVGName := utils.ReadValueFromTags(vg.VGTags, internal.LVMVolumeGroupTag) + for _, pv := range filteredPV { if strings.HasPrefix(pv.PVName, "/dev/loop") { - fileDev := d.buildFileDeviceFromLoopPV(pv) + fileDev := d.buildFileDeviceFromLoopPV(pv, ownerLVGName) if fileDev != nil { fileResult[currentNode] = append(fileResult[currentNode], *fileDev) } @@ -614,21 +618,42 @@ func (d *Discoverer) configureCandidateNodeDevices(pvs map[string][]internal.PVD return result, fileResult } -func (d *Discoverer) buildFileDeviceFromLoopPV(pv internal.PVData) *internal.LVMVGFileDevice { - loopBase := strings.TrimPrefix(pv.PVName, "/dev/") - sysPath := fmt.Sprintf("/sys/block/%s/loop/backing_file", loopBase) +// buildFileDeviceFromLoopPV resolves the backing file for a loop PV and +// returns a managed file-device record iff the basename matches the +// owner pattern produced by utils.BuildFileDevicePath. Any other loop +// PV — a foreign losetup-attached qcow2, a snap squashfs loop that +// somehow ended up in a VG, a manual experiment — is skipped with a +// warning so the agent never writes its path into status (and therefore +// never tries to rm it later during cleanup). +// +// expectedLVGName is the owner name pulled from the VG tag; when empty +// the function refuses to claim any loop PV. +func (d *Discoverer) buildFileDeviceFromLoopPV(pv internal.PVData, expectedLVGName string) *internal.LVMVGFileDevice { + if expectedLVGName == "" { + d.log.Warning(fmt.Sprintf("[buildFileDeviceFromLoopPV] VG of loop PV %s carries no %s tag; skipping file-device discovery", pv.PVName, internal.LVMVolumeGroupTag)) + return nil + } + + // A short timeout protects the discoverer's reconcile loop from a + // hung losetup: this call runs once per loop PV per cycle. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() - cmd, backingFile, err := d.readSysFile(sysPath) + cmd, backingFile, err := d.commands.GetLoopBackingFile(ctx, pv.PVName) d.log.Debug(cmd) if err != nil { - d.log.Warning(fmt.Sprintf("[buildFileDeviceFromLoopPV] unable to read backing_file for %s: %v", pv.PVName, err)) + d.log.Warning(fmt.Sprintf("[buildFileDeviceFromLoopPV] unable to read backing file for %s: %v", pv.PVName, err)) return nil } - if backingFile == "" { return nil } + if !utils.IsManagedFileDevicePath(backingFile, expectedLVGName) { + d.log.Warning(fmt.Sprintf("[buildFileDeviceFromLoopPV] backing file %q of loop PV %s does not match managed pattern for LVG %s; skipping", backingFile, pv.PVName, expectedLVGName)) + return nil + } + return &internal.LVMVGFileDevice{ FilePath: backingFile, LoopDevice: pv.PVName, @@ -637,20 +662,6 @@ func (d *Discoverer) buildFileDeviceFromLoopPV(pv internal.PVData) *internal.LVM } } -func (d *Discoverer) readSysFile(path string) (string, string, error) { - args := []string{"-t", "1", "-m", "-u", "-i", "-n", "-p", "--", "/bin/cat", path} - cmd := exec.Command(internal.NSENTERCmd, args...) - - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return cmd.String(), "", fmt.Errorf("unable to read %s: %w, stderr: %s", path, err, stderr.String()) - } - return cmd.String(), strings.TrimSpace(stdout.String()), nil -} - func checkVGHealth(vgIssues map[string]string, pvIssues map[string][]string, lvIssues map[string]map[string]string, vg internal.VGData) (health, message string) { issues := make([]string, 0, len(vgIssues)+len(pvIssues)+len(lvIssues)+1) diff --git a/images/agent/internal/mock_utils/commands.go b/images/agent/internal/mock_utils/commands.go index 63300047c..c6ed62d85 100644 --- a/images/agent/internal/mock_utils/commands.go +++ b/images/agent/internal/mock_utils/commands.go @@ -615,6 +615,22 @@ func (mr *MockCommandsMockRecorder) FindLoopDeviceByFile(filePath any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindLoopDeviceByFile", reflect.TypeOf((*MockCommands)(nil).FindLoopDeviceByFile), filePath) } +// GetLoopBackingFile mocks base method. +func (m *MockCommands) GetLoopBackingFile(ctx context.Context, loopDev string) (string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLoopBackingFile", ctx, loopDev) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetLoopBackingFile indicates an expected call of GetLoopBackingFile. +func (mr *MockCommandsMockRecorder) GetLoopBackingFile(ctx, loopDev any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoopBackingFile", reflect.TypeOf((*MockCommands)(nil).GetLoopBackingFile), ctx, loopDev) +} + // RemoveFileDevice mocks base method. func (m *MockCommands) RemoveFileDevice(path string) (string, error) { m.ctrl.T.Helper() diff --git a/images/agent/internal/utils/commands.go b/images/agent/internal/utils/commands.go index 265cdf5cc..69610ad15 100644 --- a/images/agent/internal/utils/commands.go +++ b/images/agent/internal/utils/commands.go @@ -28,6 +28,7 @@ import ( "os/exec" "path/filepath" "regexp" + "strconv" "strings" "time" @@ -75,6 +76,7 @@ type Commands interface { SetupLoopDevice(filePath string) (string, string, error) DetachLoopDevice(loopDev string) (string, error) FindLoopDeviceByFile(filePath string) (string, string, error) + GetLoopBackingFile(ctx context.Context, loopDev string) (string, string, error) RemoveFileDevice(path string) (string, error) } @@ -791,7 +793,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito } func (commands) CreateFileDevice(path string, sizeBytes int64) (string, error) { - args := nsentrerExpendedArgs("/usr/bin/fallocate", "-l", fmt.Sprintf("%d", sizeBytes), path) + args := nsentrerExpendedArgs("/usr/bin/fallocate", "-l", strconv.FormatInt(sizeBytes, 10), path) cmd := exec.Command(internal.NSENTERCmd, args...) var stderr bytes.Buffer @@ -850,6 +852,28 @@ func (commands) FindLoopDeviceByFile(filePath string) (string, string, error) { return cmd.String(), strings.TrimSpace(parts[0]), nil } +// GetLoopBackingFile returns the canonical backing-file path that loopDev +// is currently attached to. Empty string means the device is not attached. +// +// Implemented via `losetup --noheadings --output BACK-FILE ` so +// the agent does not have to spawn `cat /sys/block//loop/backing_file` +// in the host PID namespace just to read a single value. Goes through the +// same nsenter wrapper as every other host command in this package so the +// argv stays unit-testable. +func (commands) GetLoopBackingFile(ctx context.Context, loopDev string) (string, string, error) { + args := nsentrerExpendedArgs("/sbin/losetup", "--noheadings", "--output", "BACK-FILE", loopDev) + cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return cmd.String(), "", fmt.Errorf("unable to read backing file for %s: %w, stderr: %s", loopDev, err, stderr.String()) + } + return cmd.String(), strings.TrimSpace(stdout.String()), nil +} + func (commands) RemoveFileDevice(path string) (string, error) { args := nsentrerExpendedArgs("/bin/rm", "-f", path) cmd := exec.Command(internal.NSENTERCmd, args...) From ca495134e38b195be04074470dedcdbbc0d1e33d Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Tue, 23 Jun 2026 18:22:07 +0300 Subject: [PATCH 05/32] fix(crd): make fileDevices entries immutable and symmetrize omitempty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CRD validation now rejects in-place edits of fileDevices[].directory and fileDevices[].size via `x-kubernetes-validations: self == oldSelf`. docs/FAQ.md already promised "No resize: file device size cannot be changed after creation", but nothing actually enforced it: increasing size would fallocate-extend the file while the loop device kept its captured (old) size, and shrinking would leave a file larger than the spec entry asked for. With the basename derived from (directory, size) this would also silently retarget the entry to a different file. The Go side gets a small bit of housekeeping: LVMVolumeGroupTemplate in lvm_volume_group_set.go now uses `json:"blockDeviceSelector,omitempty"`, matching the change applied to LVMVolumeGroupSpec in this PR. Without it, marshalling a template with no selector would emit `"blockDeviceSelector":null` while marshalling the spec form would omit the field entirely. The struct field alignment of both files is re-`gofmt`-ed for the columns to line up after the json-tag change. Agents: Claude Opus 4.7 (ведущий, авто-режим) --- api/v1alpha1/lvm_volume_group.go | 12 ++++++------ api/v1alpha1/lvm_volume_group_set.go | 12 ++++++------ crds/lvmvolumegroup.yaml | 14 +++++++++++++- crds/lvmvolumegroupset.yaml | 14 ++++++++++++-- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/api/v1alpha1/lvm_volume_group.go b/api/v1alpha1/lvm_volume_group.go index 48fdc9c4d..da9210ae8 100644 --- a/api/v1alpha1/lvm_volume_group.go +++ b/api/v1alpha1/lvm_volume_group.go @@ -40,12 +40,12 @@ type LVMVolumeGroup struct { // +k8s:deepcopy-gen=true type LVMVolumeGroupSpec struct { - ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` - BlockDeviceSelector *metav1.LabelSelector `json:"blockDeviceSelector,omitempty"` - ThinPools []LVMVolumeGroupThinPoolSpec `json:"thinPools"` - Type string `json:"type"` - Local LVMVolumeGroupLocalSpec `json:"local"` - FileDevices []LVMVolumeGroupFileDeviceSpec `json:"fileDevices,omitempty"` + ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` + BlockDeviceSelector *metav1.LabelSelector `json:"blockDeviceSelector,omitempty"` + ThinPools []LVMVolumeGroupThinPoolSpec `json:"thinPools"` + Type string `json:"type"` + Local LVMVolumeGroupLocalSpec `json:"local"` + FileDevices []LVMVolumeGroupFileDeviceSpec `json:"fileDevices,omitempty"` } // +k8s:deepcopy-gen=true diff --git a/api/v1alpha1/lvm_volume_group_set.go b/api/v1alpha1/lvm_volume_group_set.go index 30885346a..07b9536a0 100644 --- a/api/v1alpha1/lvm_volume_group_set.go +++ b/api/v1alpha1/lvm_volume_group_set.go @@ -48,12 +48,12 @@ type LVMVolumeGroupSetSpec struct { // +k8s:deepcopy-gen=true type LVMVolumeGroupTemplate struct { - Metadata LVMVolumeGroupTemplateMeta `json:"metadata"` - BlockDeviceSelector *metav1.LabelSelector `json:"blockDeviceSelector"` - ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` - ThinPools []LVMVolumeGroupThinPoolSpec `json:"thinPools"` - Type string `json:"type"` - FileDevices []LVMVolumeGroupFileDeviceSpec `json:"fileDevices,omitempty"` + Metadata LVMVolumeGroupTemplateMeta `json:"metadata"` + BlockDeviceSelector *metav1.LabelSelector `json:"blockDeviceSelector,omitempty"` + ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` + ThinPools []LVMVolumeGroupThinPoolSpec `json:"thinPools"` + Type string `json:"type"` + FileDevices []LVMVolumeGroupFileDeviceSpec `json:"fileDevices,omitempty"` } // +k8s:deepcopy-gen=true diff --git a/crds/lvmvolumegroup.yaml b/crds/lvmvolumegroup.yaml index c21fa27e4..4bebe4508 100644 --- a/crds/lvmvolumegroup.yaml +++ b/crds/lvmvolumegroup.yaml @@ -130,6 +130,11 @@ spec: The agent will create a preallocated file in the specified directory, attach it as a loop device, and use it as a Physical Volume. File devices and block devices can be used together in the same VG. + + Existing entries are immutable: `directory` and `size` cannot be changed + after creation, because the backing file's path is derived from them and + the loop device captures its size at attach time. To increase capacity, + add a new `fileDevices` entry instead of editing an existing one. items: type: object required: @@ -139,14 +144,21 @@ spec: directory: type: string description: | - Directory on the node's filesystem where the backing file will be created. + Absolute directory on the node's filesystem where the backing file + will be created. The directory must already exist and be writable. + x-kubernetes-validations: + - rule: self == oldSelf + message: "fileDevices[].directory is immutable. Add a new entry instead of editing an existing one." size: type: string pattern: '^[0-9]+(\.[0-9]+)?(E|P|T|G|M|k|Ei|Pi|Ti|Gi|Mi|Ki)?$' description: | Size of the preallocated backing file. Must be at least 1Gi. + x-kubernetes-validations: + - rule: self == oldSelf + message: "fileDevices[].size is immutable. Add a new entry instead of resizing." thinPools: type: array description: | diff --git a/crds/lvmvolumegroupset.yaml b/crds/lvmvolumegroupset.yaml index 23d2b7957..73058c099 100644 --- a/crds/lvmvolumegroupset.yaml +++ b/crds/lvmvolumegroupset.yaml @@ -172,7 +172,10 @@ spec: The agent will create a preallocated file in the specified directory, attach it as a loop device, and use it as a Physical Volume. - + + Existing entries are immutable: `directory` and `size` cannot be changed + after creation. To increase capacity, add a new entry instead. + > Note that the configuration will be common for every LVMVolumeGroup created by the set. items: type: object @@ -183,12 +186,19 @@ spec: directory: type: string description: | - Directory on the node's filesystem where the backing file will be created. + Absolute directory on the node's filesystem where the backing file + will be created. + x-kubernetes-validations: + - rule: self == oldSelf + message: "fileDevices[].directory is immutable. Add a new entry instead of editing an existing one." size: type: string pattern: '^[0-9]+(\.[0-9]+)?(E|P|T|G|M|k|Ei|Pi|Ti|Gi|Mi|Ki)?$' description: | Size of the preallocated backing file. Must be at least 1Gi. + x-kubernetes-validations: + - rule: self == oldSelf + message: "fileDevices[].size is immutable. Add a new entry instead of resizing." thinPools: type: array description: | From af2dca30cf69e2a0cffa1bdeececcfb24a3b2f91 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Tue, 23 Jun 2026 18:22:20 +0300 Subject: [PATCH 06/32] test(agent): cover fileDevices provisioning, cleanup, reattach and naming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds unit tests for the safety properties added in the previous commits, all driven through gomock against the Commands interface: - BuildFileDevicePath / IsManagedFileDevicePath: determinism, that different (directory, size) tuples produce different basenames, and that foreign paths (libvirt qcow2, snap squashfs, a path with a too short or non-hex hash) are correctly rejected even when an empty lvgName is passed. - ReattachFileDevices: already-attached files are skipped (no fresh losetup --find --show, which would have leaked loops); a per-device losetup failure does not abort the loop and the joined error surfaces every failing file path; FindLoopDeviceByFile errors are propagated. - provisionFileDevices: reuses an existing loop attachment, and on partial failure rolls back both the file and the loop in LIFO order so a retry sees a clean slate. - cleanupFileDevices: walks the union of spec.fileDevices and status.nodes[].fileDevices (so files created but never reflected in status are still removed), returns an error when losetup -d fails (so the finalizer cannot be removed while a busy loop is still alive), and refuses to act on paths whose basename does not match the managed pattern. - validateFileDevice: rejects relative directories and `..` segments. Agents: Claude Opus 4.7 (ведущий, авто-режим) --- .../controller/lvg/reconciler_test.go | 201 ++++++++++++++++++ .../utils/activation_filedevices_test.go | 97 +++++++++ .../agent/internal/utils/filedevices_test.go | 85 ++++++++ 3 files changed, 383 insertions(+) create mode 100644 images/agent/internal/utils/activation_filedevices_test.go create mode 100644 images/agent/internal/utils/filedevices_test.go diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 103b3a091..93ed25574 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -19,11 +19,13 @@ package lvg import ( "bytes" "context" + "errors" "strings" "testing" "time" "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" errors2 "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,6 +36,7 @@ import ( "github.com/deckhouse/sds-node-configurator/images/agent/internal" "github.com/deckhouse/sds-node-configurator/images/agent/internal/cache" "github.com/deckhouse/sds-node-configurator/images/agent/internal/logger" + "github.com/deckhouse/sds-node-configurator/images/agent/internal/mock_utils" "github.com/deckhouse/sds-node-configurator/images/agent/internal/monitoring" "github.com/deckhouse/sds-node-configurator/images/agent/internal/test_utils" "github.com/deckhouse/sds-node-configurator/images/agent/internal/utils" @@ -1536,4 +1539,202 @@ func TestValidateFileDevice(t *testing.T) { expected.Add(add) assert.Equal(t, expected.Value(), totalSize.Value()) }) + + t.Run("rejects_relative_directory", func(t *testing.T) { + var reason strings.Builder + totalSize := resource.MustParse("0") + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ + Directory: "data/lvm", + Size: resource.MustParse("10Gi"), + } + r.validateFileDevice(fd, 0, &reason, &totalSize) + assert.Contains(t, reason.String(), "must be an absolute path") + }) + + t.Run("rejects_dot_dot_segment", func(t *testing.T) { + var reason strings.Builder + totalSize := resource.MustParse("0") + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ + Directory: "/data/../etc", + Size: resource.MustParse("10Gi"), + } + r.validateFileDevice(fd, 0, &reason, &totalSize) + assert.Contains(t, reason.String(), "must not contain '..'") + }) +} + +// reconcilerWithMockedCommands wires a Reconciler around a gomock +// Commands so we can exercise provisionFileDevices / cleanupFileDevices +// against deterministic expectations. +func reconcilerWithMockedCommands(t *testing.T, mc *mock_utils.MockCommands) *Reconciler { + t.Helper() + return NewReconciler( + test_utils.NewFakeClient(&v1alpha1.LVMVolumeGroup{}, &v1alpha1.LVMLogicalVolume{}), + logger.Logger{}, + monitoring.GetMetrics(""), + cache.New(), + mc, + ReconcilerConfig{NodeName: "test_node"}, + ) +} + +// provisionFileDevices is idempotent: when a backing file is already +// attached to a loop device, the existing loop must be reused. +// `losetup --find --show` would otherwise pick a fresh minor on every +// reconcile retry and slowly leak loops up to the system-wide limit. +func TestProvisionFileDevices_ReusesExistingLoop(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{ + FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{ + {Directory: "/data", Size: resource.MustParse("10Gi")}, + }, + }, + } + want := utils.BuildFileDevicePath("/data", "vg-a", resource.MustParse("10Gi")) + + mc.EXPECT().FindLoopDeviceByFile(want).Return("losetup -j ...", "/dev/loop7", nil) + // CreateFileDevice / SetupLoopDevice MUST NOT be invoked. + + got, err := r.provisionFileDevices(lvg) + assert.NoError(t, err) + assert.Equal(t, []string{"/dev/loop7"}, got) +} + +// When the second file's losetup fails, the first file's freshly +// created loop and backing file must be rolled back so a retry sees a +// clean state instead of a half-attached carcass. +func TestProvisionFileDevices_RollsBackOnPartialFailure(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + + fdA := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + fdB := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("20Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fdA, fdB}}, + } + pathA := utils.BuildFileDevicePath(fdA.Directory, "vg-a", fdA.Size) + pathB := utils.BuildFileDevicePath(fdB.Directory, "vg-a", fdB.Size) + + sizeA := fdA.Size.Value() + sizeB := fdB.Size.Value() + gomock.InOrder( + mc.EXPECT().FindLoopDeviceByFile(pathA).Return("losetup -j", "", nil), + mc.EXPECT().CreateFileDevice(pathA, sizeA).Return("fallocate ...", nil), + mc.EXPECT().SetupLoopDevice(pathA).Return("losetup --find ...", "/dev/loop0", nil), + mc.EXPECT().FindLoopDeviceByFile(pathB).Return("losetup -j", "", nil), + mc.EXPECT().CreateFileDevice(pathB, sizeB).Return("fallocate ...", nil), + mc.EXPECT().SetupLoopDevice(pathB).Return("losetup --find ...", "", errors.New("ENOSPC")), + // rollback in LIFO order: first pathB (file only, loop never attached), then pathA (loop + file). + mc.EXPECT().RemoveFileDevice(pathB).Return("rm ...", nil), + mc.EXPECT().DetachLoopDevice("/dev/loop0").Return("losetup -d ...", nil), + mc.EXPECT().RemoveFileDevice(pathA).Return("rm ...", nil), + ) + + _, err := r.provisionFileDevices(lvg) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "ENOSPC") + } +} + +// cleanupFileDevices must walk the union of spec.fileDevices and +// status.nodes[].fileDevices so files that were created mid-provision +// (and therefore never landed in status) are not leaked when the +// resource is deleted before the discoverer ran. +func TestCleanupFileDevices_UnionOfSpecAndStatus(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + + fdSpec := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + pathSpec := utils.BuildFileDevicePath(fdSpec.Directory, "vg-a", fdSpec.Size) + // status has a different (older) entry — still managed-named. + fdStatusPath := utils.BuildFileDevicePath("/data", "vg-a", resource.MustParse("20Gi")) + + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fdSpec}}, + Status: v1alpha1.LVMVolumeGroupStatus{ + Nodes: []v1alpha1.LVMVolumeGroupNode{{ + Name: "n", + FileDevices: []v1alpha1.LVMVolumeGroupFileDevice{ + {FilePath: fdStatusPath, LoopDevice: "/dev/loop9"}, + }, + }}, + }, + } + + mc.EXPECT().DetachLoopDevice("/dev/loop9").Return("losetup -d ...", nil) + mc.EXPECT().RemoveFileDevice(fdStatusPath).Return("rm ...", nil) + mc.EXPECT().RemoveFileDevice(pathSpec).Return("rm ...", nil) + + assert.NoError(t, r.cleanupFileDevices(lvg)) +} + +// A busy loop device must surface as an error so the finalizer stays +// in place and the operator can retry — silently leaving a busy loop +// stranded on the node when the LVG object is gone is the worst case. +func TestCleanupFileDevices_DetachErrorBlocksFinalizer(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + + pathStatus := utils.BuildFileDevicePath("/data", "vg-a", resource.MustParse("20Gi")) + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Status: v1alpha1.LVMVolumeGroupStatus{ + Nodes: []v1alpha1.LVMVolumeGroupNode{{ + Name: "n", + FileDevices: []v1alpha1.LVMVolumeGroupFileDevice{ + {FilePath: pathStatus, LoopDevice: "/dev/loop9"}, + }, + }}, + }, + } + + mc.EXPECT().DetachLoopDevice("/dev/loop9").Return("losetup -d ...", errors.New("EBUSY")) + // RemoveFileDevice MUST NOT be called when detach failed: we don't + // want to rm the backing file while the kernel still has it open. + + err := r.cleanupFileDevices(lvg) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "EBUSY") + } +} + +// A foreign-looking path that somehow slipped into status (bug, manual +// edit, …) must be skipped, not rm-ed, even if the LVG is being +// deleted. This is the hard owner check that protects unrelated host +// files from being clobbered. +func TestCleanupFileDevices_RefusesUnmanagedPath(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Status: v1alpha1.LVMVolumeGroupStatus{ + Nodes: []v1alpha1.LVMVolumeGroupNode{{ + Name: "n", + FileDevices: []v1alpha1.LVMVolumeGroupFileDevice{ + {FilePath: "/var/lib/libvirt/images/disk0.qcow2", LoopDevice: "/dev/loop3"}, + }, + }}, + }, + } + // No EXPECTations on DetachLoopDevice / RemoveFileDevice: the + // cleanup must refuse to touch this path. + + assert.NoError(t, r.cleanupFileDevices(lvg)) } diff --git a/images/agent/internal/utils/activation_filedevices_test.go b/images/agent/internal/utils/activation_filedevices_test.go new file mode 100644 index 000000000..006a9516b --- /dev/null +++ b/images/agent/internal/utils/activation_filedevices_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils_test + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" + + "github.com/deckhouse/sds-node-configurator/images/agent/internal/logger" + "github.com/deckhouse/sds-node-configurator/images/agent/internal/mock_utils" + "github.com/deckhouse/sds-node-configurator/images/agent/internal/utils" +) + +func testLogger(t *testing.T) logger.Logger { + t.Helper() + log, err := logger.NewLogger(logger.ErrorLevel) + if err != nil { + t.Fatalf("init logger: %v", err) + } + return log +} + +// ReattachFileDevices must reuse an existing loop attachment instead of +// allocating a fresh one — otherwise a node reboot followed by a +// successful reattach leaks loop minors on every restart. +func TestReattachFileDevices_SkipsAlreadyAttached(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + + mc.EXPECT().FindLoopDeviceByFile("/data/sds-vg-a-deadbeef0011.img").Return("losetup -j ...", "/dev/loop0", nil) + // SetupLoopDevice MUST NOT be called when the file is already attached. + + err := utils.ReattachFileDevices(context.Background(), testLogger(t), mc, []utils.LVGWithFileDevices{ + {LVGName: "vg-a", FileDevices: []utils.FileDeviceStatus{{FilePath: "/data/sds-vg-a-deadbeef0011.img", LoopDevice: "/dev/loop0"}}}, + }) + assert.NoError(t, err) +} + +// When losetup fails on a single file, ReattachFileDevices must still +// process every other entry and return a joined error so the caller can +// abort the activation step. +func TestReattachFileDevices_AggregatesErrors(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + + mc.EXPECT().FindLoopDeviceByFile("/data/sds-vg-a-aaaaaaaaaaaa.img").Return("losetup -j ...", "", nil) + mc.EXPECT().SetupLoopDevice("/data/sds-vg-a-aaaaaaaaaaaa.img").Return("losetup --find --show ...", "", errors.New("ENOENT")) + mc.EXPECT().FindLoopDeviceByFile("/data/sds-vg-a-bbbbbbbbbbbb.img").Return("losetup -j ...", "", nil) + mc.EXPECT().SetupLoopDevice("/data/sds-vg-a-bbbbbbbbbbbb.img").Return("losetup --find --show ...", "/dev/loop3", nil) + + err := utils.ReattachFileDevices(context.Background(), testLogger(t), mc, []utils.LVGWithFileDevices{ + {LVGName: "vg-a", FileDevices: []utils.FileDeviceStatus{ + {FilePath: "/data/sds-vg-a-aaaaaaaaaaaa.img"}, + {FilePath: "/data/sds-vg-a-bbbbbbbbbbbb.img"}, + }}, + }) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "aaaaaaaaaaaa.img") + } +} + +// FindLoopDeviceByFile failures must also be surfaced (and remaining +// files still processed best-effort). +func TestReattachFileDevices_QueryFailure(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + + mc.EXPECT().FindLoopDeviceByFile("/data/sds-vg-a-cafebabec0de.img").Return("losetup -j ...", "", errors.New("EIO")) + + err := utils.ReattachFileDevices(context.Background(), testLogger(t), mc, []utils.LVGWithFileDevices{ + {LVGName: "vg-a", FileDevices: []utils.FileDeviceStatus{{FilePath: "/data/sds-vg-a-cafebabec0de.img"}}}, + }) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "EIO") + } +} diff --git a/images/agent/internal/utils/filedevices_test.go b/images/agent/internal/utils/filedevices_test.go new file mode 100644 index 000000000..ff084b85f --- /dev/null +++ b/images/agent/internal/utils/filedevices_test.go @@ -0,0 +1,85 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestBuildFileDevicePath_DeterministicAndDistinguishesEntries(t *testing.T) { + dir := "/data" + lvg := "vg-a" + size10 := resource.MustParse("10Gi") + size20 := resource.MustParse("20Gi") + + p1 := BuildFileDevicePath(dir, lvg, size10) + p2 := BuildFileDevicePath(dir, lvg, size10) + p3 := BuildFileDevicePath(dir, lvg, size20) + p4 := BuildFileDevicePath("/other", lvg, size10) + + assert.Equal(t, p1, p2, "same inputs must produce the same path") + assert.NotEqual(t, p1, p3, "different size must produce a different basename") + assert.NotEqual(t, p1, p4, "different directory must produce a different path") + + assert.True(t, IsManagedFileDevicePath(p1, lvg)) + assert.True(t, IsManagedFileDevicePath(p3, lvg)) + assert.True(t, IsManagedFileDevicePath(p4, lvg)) +} + +func TestIsManagedFileDevicePath(t *testing.T) { + tests := []struct { + name string + path string + lvg string + want bool + }{ + {"empty path", "", "vg-a", false}, + {"no prefix", "/data/random.img", "vg-a", false}, + {"no suffix", "/data/sds-vg-a-deadbeef0011", "vg-a", false}, + {"wrong lvg", "/data/sds-vg-b-deadbeef0011.img", "vg-a", false}, + {"matching lvg", "/data/sds-vg-a-deadbeef0011.img", "vg-a", true}, + {"hash too short", "/data/sds-vg-a-dead.img", "vg-a", false}, + {"non-hex hash", "/data/sds-vg-a-zzzzzzzzzzzz.img", "vg-a", false}, + {"any-lvg ok", "/data/sds-vg-a-deadbeef0011.img", "", true}, + {"any-lvg rejects bogus", "/etc/passwd", "", false}, + {"libvirt qcow2 looks nothing like ours", "/var/lib/libvirt/images/disk0.qcow2", "vg-a", false}, + {"snap loop image", "/var/lib/snapd/snaps/core22.snap", "vg-a", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, IsManagedFileDevicePath(tc.path, tc.lvg)) + }) + } +} + +func TestIsManagedFileDevicePath_RoundTripWithBuildFileDevicePath(t *testing.T) { + // Walk a representative grid of inputs to make sure Build/IsManaged + // stay in sync no matter what (directory, size) pair the user gives. + for _, dir := range []string{"/data", "/var/lib/sds", "/mnt/x"} { + for _, sz := range []string{"1Gi", "10Gi", "999Gi", "2Ti"} { + for _, lvg := range []string{"vg-a", "data-vg", "long-name-with-dashes"} { + q := resource.MustParse(sz) + p := BuildFileDevicePath(dir, lvg, q) + assert.True(t, IsManagedFileDevicePath(p, lvg), "%s should match lvg %s", p, lvg) + assert.False(t, IsManagedFileDevicePath(p, "other-lvg"), "%s must NOT match foreign lvg", p) + } + } + } +} From ec49aec6137dfcd6c04669f5eda5d61fb20b6ef5 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Tue, 23 Jun 2026 18:22:31 +0300 Subject: [PATCH 07/32] style(templates): add trailing newline to NodeGroupConfiguration shell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POSIX text files end with a newline; YAML tools and shell editors otherwise show "No newline at end of file" on every diff. Agents: Claude Opus 4.7 (ведущий, авто-режим) --- .../agent/nodegroupconfiguration-blacklist-loop-devices.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml b/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml index 9d3cf6017..eab622dcc 100644 --- a/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml +++ b/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml @@ -44,4 +44,4 @@ spec: EOF } - configure_multipath \ No newline at end of file + configure_multipath From 9eef67ad1f20737ed9001e51381ebda23d8b3cb1 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Tue, 23 Jun 2026 18:38:22 +0300 Subject: [PATCH 08/32] style(agent): use fmt.Fprintf in validateFileDevice (staticcheck QF1012) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agents: Claude Opus 4.7 (ведущий, авто-режим) --- images/agent/internal/controller/lvg/reconciler.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 7ead63725..77297dad1 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -836,7 +836,7 @@ var minFileDeviceSize = resource.MustParse("1Gi") func (r *Reconciler) validateFileDevice(fd v1alpha1.LVMVolumeGroupFileDeviceSpec, index int, reason *strings.Builder, totalVGSize *resource.Quantity) { if fd.Directory == "" { - reason.WriteString(fmt.Sprintf("fileDevices[%d].directory is empty. ", index)) + fmt.Fprintf(reason, "fileDevices[%d].directory is empty. ", index) return } @@ -846,16 +846,16 @@ func (r *Reconciler) validateFileDevice(fd v1alpha1.LVMVolumeGroupFileDeviceSpec // admin intended. cleaned := filepath.Clean(fd.Directory) if !filepath.IsAbs(cleaned) { - reason.WriteString(fmt.Sprintf("fileDevices[%d].directory %q must be an absolute path. ", index, fd.Directory)) + fmt.Fprintf(reason, "fileDevices[%d].directory %q must be an absolute path. ", index, fd.Directory) return } if cleaned != fd.Directory && strings.Contains(fd.Directory, "..") { - reason.WriteString(fmt.Sprintf("fileDevices[%d].directory %q must not contain '..' segments. ", index, fd.Directory)) + fmt.Fprintf(reason, "fileDevices[%d].directory %q must not contain '..' segments. ", index, fd.Directory) return } if fd.Size.Value() < minFileDeviceSize.Value() { - reason.WriteString(fmt.Sprintf("fileDevices[%d].size %s is less than minimum %s. ", index, fd.Size.String(), minFileDeviceSize.String())) + fmt.Fprintf(reason, "fileDevices[%d].size %s is less than minimum %s. ", index, fd.Size.String(), minFileDeviceSize.String()) return } From 0dadc50150c29fe141db935559de9025edac7355 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Wed, 24 Jun 2026 08:57:07 +0300 Subject: [PATCH 09/32] fix(agent): address fileDevices code review findings Add context and command timeouts for loop/file-device operations, validate duplicate spec entries and directory writability, restore host LVM loop global_filter, fix CRD fileDevices list map semantics, and update tests/mocks. --- crds/lvmvolumegroup.yaml | 4 + crds/lvmvolumegroupset.yaml | 4 + docs/FAQ.md | 1 + images/agent/cmd/main.go | 7 +- .../internal/controller/lvg/discoverer.go | 14 +- .../controller/lvg/discoverer_test.go | 2 +- .../internal/controller/lvg/reconciler.go | 133 +++++++--- .../controller/lvg/reconciler_test.go | 109 ++++++--- .../agent/internal/mock_utils/block_device.go | 33 ++- images/agent/internal/mock_utils/commands.go | 231 ++++++++++-------- images/agent/internal/mock_utils/syscall.go | 19 +- images/agent/internal/utils/activation.go | 51 ++-- .../utils/activation_filedevices_test.go | 19 +- images/agent/internal/utils/commands.go | 58 +++-- images/agent/internal/utils/devicefilter.go | 4 +- ...pconfiguration-blacklist-loop-devices.yaml | 26 +- 16 files changed, 442 insertions(+), 273 deletions(-) diff --git a/crds/lvmvolumegroup.yaml b/crds/lvmvolumegroup.yaml index 4bebe4508..dedaceb23 100644 --- a/crds/lvmvolumegroup.yaml +++ b/crds/lvmvolumegroup.yaml @@ -124,6 +124,10 @@ spec: message: "The actualVGNameOnTheNode field is immutable." fileDevices: type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - directory + - size description: | File-backed block devices to create as LVM PVs for this Volume Group. diff --git a/crds/lvmvolumegroupset.yaml b/crds/lvmvolumegroupset.yaml index 73058c099..994f4550b 100644 --- a/crds/lvmvolumegroupset.yaml +++ b/crds/lvmvolumegroupset.yaml @@ -167,6 +167,10 @@ spec: message: "The actualVGNameOnTheNode field is immutable." fileDevices: type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - directory + - size description: | File-backed block devices to create as LVM PVs in LVMVolumeGroups. diff --git a/docs/FAQ.md b/docs/FAQ.md index bf49f9a7c..7dd7c2fdf 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -161,6 +161,7 @@ File-backed devices allow you to allocate part of an existing filesystem for LVM - **Preallocated only**: Files are created with `fallocate`, which preallocates space on the filesystem. The directory must have enough free space. - **Minimum size**: Each file device must be at least 1Gi. - **Performance overhead**: LVM on a loop device over a filesystem adds double indirection. Use file-backed devices only when dedicated disks are not available. +- **Host LVM filter**: NodeGroupConfiguration adds loop devices to the host-wide LVM `global_filter`, so unprivileged `lvm`/`pvs` on the node do not see them. The agent re-attaches managed backing files at startup and uses its own LVM config for managed Volume Groups. ### Reboot recovery diff --git a/images/agent/cmd/main.go b/images/agent/cmd/main.go index 9c49d4187..0887faeb7 100644 --- a/images/agent/cmd/main.go +++ b/images/agent/cmd/main.go @@ -152,7 +152,7 @@ func run() int { // with no obvious user-visible signal (PVC mount errors much // later). Reconcilers will retry; surfacing the failure here // makes the missing PV visible in pod logs and metrics. - if err := reattachFileDevicesAtStartup(ctx, log, mgr, commands, cfgParams.NodeName); err != nil { + if err := reattachFileDevicesAtStartup(ctx, log, mgr, commands, cfgParams.NodeName, cfgParams.CmdDeadlineDuration); err != nil { log.Error(err, "[main] file device reattach failed; skipping ActivateVGs this pass") return nil } @@ -243,6 +243,7 @@ func run() int { NodeName: cfgParams.NodeName, VolumeGroupScanInterval: cfgParams.VolumeGroupScanInterval, BlockDeviceScanInterval: cfgParams.BlockDeviceScanInterval, + CmdDeadlineDuration: cfgParams.CmdDeadlineDuration, }, ), ) @@ -332,7 +333,7 @@ func run() int { return 0 } -func reattachFileDevicesAtStartup(ctx context.Context, log logger.Logger, mgr manager.Manager, commands utils.Commands, nodeName string) error { +func reattachFileDevicesAtStartup(ctx context.Context, log logger.Logger, mgr manager.Manager, commands utils.Commands, nodeName string, cmdTimeout time.Duration) error { var lvgList v1alpha1.LVMVolumeGroupList if err := mgr.GetAPIReader().List(ctx, &lvgList); err != nil { log.Error(err, "[reattachFileDevicesAtStartup] unable to list LVMVolumeGroups") @@ -370,5 +371,5 @@ func reattachFileDevicesAtStartup(ctx context.Context, log logger.Logger, mgr ma return nil } - return utils.ReattachFileDevices(ctx, log, commands, items) + return utils.ReattachFileDevices(ctx, log, commands, cmdTimeout, items) } diff --git a/images/agent/internal/controller/lvg/discoverer.go b/images/agent/internal/controller/lvg/discoverer.go index bbe009571..41340dbea 100644 --- a/images/agent/internal/controller/lvg/discoverer.go +++ b/images/agent/internal/controller/lvg/discoverer.go @@ -131,7 +131,7 @@ func (d *Discoverer) LVMVolumeGroupDiscoverReconcile(ctx context.Context) bool { d.sdsCache.StoreManagedVGs(maps.Keys(filteredLVGs)) d.log.Debug("[RunLVMVolumeGroupDiscoverController] tries to get LVMVolumeGroup candidates") - candidates, err := d.GetLVMVolumeGroupCandidates(blockDevices) + candidates, err := d.GetLVMVolumeGroupCandidates(ctx, blockDevices) if err != nil { d.log.Error(err, "[RunLVMVolumeGroupDiscoverController] unable to run GetLVMVolumeGroupCandidates") for _, lvg := range filteredLVGs { @@ -355,7 +355,7 @@ func (d *Discoverer) ReconcileUnhealthyLVMVolumeGroups( return candidates, nil } -func (d *Discoverer) GetLVMVolumeGroupCandidates(bds map[string]v1alpha1.BlockDevice) ([]internal.LVMVolumeGroupCandidate, error) { +func (d *Discoverer) GetLVMVolumeGroupCandidates(ctx context.Context, bds map[string]v1alpha1.BlockDevice) ([]internal.LVMVolumeGroupCandidate, error) { vgs, vgErrs := d.sdsCache.GetVGs() vgWithTag := filterVGByTag(vgs, internal.LVMTags) candidates := make([]internal.LVMVolumeGroupCandidate, 0, len(vgWithTag)) @@ -428,7 +428,7 @@ func (d *Discoverer) GetLVMVolumeGroupCandidates(bds map[string]v1alpha1.BlockDe VGUUID: vg.VGUUID, ExtentSize: *resource.NewQuantity(vg.VGExtentSize.Value(), resource.BinarySI), } - candidate.Nodes, candidate.FileDeviceNodes = d.configureCandidateNodeDevices(sortedPVs, sortedBDs, vg, d.cfg.NodeName) + candidate.Nodes, candidate.FileDeviceNodes = d.configureCandidateNodeDevices(ctx, sortedPVs, sortedBDs, vg, d.cfg.NodeName) candidates = append(candidates, candidate) } @@ -569,7 +569,7 @@ func (d *Discoverer) UpdateLVMVolumeGroupByCandidate( return err } -func (d *Discoverer) configureCandidateNodeDevices(pvs map[string][]internal.PVData, bds map[string][]v1alpha1.BlockDevice, vg internal.VGData, currentNode string) (map[string][]internal.LVMVGDevice, map[string][]internal.LVMVGFileDevice) { +func (d *Discoverer) configureCandidateNodeDevices(ctx context.Context, pvs map[string][]internal.PVData, bds map[string][]v1alpha1.BlockDevice, vg internal.VGData, currentNode string) (map[string][]internal.LVMVGDevice, map[string][]internal.LVMVGFileDevice) { filteredPV := pvs[vg.VGName+vg.VGUUID] filteredBds := bds[vg.VGName+vg.VGUUID] bdPathStatus := make(map[string]v1alpha1.BlockDevice, len(bds)) @@ -590,7 +590,7 @@ func (d *Discoverer) configureCandidateNodeDevices(pvs map[string][]internal.PVD for _, pv := range filteredPV { if strings.HasPrefix(pv.PVName, "/dev/loop") { - fileDev := d.buildFileDeviceFromLoopPV(pv, ownerLVGName) + fileDev := d.buildFileDeviceFromLoopPV(ctx, pv, ownerLVGName) if fileDev != nil { fileResult[currentNode] = append(fileResult[currentNode], *fileDev) } @@ -628,7 +628,7 @@ func (d *Discoverer) configureCandidateNodeDevices(pvs map[string][]internal.PVD // // expectedLVGName is the owner name pulled from the VG tag; when empty // the function refuses to claim any loop PV. -func (d *Discoverer) buildFileDeviceFromLoopPV(pv internal.PVData, expectedLVGName string) *internal.LVMVGFileDevice { +func (d *Discoverer) buildFileDeviceFromLoopPV(ctx context.Context, pv internal.PVData, expectedLVGName string) *internal.LVMVGFileDevice { if expectedLVGName == "" { d.log.Warning(fmt.Sprintf("[buildFileDeviceFromLoopPV] VG of loop PV %s carries no %s tag; skipping file-device discovery", pv.PVName, internal.LVMVolumeGroupTag)) return nil @@ -636,7 +636,7 @@ func (d *Discoverer) buildFileDeviceFromLoopPV(pv internal.PVData, expectedLVGNa // A short timeout protects the discoverer's reconcile loop from a // hung losetup: this call runs once per loop PV per cycle. - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() cmd, backingFile, err := d.commands.GetLoopBackingFile(ctx, pv.PVName) diff --git a/images/agent/internal/controller/lvg/discoverer_test.go b/images/agent/internal/controller/lvg/discoverer_test.go index f230e1103..80e1603d2 100644 --- a/images/agent/internal/controller/lvg/discoverer_test.go +++ b/images/agent/internal/controller/lvg/discoverer_test.go @@ -334,7 +334,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { mp := map[string][]v1alpha1.BlockDevice{vgName + vgUUID: bds} ar := map[string][]internal.PVData{vgName + vgUUID: pvs} - actual, _ := setupDiscoverer(nil).configureCandidateNodeDevices(ar, mp, vg, nodeName) + actual, _ := setupDiscoverer(nil).configureCandidateNodeDevices(context.Background(), ar, mp, vg, nodeName) assert.Equal(t, expected, actual) }) diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 77297dad1..a1888f597 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -23,6 +23,7 @@ import ( "path/filepath" "reflect" "slices" + "sort" "strings" "time" @@ -84,6 +85,7 @@ type ReconcilerConfig struct { NodeName string BlockDeviceScanInterval time.Duration VolumeGroupScanInterval time.Duration + CmdDeadlineDuration time.Duration } func NewReconciler( @@ -362,7 +364,7 @@ func (r *Reconciler) reconcileLVGDeleteFunc(ctx context.Context, lvg *v1alpha1.L return true, err } - if err := r.cleanupFileDevices(lvg); err != nil { + if err := r.cleanupFileDevices(ctx, lvg); err != nil { r.log.Error(err, fmt.Sprintf("[reconcileLVGDeleteFunc] unable to clean up file devices for the LVMVolumeGroup %s", lvg.Name)) condErr := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonTerminating, err.Error()) if condErr != nil { @@ -406,7 +408,7 @@ func (r *Reconciler) reconcileLVGUpdateFunc( r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] tries to validate the LVMVolumeGroup %s", lvg.Name)) pvs, _ := r.sdsCache.GetPVs() - valid, reason := r.validateLVGForUpdateFunc(lvg, blockDevices) + valid, reason := r.validateLVGForUpdateFunc(ctx, lvg, blockDevices) if !valid { r.log.Warning(fmt.Sprintf("[reconcileLVGUpdateFunc] the LVMVolumeGroup %s is not valid", lvg.Name)) err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonValidationFailed, reason) @@ -527,7 +529,7 @@ func (r *Reconciler) reconcileLVGCreateFunc( } r.log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] tries to validate the LVMVolumeGroup %s", lvg.Name)) - valid, reason := r.validateLVGForCreateFunc(lvg, blockDevices) + valid, reason := r.validateLVGForCreateFunc(ctx, lvg, blockDevices) if !valid { r.log.Warning(fmt.Sprintf("[reconcileLVGCreateFunc] validation fails for the LVMVolumeGroup %s", lvg.Name)) err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonValidationFailed, reason) @@ -761,6 +763,7 @@ func (r *Reconciler) deleteLVGIfNeeded(ctx context.Context, lvg *v1alpha1.LVMVol } func (r *Reconciler) validateLVGForCreateFunc( + ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice, ) (bool, string) { @@ -781,9 +784,7 @@ func (r *Reconciler) validateLVGForCreateFunc( r.log.Debug(fmt.Sprintf("[validateLVGForCreateFunc] all BlockDevices of the LVMVolumeGroup %s are consumable", lvg.Name)) } - for i, fd := range lvg.Spec.FileDevices { - r.validateFileDevice(fd, i, &reason, &totalVGSize) - } + r.validateFileDevices(ctx, lvg, &reason, &totalVGSize) if lvg.Spec.ThinPools != nil { r.log.Debug(fmt.Sprintf("[validateLVGForCreateFunc] the LVMVolumeGroup %s has thin-pools. Validate if VG size has enough space for the thin-pools", lvg.Name)) @@ -834,7 +835,31 @@ func (r *Reconciler) validateLVGForCreateFunc( var minFileDeviceSize = resource.MustParse("1Gi") -func (r *Reconciler) validateFileDevice(fd v1alpha1.LVMVolumeGroupFileDeviceSpec, index int, reason *strings.Builder, totalVGSize *resource.Quantity) { +func (r *Reconciler) validateFileDevices( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, + reason *strings.Builder, + totalVGSize *resource.Quantity, +) { + seen := make(map[string]int, len(lvg.Spec.FileDevices)) + for i, fd := range lvg.Spec.FileDevices { + key := utils.BuildFileDevicePath(fd.Directory, lvg.Name, fd.Size) + if prev, ok := seen[key]; ok { + fmt.Fprintf(reason, "fileDevices[%d] collides with fileDevices[%d]: same backing file would be created. ", i, prev) + continue + } + seen[key] = i + r.validateFileDevice(ctx, fd, i, reason, totalVGSize) + } +} + +func (r *Reconciler) validateFileDevice( + ctx context.Context, + fd v1alpha1.LVMVolumeGroupFileDeviceSpec, + index int, + reason *strings.Builder, + totalVGSize *resource.Quantity, +) { if fd.Directory == "" { fmt.Fprintf(reason, "fileDevices[%d].directory is empty. ", index) return @@ -859,10 +884,20 @@ func (r *Reconciler) validateFileDevice(fd v1alpha1.LVMVolumeGroupFileDeviceSpec return } - totalVGSize.Add(fd.Size) + if _, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { + return r.commands.CheckFileDeviceDirectory(ctx, fd.Directory) + }); err != nil { + fmt.Fprintf(reason, "fileDevices[%d].directory %q: %v. ", index, fd.Directory, err) + return + } + + if totalVGSize != nil { + totalVGSize.Add(fd.Size) + } } func (r *Reconciler) validateLVGForUpdateFunc( + ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice, ) (bool, string) { @@ -924,6 +959,8 @@ func (r *Reconciler) validateLVGForUpdateFunc( } } + r.validateFileDevices(ctx, lvg, &reason, nil) + if lvg.Spec.ThinPools != nil { r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] the LVMVolumeGroup %s has thin-pools. Validate them", lvg.Name)) actualThinPools := make(map[string]internal.LVData, len(lvg.Spec.ThinPools)) @@ -1410,7 +1447,7 @@ func (r *Reconciler) triggerUdevForPaths(parent context.Context, paths []string) // that pre-existed (e.g. left behind by an earlier successful step) // are preserved on purpose: they will be picked up by the next // reconcile as "already attached". -func (r *Reconciler) provisionFileDevices(lvg *v1alpha1.LVMVolumeGroup) (loopPaths []string, retErr error) { +func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup) (loopPaths []string, retErr error) { if len(lvg.Spec.FileDevices) == 0 { return nil, nil } @@ -1431,12 +1468,12 @@ func (r *Reconciler) provisionFileDevices(lvg *v1alpha1.LVMVolumeGroup) (loopPat for i := len(created) - 1; i >= 0; i-- { rb := created[i] if rb.attachedLoopOK && rb.loopDev != "" { - if cmd, err := r.commands.DetachLoopDevice(rb.loopDev); err != nil { + if cmd, err := r.commands.DetachLoopDevice(ctx, rb.loopDev); err != nil { r.log.Warning(fmt.Sprintf("[provisionFileDevices][rollback] unable to detach %s: %v (cmd: %s)", rb.loopDev, err, cmd)) } } if rb.createdFile && rb.filePath != "" { - if cmd, err := r.commands.RemoveFileDevice(rb.filePath); err != nil { + if cmd, err := r.commands.RemoveFileDevice(ctx, rb.filePath); err != nil { r.log.Warning(fmt.Sprintf("[provisionFileDevices][rollback] unable to remove %s: %v (cmd: %s)", rb.filePath, err, cmd)) } } @@ -1444,28 +1481,43 @@ func (r *Reconciler) provisionFileDevices(lvg *v1alpha1.LVMVolumeGroup) (loopPat }() loopPaths = make([]string, 0, len(lvg.Spec.FileDevices)) + seenLoops := make(map[string]struct{}, len(lvg.Spec.FileDevices)) for _, fd := range lvg.Spec.FileDevices { + if err := ctx.Err(); err != nil { + return nil, err + } + filePath := utils.BuildFileDevicePath(fd.Directory, lvg.Name, fd.Size) sizeBytes := fd.Size.Value() - // Already attached? Reuse — calling losetup --find --show again - // would allocate a fresh loop minor and orphan the previous one. - cmd, existing, err := r.commands.FindLoopDeviceByFile(filePath) - r.log.Debug(cmd) + type findResult struct { + cmd string + loopDev string + } + findRes, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (findResult, error) { + cmd, existing, err := r.commands.FindLoopDeviceByFile(ctx, filePath) + return findResult{cmd: cmd, loopDev: existing}, err + }) + r.log.Debug(findRes.cmd) if err != nil { return nil, fmt.Errorf("query loop for %s: %w", filePath, err) } - if existing != "" { - r.log.Info(fmt.Sprintf("[provisionFileDevices] %s already attached to %s; reusing", filePath, existing)) - loopPaths = append(loopPaths, existing) + if findRes.loopDev != "" { + r.log.Info(fmt.Sprintf("[provisionFileDevices] %s already attached to %s; reusing", filePath, findRes.loopDev)) + if _, ok := seenLoops[findRes.loopDev]; !ok { + seenLoops[findRes.loopDev] = struct{}{} + loopPaths = append(loopPaths, findRes.loopDev) + } continue } r.log.Info(fmt.Sprintf("[provisionFileDevices] creating file device %s (%d bytes) for LVMVolumeGroup %s", filePath, sizeBytes, lvg.Name)) rb := rollback{filePath: filePath} - cmd, err = r.commands.CreateFileDevice(filePath, sizeBytes) - r.log.Debug(cmd) + createCmd, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { + return r.commands.CreateFileDevice(ctx, filePath, sizeBytes) + }) + r.log.Debug(createCmd) if err != nil { r.log.Error(err, fmt.Sprintf("[provisionFileDevices] unable to create file %s", filePath)) created = append(created, rb) @@ -1473,19 +1525,29 @@ func (r *Reconciler) provisionFileDevices(lvg *v1alpha1.LVMVolumeGroup) (loopPat } rb.createdFile = true - cmd, loopDev, err := r.commands.SetupLoopDevice(filePath) - r.log.Debug(cmd) + type setupResult struct { + cmd string + loopDev string + } + setupRes, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (setupResult, error) { + cmd, loopDev, err := r.commands.SetupLoopDevice(ctx, filePath) + return setupResult{cmd: cmd, loopDev: loopDev}, err + }) + r.log.Debug(setupRes.cmd) if err != nil { r.log.Error(err, fmt.Sprintf("[provisionFileDevices] unable to setup loop device for %s", filePath)) created = append(created, rb) return nil, err } - rb.loopDev = loopDev + rb.loopDev = setupRes.loopDev rb.attachedLoopOK = true created = append(created, rb) - r.log.Info(fmt.Sprintf("[provisionFileDevices] file %s attached to %s", filePath, loopDev)) - loopPaths = append(loopPaths, loopDev) + r.log.Info(fmt.Sprintf("[provisionFileDevices] file %s attached to %s", filePath, setupRes.loopDev)) + if _, ok := seenLoops[setupRes.loopDev]; !ok { + seenLoops[setupRes.loopDev] = struct{}{} + loopPaths = append(loopPaths, setupRes.loopDev) + } } return loopPaths, nil } @@ -1502,7 +1564,7 @@ func (r *Reconciler) provisionFileDevices(lvg *v1alpha1.LVMVolumeGroup) (loopPat // device means there is still a live reference to the file (a mount, // an LV, a sidecar) and removing the LVG resource at this point would // strand state on the node. -func (r *Reconciler) cleanupFileDevices(lvg *v1alpha1.LVMVolumeGroup) error { +func (r *Reconciler) cleanupFileDevices(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup) error { type target struct { filePath string loopDevice string @@ -1535,8 +1597,15 @@ func (r *Reconciler) cleanupFileDevices(lvg *v1alpha1.LVMVolumeGroup) error { add(target{filePath: utils.BuildFileDevicePath(fd.Directory, lvg.Name, fd.Size)}) } + keys := make([]string, 0, len(seen)) + for k := range seen { + keys = append(keys, k) + } + sort.Strings(keys) + var detachErrs []error - for _, t := range seen { + for _, key := range keys { + t := seen[key] // Defense in depth: never act on a path whose basename does not // match the agent's managed pattern. If it slipped into status // via a foreign loop PV (or a bug), refuse to rm and warn. @@ -1546,7 +1615,9 @@ func (r *Reconciler) cleanupFileDevices(lvg *v1alpha1.LVMVolumeGroup) error { } if t.loopDevice != "" { - cmd, err := r.commands.DetachLoopDevice(t.loopDevice) + cmd, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { + return r.commands.DetachLoopDevice(ctx, t.loopDevice) + }) r.log.Debug(cmd) if err != nil { detachErrs = append(detachErrs, fmt.Errorf("detach %s: %w", t.loopDevice, err)) @@ -1555,7 +1626,9 @@ func (r *Reconciler) cleanupFileDevices(lvg *v1alpha1.LVMVolumeGroup) error { } } if t.filePath != "" { - cmd, err := r.commands.RemoveFileDevice(t.filePath) + cmd, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { + return r.commands.RemoveFileDevice(ctx, t.filePath) + }) r.log.Debug(cmd) if err != nil { r.log.Warning(fmt.Sprintf("[cleanupFileDevices] unable to remove file %s: %v", t.filePath, err)) @@ -1571,7 +1644,7 @@ func (r *Reconciler) cleanupFileDevices(lvg *v1alpha1.LVMVolumeGroup) error { func (r *Reconciler) createVGComplex(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) error { paths := extractPathsFromBlockDevices(nil, blockDevices) - loopPaths, err := r.provisionFileDevices(lvg) + loopPaths, err := r.provisionFileDevices(ctx, lvg) if err != nil { return fmt.Errorf("file device provisioning failed: %w", err) } diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 93ed25574..2ab8836dc 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -99,7 +99,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { r.sdsCache.StorePVs(pvs, bytes.Buffer{}) - valid, reason := r.validateLVGForUpdateFunc(lvg, bds) + valid, reason := r.validateLVGForUpdateFunc(ctx, lvg, bds) if assert.True(t, valid) { assert.Equal(t, "", reason) } @@ -159,7 +159,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { r.sdsCache.StorePVs(pvs, bytes.Buffer{}) // new block device is not consumable - valid, _ := r.validateLVGForUpdateFunc(lvg, bds) + valid, _ := r.validateLVGForUpdateFunc(ctx, lvg, bds) assert.False(t, valid) }) @@ -229,7 +229,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { r.sdsCache.StorePVs(pvs, bytes.Buffer{}) r.sdsCache.StoreVGs(vgs, bytes.Buffer{}) - valid, reason := r.validateLVGForUpdateFunc(lvg, bds) + valid, reason := r.validateLVGForUpdateFunc(ctx, lvg, bds) if assert.True(t, valid) { assert.Equal(t, "", reason) } @@ -306,7 +306,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { r.sdsCache.StorePVs(pvs, bytes.Buffer{}) r.sdsCache.StoreVGs(vgs, bytes.Buffer{}) - valid, _ := r.validateLVGForUpdateFunc(lvg, bds) + valid, _ := r.validateLVGForUpdateFunc(ctx, lvg, bds) assert.False(t, valid) }) }) @@ -344,7 +344,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { Spec: v1alpha1.LVMVolumeGroupSpec{}, } - valid, reason := r.validateLVGForCreateFunc(lvg, bds) + valid, reason := r.validateLVGForCreateFunc(ctx, lvg, bds) if assert.True(t, valid) { assert.Equal(t, "", reason) } @@ -371,7 +371,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { Spec: v1alpha1.LVMVolumeGroupSpec{}, } - valid, _ := r.validateLVGForCreateFunc(lvg, bds) + valid, _ := r.validateLVGForCreateFunc(ctx, lvg, bds) assert.False(t, valid) }) @@ -411,7 +411,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - valid, reason := r.validateLVGForCreateFunc(lvg, bds) + valid, reason := r.validateLVGForCreateFunc(ctx, lvg, bds) if assert.True(t, valid) { assert.Equal(t, "", reason) } @@ -453,7 +453,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { }, } - valid, _ := r.validateLVGForCreateFunc(lvg, bds) + valid, _ := r.validateLVGForCreateFunc(ctx, lvg, bds) assert.False(t, valid) }) }) @@ -1484,55 +1484,69 @@ func setupReconciler() *Reconciler { monitoring.GetMetrics(""), cache.New(), utils.NewCommands(), - ReconcilerConfig{NodeName: "test_node"}) + ReconcilerConfig{NodeName: "test_node", CmdDeadlineDuration: 30 * time.Second}) } func TestValidateFileDevice(t *testing.T) { - r := setupReconciler() + ctx := context.Background() t.Run("valid_file_device", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + mc.EXPECT().CheckFileDeviceDirectory(gomock.Any(), "/data").Return("test -d ...", nil) + r := reconcilerWithMockedCommands(t, mc) + var reason strings.Builder totalSize := resource.MustParse("0") fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ Directory: "/data", Size: resource.MustParse("10Gi"), } - r.validateFileDevice(fd, 0, &reason, &totalSize) + r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) assert.Empty(t, reason.String()) want := resource.MustParse("10Gi") assert.Equal(t, want.Value(), totalSize.Value()) }) t.Run("empty_directory", func(t *testing.T) { + r := setupReconciler() var reason strings.Builder totalSize := resource.MustParse("0") fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ Directory: "", Size: resource.MustParse("10Gi"), } - r.validateFileDevice(fd, 0, &reason, &totalSize) + r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) assert.Contains(t, reason.String(), "directory is empty") }) t.Run("size_too_small", func(t *testing.T) { + r := setupReconciler() var reason strings.Builder totalSize := resource.MustParse("0") fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ Directory: "/data", Size: resource.MustParse("500Mi"), } - r.validateFileDevice(fd, 0, &reason, &totalSize) + r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) assert.Contains(t, reason.String(), "less than minimum") }) t.Run("size_adds_to_total_vg_size", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + mc.EXPECT().CheckFileDeviceDirectory(gomock.Any(), "/data").Return("test -d ...", nil) + r := reconcilerWithMockedCommands(t, mc) + var reason strings.Builder totalSize := resource.MustParse("5Gi") fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ Directory: "/data", Size: resource.MustParse("10Gi"), } - r.validateFileDevice(fd, 0, &reason, &totalSize) + r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) assert.Empty(t, reason.String()) expected := resource.MustParse("5Gi") add := resource.MustParse("10Gi") @@ -1541,26 +1555,44 @@ func TestValidateFileDevice(t *testing.T) { }) t.Run("rejects_relative_directory", func(t *testing.T) { + r := setupReconciler() var reason strings.Builder totalSize := resource.MustParse("0") fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ Directory: "data/lvm", Size: resource.MustParse("10Gi"), } - r.validateFileDevice(fd, 0, &reason, &totalSize) + r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) assert.Contains(t, reason.String(), "must be an absolute path") }) t.Run("rejects_dot_dot_segment", func(t *testing.T) { + r := setupReconciler() var reason strings.Builder totalSize := resource.MustParse("0") fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ Directory: "/data/../etc", Size: resource.MustParse("10Gi"), } - r.validateFileDevice(fd, 0, &reason, &totalSize) + r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) assert.Contains(t, reason.String(), "must not contain '..'") }) + + t.Run("rejects_duplicate_backing_file", func(t *testing.T) { + r := setupReconciler() + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{ + FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{ + {Directory: "/data", Size: resource.MustParse("10Gi")}, + {Directory: "/data", Size: resource.MustParse("10Gi")}, + }, + }, + } + var reason strings.Builder + r.validateFileDevices(ctx, lvg, &reason, nil) + assert.Contains(t, reason.String(), "collides with fileDevices") + }) } // reconcilerWithMockedCommands wires a Reconciler around a gomock @@ -1574,7 +1606,7 @@ func reconcilerWithMockedCommands(t *testing.T, mc *mock_utils.MockCommands) *Re monitoring.GetMetrics(""), cache.New(), mc, - ReconcilerConfig{NodeName: "test_node"}, + ReconcilerConfig{NodeName: "test_node", CmdDeadlineDuration: 30 * time.Second}, ) } @@ -1587,6 +1619,7 @@ func TestProvisionFileDevices_ReusesExistingLoop(t *testing.T) { defer ctrl.Finish() mc := mock_utils.NewMockCommands(ctrl) r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() lvg := &v1alpha1.LVMVolumeGroup{ ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, @@ -1598,10 +1631,10 @@ func TestProvisionFileDevices_ReusesExistingLoop(t *testing.T) { } want := utils.BuildFileDevicePath("/data", "vg-a", resource.MustParse("10Gi")) - mc.EXPECT().FindLoopDeviceByFile(want).Return("losetup -j ...", "/dev/loop7", nil) + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), want).Return("losetup -j ...", "/dev/loop7", nil) // CreateFileDevice / SetupLoopDevice MUST NOT be invoked. - got, err := r.provisionFileDevices(lvg) + got, err := r.provisionFileDevices(ctx, lvg) assert.NoError(t, err) assert.Equal(t, []string{"/dev/loop7"}, got) } @@ -1614,6 +1647,7 @@ func TestProvisionFileDevices_RollsBackOnPartialFailure(t *testing.T) { defer ctrl.Finish() mc := mock_utils.NewMockCommands(ctrl) r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() fdA := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} fdB := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("20Gi")} @@ -1627,19 +1661,19 @@ func TestProvisionFileDevices_RollsBackOnPartialFailure(t *testing.T) { sizeA := fdA.Size.Value() sizeB := fdB.Size.Value() gomock.InOrder( - mc.EXPECT().FindLoopDeviceByFile(pathA).Return("losetup -j", "", nil), - mc.EXPECT().CreateFileDevice(pathA, sizeA).Return("fallocate ...", nil), - mc.EXPECT().SetupLoopDevice(pathA).Return("losetup --find ...", "/dev/loop0", nil), - mc.EXPECT().FindLoopDeviceByFile(pathB).Return("losetup -j", "", nil), - mc.EXPECT().CreateFileDevice(pathB, sizeB).Return("fallocate ...", nil), - mc.EXPECT().SetupLoopDevice(pathB).Return("losetup --find ...", "", errors.New("ENOSPC")), + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), pathA).Return("losetup -j", "", nil), + mc.EXPECT().CreateFileDevice(gomock.Any(), pathA, sizeA).Return("fallocate ...", nil), + mc.EXPECT().SetupLoopDevice(gomock.Any(), pathA).Return("losetup --find ...", "/dev/loop0", nil), + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), pathB).Return("losetup -j", "", nil), + mc.EXPECT().CreateFileDevice(gomock.Any(), pathB, sizeB).Return("fallocate ...", nil), + mc.EXPECT().SetupLoopDevice(gomock.Any(), pathB).Return("losetup --find ...", "", errors.New("ENOSPC")), // rollback in LIFO order: first pathB (file only, loop never attached), then pathA (loop + file). - mc.EXPECT().RemoveFileDevice(pathB).Return("rm ...", nil), - mc.EXPECT().DetachLoopDevice("/dev/loop0").Return("losetup -d ...", nil), - mc.EXPECT().RemoveFileDevice(pathA).Return("rm ...", nil), + mc.EXPECT().RemoveFileDevice(gomock.Any(), pathB).Return("rm ...", nil), + mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop0").Return("losetup -d ...", nil), + mc.EXPECT().RemoveFileDevice(gomock.Any(), pathA).Return("rm ...", nil), ) - _, err := r.provisionFileDevices(lvg) + _, err := r.provisionFileDevices(ctx, lvg) if assert.Error(t, err) { assert.Contains(t, err.Error(), "ENOSPC") } @@ -1654,6 +1688,7 @@ func TestCleanupFileDevices_UnionOfSpecAndStatus(t *testing.T) { defer ctrl.Finish() mc := mock_utils.NewMockCommands(ctrl) r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() fdSpec := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} pathSpec := utils.BuildFileDevicePath(fdSpec.Directory, "vg-a", fdSpec.Size) @@ -1673,11 +1708,11 @@ func TestCleanupFileDevices_UnionOfSpecAndStatus(t *testing.T) { }, } - mc.EXPECT().DetachLoopDevice("/dev/loop9").Return("losetup -d ...", nil) - mc.EXPECT().RemoveFileDevice(fdStatusPath).Return("rm ...", nil) - mc.EXPECT().RemoveFileDevice(pathSpec).Return("rm ...", nil) + mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop9").Return("losetup -d ...", nil) + mc.EXPECT().RemoveFileDevice(gomock.Any(), fdStatusPath).Return("rm ...", nil) + mc.EXPECT().RemoveFileDevice(gomock.Any(), pathSpec).Return("rm ...", nil) - assert.NoError(t, r.cleanupFileDevices(lvg)) + assert.NoError(t, r.cleanupFileDevices(ctx, lvg)) } // A busy loop device must surface as an error so the finalizer stays @@ -1688,6 +1723,7 @@ func TestCleanupFileDevices_DetachErrorBlocksFinalizer(t *testing.T) { defer ctrl.Finish() mc := mock_utils.NewMockCommands(ctrl) r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() pathStatus := utils.BuildFileDevicePath("/data", "vg-a", resource.MustParse("20Gi")) lvg := &v1alpha1.LVMVolumeGroup{ @@ -1702,11 +1738,11 @@ func TestCleanupFileDevices_DetachErrorBlocksFinalizer(t *testing.T) { }, } - mc.EXPECT().DetachLoopDevice("/dev/loop9").Return("losetup -d ...", errors.New("EBUSY")) + mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop9").Return("losetup -d ...", errors.New("EBUSY")) // RemoveFileDevice MUST NOT be called when detach failed: we don't // want to rm the backing file while the kernel still has it open. - err := r.cleanupFileDevices(lvg) + err := r.cleanupFileDevices(ctx, lvg) if assert.Error(t, err) { assert.Contains(t, err.Error(), "EBUSY") } @@ -1721,6 +1757,7 @@ func TestCleanupFileDevices_RefusesUnmanagedPath(t *testing.T) { defer ctrl.Finish() mc := mock_utils.NewMockCommands(ctrl) r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() lvg := &v1alpha1.LVMVolumeGroup{ ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, @@ -1736,5 +1773,5 @@ func TestCleanupFileDevices_RefusesUnmanagedPath(t *testing.T) { // No EXPECTations on DetachLoopDevice / RemoveFileDevice: the // cleanup must refuse to touch this path. - assert.NoError(t, r.cleanupFileDevices(lvg)) + assert.NoError(t, r.cleanupFileDevices(ctx, lvg)) } diff --git a/images/agent/internal/mock_utils/block_device.go b/images/agent/internal/mock_utils/block_device.go index 07b0e5b3f..e2004465d 100644 --- a/images/agent/internal/mock_utils/block_device.go +++ b/images/agent/internal/mock_utils/block_device.go @@ -1,18 +1,18 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ +// /* +// Copyright YEAR Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// */ // Code generated by MockGen. DO NOT EDIT. // Source: block_device.go @@ -29,9 +29,8 @@ import ( fs "io/fs" reflect "reflect" - gomock "go.uber.org/mock/gomock" - utils "github.com/deckhouse/sds-node-configurator/images/agent/internal/utils" + gomock "go.uber.org/mock/gomock" ) // MockDiscarder is a mock of Discarder interface. diff --git a/images/agent/internal/mock_utils/commands.go b/images/agent/internal/mock_utils/commands.go index c6ed62d85..7ac72eff2 100644 --- a/images/agent/internal/mock_utils/commands.go +++ b/images/agent/internal/mock_utils/commands.go @@ -45,6 +45,36 @@ func (m *MockCommands) EXPECT() *MockCommandsMockRecorder { return m.recorder } +// CheckFileDeviceDirectory mocks base method. +func (m *MockCommands) CheckFileDeviceDirectory(ctx context.Context, directory string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckFileDeviceDirectory", ctx, directory) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CheckFileDeviceDirectory indicates an expected call of CheckFileDeviceDirectory. +func (mr *MockCommandsMockRecorder) CheckFileDeviceDirectory(ctx, directory any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckFileDeviceDirectory", reflect.TypeOf((*MockCommands)(nil).CheckFileDeviceDirectory), ctx, directory) +} + +// CreateFileDevice mocks base method. +func (m *MockCommands) CreateFileDevice(ctx context.Context, path string, sizeBytes int64) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFileDevice", ctx, path, sizeBytes) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFileDevice indicates an expected call of CreateFileDevice. +func (mr *MockCommandsMockRecorder) CreateFileDevice(ctx, path, sizeBytes any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFileDevice", reflect.TypeOf((*MockCommands)(nil).CreateFileDevice), ctx, path, sizeBytes) +} + // CreatePV mocks base method. func (m *MockCommands) CreatePV(path string) (string, error) { m.ctrl.T.Helper() @@ -180,6 +210,21 @@ func (mr *MockCommandsMockRecorder) CreateVGShared(vgName, lvmVolumeGroupName, p return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVGShared", reflect.TypeOf((*MockCommands)(nil).CreateVGShared), vgName, lvmVolumeGroupName, pvNames) } +// DetachLoopDevice mocks base method. +func (m *MockCommands) DetachLoopDevice(ctx context.Context, loopDev string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DetachLoopDevice", ctx, loopDev) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DetachLoopDevice indicates an expected call of DetachLoopDevice. +func (mr *MockCommandsMockRecorder) DetachLoopDevice(ctx, loopDev any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DetachLoopDevice", reflect.TypeOf((*MockCommands)(nil).DetachLoopDevice), ctx, loopDev) +} + // ExtendLV mocks base method. func (m *MockCommands) ExtendLV(size int64, vgName, lvName string) (string, error) { m.ctrl.T.Helper() @@ -225,6 +270,22 @@ func (mr *MockCommandsMockRecorder) ExtendVG(vgName, paths any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtendVG", reflect.TypeOf((*MockCommands)(nil).ExtendVG), vgName, paths) } +// FindLoopDeviceByFile mocks base method. +func (m *MockCommands) FindLoopDeviceByFile(ctx context.Context, filePath string) (string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindLoopDeviceByFile", ctx, filePath) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// FindLoopDeviceByFile indicates an expected call of FindLoopDeviceByFile. +func (mr *MockCommandsMockRecorder) FindLoopDeviceByFile(ctx, filePath any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindLoopDeviceByFile", reflect.TypeOf((*MockCommands)(nil).FindLoopDeviceByFile), ctx, filePath) +} + // GetAllLVs mocks base method. func (m *MockCommands) GetAllLVs(ctx context.Context) ([]internal.LVData, string, bytes.Buffer, error) { m.ctrl.T.Helper() @@ -310,6 +371,22 @@ func (mr *MockCommandsMockRecorder) GetLV(vgName, lvName any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLV", reflect.TypeOf((*MockCommands)(nil).GetLV), vgName, lvName) } +// GetLoopBackingFile mocks base method. +func (m *MockCommands) GetLoopBackingFile(ctx context.Context, loopDev string) (string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLoopBackingFile", ctx, loopDev) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetLoopBackingFile indicates an expected call of GetLoopBackingFile. +func (mr *MockCommandsMockRecorder) GetLoopBackingFile(ctx, loopDev any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoopBackingFile", reflect.TypeOf((*MockCommands)(nil).GetLoopBackingFile), ctx, loopDev) +} + // GetPV mocks base method. func (m *MockCommands) GetPV(pvName string) (internal.PVData, string, bytes.Buffer, error) { m.ctrl.T.Helper() @@ -389,21 +466,6 @@ func (mr *MockCommandsMockRecorder) PVScan(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PVScan", reflect.TypeOf((*MockCommands)(nil).PVScan), ctx) } -// UdevadmTrigger mocks base method. -func (m *MockCommands) UdevadmTrigger(ctx context.Context, paths []string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UdevadmTrigger", ctx, paths) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// UdevadmTrigger indicates an expected call of UdevadmTrigger. -func (mr *MockCommandsMockRecorder) UdevadmTrigger(ctx, paths any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UdevadmTrigger", reflect.TypeOf((*MockCommands)(nil).UdevadmTrigger), ctx, paths) -} - // ReTag mocks base method. func (m *MockCommands) ReTag(ctx context.Context, log logger.Logger, metrics *monitoring.Metrics, ctrlName string, cmdTimeout time.Duration) error { m.ctrl.T.Helper() @@ -418,6 +480,21 @@ func (mr *MockCommandsMockRecorder) ReTag(ctx, log, metrics, ctrlName, cmdTimeou return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReTag", reflect.TypeOf((*MockCommands)(nil).ReTag), ctx, log, metrics, ctrlName, cmdTimeout) } +// RemoveFileDevice mocks base method. +func (m *MockCommands) RemoveFileDevice(ctx context.Context, path string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveFileDevice", ctx, path) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveFileDevice indicates an expected call of RemoveFileDevice. +func (mr *MockCommandsMockRecorder) RemoveFileDevice(ctx, path any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveFileDevice", reflect.TypeOf((*MockCommands)(nil).RemoveFileDevice), ctx, path) +} + // RemoveLV mocks base method. func (m *MockCommands) RemoveLV(vgName, lvName string) (string, error) { m.ctrl.T.Helper() @@ -478,6 +555,37 @@ func (mr *MockCommandsMockRecorder) ResizePV(pvName any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResizePV", reflect.TypeOf((*MockCommands)(nil).ResizePV), pvName) } +// SetupLoopDevice mocks base method. +func (m *MockCommands) SetupLoopDevice(ctx context.Context, filePath string) (string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetupLoopDevice", ctx, filePath) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// SetupLoopDevice indicates an expected call of SetupLoopDevice. +func (mr *MockCommandsMockRecorder) SetupLoopDevice(ctx, filePath any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupLoopDevice", reflect.TypeOf((*MockCommands)(nil).SetupLoopDevice), ctx, filePath) +} + +// UdevadmTrigger mocks base method. +func (m *MockCommands) UdevadmTrigger(ctx context.Context, paths []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UdevadmTrigger", ctx, paths) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UdevadmTrigger indicates an expected call of UdevadmTrigger. +func (mr *MockCommandsMockRecorder) UdevadmTrigger(ctx, paths any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UdevadmTrigger", reflect.TypeOf((*MockCommands)(nil).UdevadmTrigger), ctx, paths) +} + // UnmarshalDevices mocks base method. func (m *MockCommands) UnmarshalDevices(out []byte) ([]internal.Device, error) { m.ctrl.T.Helper() @@ -552,96 +660,3 @@ func (mr *MockCommandsMockRecorder) VGScan(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VGScan", reflect.TypeOf((*MockCommands)(nil).VGScan), ctx) } - -// CreateFileDevice mocks base method. -func (m *MockCommands) CreateFileDevice(path string, sizeBytes int64) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateFileDevice", path, sizeBytes) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateFileDevice indicates an expected call of CreateFileDevice. -func (mr *MockCommandsMockRecorder) CreateFileDevice(path, sizeBytes any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFileDevice", reflect.TypeOf((*MockCommands)(nil).CreateFileDevice), path, sizeBytes) -} - -// SetupLoopDevice mocks base method. -func (m *MockCommands) SetupLoopDevice(filePath string) (string, string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetupLoopDevice", filePath) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(string) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// SetupLoopDevice indicates an expected call of SetupLoopDevice. -func (mr *MockCommandsMockRecorder) SetupLoopDevice(filePath any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupLoopDevice", reflect.TypeOf((*MockCommands)(nil).SetupLoopDevice), filePath) -} - -// DetachLoopDevice mocks base method. -func (m *MockCommands) DetachLoopDevice(loopDev string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DetachLoopDevice", loopDev) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DetachLoopDevice indicates an expected call of DetachLoopDevice. -func (mr *MockCommandsMockRecorder) DetachLoopDevice(loopDev any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DetachLoopDevice", reflect.TypeOf((*MockCommands)(nil).DetachLoopDevice), loopDev) -} - -// FindLoopDeviceByFile mocks base method. -func (m *MockCommands) FindLoopDeviceByFile(filePath string) (string, string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FindLoopDeviceByFile", filePath) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(string) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// FindLoopDeviceByFile indicates an expected call of FindLoopDeviceByFile. -func (mr *MockCommandsMockRecorder) FindLoopDeviceByFile(filePath any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindLoopDeviceByFile", reflect.TypeOf((*MockCommands)(nil).FindLoopDeviceByFile), filePath) -} - -// GetLoopBackingFile mocks base method. -func (m *MockCommands) GetLoopBackingFile(ctx context.Context, loopDev string) (string, string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLoopBackingFile", ctx, loopDev) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(string) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetLoopBackingFile indicates an expected call of GetLoopBackingFile. -func (mr *MockCommandsMockRecorder) GetLoopBackingFile(ctx, loopDev any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoopBackingFile", reflect.TypeOf((*MockCommands)(nil).GetLoopBackingFile), ctx, loopDev) -} - -// RemoveFileDevice mocks base method. -func (m *MockCommands) RemoveFileDevice(path string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RemoveFileDevice", path) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// RemoveFileDevice indicates an expected call of RemoveFileDevice. -func (mr *MockCommandsMockRecorder) RemoveFileDevice(path any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveFileDevice", reflect.TypeOf((*MockCommands)(nil).RemoveFileDevice), path) -} diff --git a/images/agent/internal/mock_utils/syscall.go b/images/agent/internal/mock_utils/syscall.go index 18ae434d1..f5771ed36 100644 --- a/images/agent/internal/mock_utils/syscall.go +++ b/images/agent/internal/mock_utils/syscall.go @@ -1,19 +1,3 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - // Code generated by MockGen. DO NOT EDIT. // Source: syscall.go // @@ -28,9 +12,8 @@ package mock_utils import ( reflect "reflect" - gomock "go.uber.org/mock/gomock" - utils "github.com/deckhouse/sds-node-configurator/images/agent/internal/utils" + gomock "go.uber.org/mock/gomock" ) // MockSysCall is a mock of SysCall interface. diff --git a/images/agent/internal/utils/activation.go b/images/agent/internal/utils/activation.go index 3d91ef514..2250ce021 100644 --- a/images/agent/internal/utils/activation.go +++ b/images/agent/internal/utils/activation.go @@ -63,10 +63,10 @@ func FilterVGsByTag(vgs []internal.VGData, tags []string) []internal.VGData { return filtered } -// runWithTimeout invokes fn under a child context with the given timeout, so a +// RunWithTimeout invokes fn under a child context with the given timeout, so a // stuck nsenter-backed LVM command does not block the caller indefinitely. // A non-positive timeout disables the deadline and is treated as "no timeout". -func runWithTimeout[T any](ctx context.Context, timeout time.Duration, fn func(context.Context) (T, error)) (T, error) { +func RunWithTimeout[T any](ctx context.Context, timeout time.Duration, fn func(context.Context) (T, error)) (T, error) { if timeout <= 0 { return fn(ctx) } @@ -87,34 +87,51 @@ func runWithTimeout[T any](ctx context.Context, timeout time.Duration, fn func(c // non-nil return: activating a VG whose backing file silently failed to // reattach would leave the VG in a degraded state with no obvious // user-visible signal. -func ReattachFileDevices(ctx context.Context, log logger.Logger, commands Commands, lvgs []LVGWithFileDevices) error { - _ = ctx // future-proof: a context-aware losetup wrapper will use this. - +func ReattachFileDevices(ctx context.Context, log logger.Logger, commands Commands, cmdTimeout time.Duration, lvgs []LVGWithFileDevices) error { var failures []error for _, item := range lvgs { for _, fd := range item.FileDevices { if fd.FilePath == "" { continue } - cmd, existing, err := commands.FindLoopDeviceByFile(fd.FilePath) - log.Debug(cmd) + if err := ctx.Err(); err != nil { + return errors.Join(append(failures, fmt.Errorf("aborted: %w", err))...) + } + + type findResult struct { + cmd string + loopDev string + } + findRes, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (findResult, error) { + cmd, existing, err := commands.FindLoopDeviceByFile(ctx, fd.FilePath) + return findResult{cmd: cmd, loopDev: existing}, err + }) + log.Debug(findRes.cmd) if err != nil { failures = append(failures, fmt.Errorf("LVG %s: query loop for %s: %w", item.LVGName, fd.FilePath, err)) log.Warning(fmt.Sprintf("[ReattachFileDevices] unable to query loop for %s: %v", fd.FilePath, err)) continue } - if existing != "" { - log.Debug(fmt.Sprintf("[ReattachFileDevices] %s already attached to %s", fd.FilePath, existing)) + if findRes.loopDev != "" { + log.Debug(fmt.Sprintf("[ReattachFileDevices] %s already attached to %s", fd.FilePath, findRes.loopDev)) continue } - cmd, loopDev, err := commands.SetupLoopDevice(fd.FilePath) - log.Debug(cmd) + + type setupResult struct { + cmd string + loopDev string + } + setupRes, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (setupResult, error) { + cmd, loopDev, err := commands.SetupLoopDevice(ctx, fd.FilePath) + return setupResult{cmd: cmd, loopDev: loopDev}, err + }) + log.Debug(setupRes.cmd) if err != nil { failures = append(failures, fmt.Errorf("LVG %s: setup loop for %s: %w", item.LVGName, fd.FilePath, err)) log.Error(err, fmt.Sprintf("[ReattachFileDevices] unable to reattach %s", fd.FilePath)) continue } - log.Info(fmt.Sprintf("[ReattachFileDevices] reattached %s → %s (LVG %s)", fd.FilePath, loopDev, item.LVGName)) + log.Info(fmt.Sprintf("[ReattachFileDevices] reattached %s → %s (LVG %s)", fd.FilePath, setupRes.loopDev, item.LVGName)) } } if len(failures) > 0 { @@ -137,12 +154,12 @@ type FileDeviceStatus struct { func ActivateAllManagedVGs(ctx context.Context, log logger.Logger, commands Commands, metrics *monitoring.Metrics, cmdTimeout time.Duration) error { log.Info("[ActivateVGs] refreshing LVM metadata cache") - if cmd, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + if cmd, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return commands.PVScan(ctx) }); err != nil { log.Warning(fmt.Sprintf("[ActivateVGs] pvscan --cache failed (cmd: %s): %v", cmd, err)) } - if cmd, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + if cmd, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return commands.VGScan(ctx) }); err != nil { log.Warning(fmt.Sprintf("[ActivateVGs] vgscan --cache failed (cmd: %s): %v", cmd, err)) @@ -152,7 +169,7 @@ func ActivateAllManagedVGs(ctx context.Context, log logger.Logger, commands Comm data []internal.VGData cmdStr string } - res, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (vgsResult, error) { + res, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (vgsResult, error) { data, cmdStr, _, err := commands.GetAllVGs(ctx) return vgsResult{data: data, cmdStr: cmdStr}, err }) @@ -172,7 +189,7 @@ func ActivateAllManagedVGs(ctx context.Context, log logger.Logger, commands Comm var activationErrors []error for _, vg := range managedVGs { shared := vg.VGShared != "" - cmd, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + cmd, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return commands.VGActivate(ctx, vg.VGName, shared) }) if err != nil { @@ -237,7 +254,7 @@ func EnsureVGActivation( activated := false for _, vg := range vgsToActivate { shared := vg.VGShared != "" - cmd, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + cmd, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return commands.VGActivate(ctx, vg.VGName, shared) }) if err != nil { diff --git a/images/agent/internal/utils/activation_filedevices_test.go b/images/agent/internal/utils/activation_filedevices_test.go index 006a9516b..0d8be6d71 100644 --- a/images/agent/internal/utils/activation_filedevices_test.go +++ b/images/agent/internal/utils/activation_filedevices_test.go @@ -20,6 +20,7 @@ import ( "context" "errors" "testing" + "time" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" @@ -46,10 +47,10 @@ func TestReattachFileDevices_SkipsAlreadyAttached(t *testing.T) { defer ctrl.Finish() mc := mock_utils.NewMockCommands(ctrl) - mc.EXPECT().FindLoopDeviceByFile("/data/sds-vg-a-deadbeef0011.img").Return("losetup -j ...", "/dev/loop0", nil) + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), "/data/sds-vg-a-deadbeef0011.img").Return("losetup -j ...", "/dev/loop0", nil) // SetupLoopDevice MUST NOT be called when the file is already attached. - err := utils.ReattachFileDevices(context.Background(), testLogger(t), mc, []utils.LVGWithFileDevices{ + err := utils.ReattachFileDevices(context.Background(), testLogger(t), mc, 30*time.Second, []utils.LVGWithFileDevices{ {LVGName: "vg-a", FileDevices: []utils.FileDeviceStatus{{FilePath: "/data/sds-vg-a-deadbeef0011.img", LoopDevice: "/dev/loop0"}}}, }) assert.NoError(t, err) @@ -63,12 +64,12 @@ func TestReattachFileDevices_AggregatesErrors(t *testing.T) { defer ctrl.Finish() mc := mock_utils.NewMockCommands(ctrl) - mc.EXPECT().FindLoopDeviceByFile("/data/sds-vg-a-aaaaaaaaaaaa.img").Return("losetup -j ...", "", nil) - mc.EXPECT().SetupLoopDevice("/data/sds-vg-a-aaaaaaaaaaaa.img").Return("losetup --find --show ...", "", errors.New("ENOENT")) - mc.EXPECT().FindLoopDeviceByFile("/data/sds-vg-a-bbbbbbbbbbbb.img").Return("losetup -j ...", "", nil) - mc.EXPECT().SetupLoopDevice("/data/sds-vg-a-bbbbbbbbbbbb.img").Return("losetup --find --show ...", "/dev/loop3", nil) + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), "/data/sds-vg-a-aaaaaaaaaaaa.img").Return("losetup -j ...", "", nil) + mc.EXPECT().SetupLoopDevice(gomock.Any(), "/data/sds-vg-a-aaaaaaaaaaaa.img").Return("losetup --find --show ...", "", errors.New("ENOENT")) + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), "/data/sds-vg-a-bbbbbbbbbbbb.img").Return("losetup -j ...", "", nil) + mc.EXPECT().SetupLoopDevice(gomock.Any(), "/data/sds-vg-a-bbbbbbbbbbbb.img").Return("losetup --find --show ...", "/dev/loop3", nil) - err := utils.ReattachFileDevices(context.Background(), testLogger(t), mc, []utils.LVGWithFileDevices{ + err := utils.ReattachFileDevices(context.Background(), testLogger(t), mc, 30*time.Second, []utils.LVGWithFileDevices{ {LVGName: "vg-a", FileDevices: []utils.FileDeviceStatus{ {FilePath: "/data/sds-vg-a-aaaaaaaaaaaa.img"}, {FilePath: "/data/sds-vg-a-bbbbbbbbbbbb.img"}, @@ -86,9 +87,9 @@ func TestReattachFileDevices_QueryFailure(t *testing.T) { defer ctrl.Finish() mc := mock_utils.NewMockCommands(ctrl) - mc.EXPECT().FindLoopDeviceByFile("/data/sds-vg-a-cafebabec0de.img").Return("losetup -j ...", "", errors.New("EIO")) + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), "/data/sds-vg-a-cafebabec0de.img").Return("losetup -j ...", "", errors.New("EIO")) - err := utils.ReattachFileDevices(context.Background(), testLogger(t), mc, []utils.LVGWithFileDevices{ + err := utils.ReattachFileDevices(context.Background(), testLogger(t), mc, 30*time.Second, []utils.LVGWithFileDevices{ {LVGName: "vg-a", FileDevices: []utils.FileDeviceStatus{{FilePath: "/data/sds-vg-a-cafebabec0de.img"}}}, }) if assert.Error(t, err) { diff --git a/images/agent/internal/utils/commands.go b/images/agent/internal/utils/commands.go index 69610ad15..63b1cad36 100644 --- a/images/agent/internal/utils/commands.go +++ b/images/agent/internal/utils/commands.go @@ -72,12 +72,13 @@ type Commands interface { UnmarshalDevices(out []byte) ([]internal.Device, error) ReTag(ctx context.Context, log logger.Logger, metrics *monitoring.Metrics, ctrlName string, cmdTimeout time.Duration) error - CreateFileDevice(path string, sizeBytes int64) (string, error) - SetupLoopDevice(filePath string) (string, string, error) - DetachLoopDevice(loopDev string) (string, error) - FindLoopDeviceByFile(filePath string) (string, string, error) + CreateFileDevice(ctx context.Context, path string, sizeBytes int64) (string, error) + SetupLoopDevice(ctx context.Context, filePath string) (string, string, error) + DetachLoopDevice(ctx context.Context, loopDev string) (string, error) + FindLoopDeviceByFile(ctx context.Context, filePath string) (string, string, error) GetLoopBackingFile(ctx context.Context, loopDev string) (string, string, error) - RemoveFileDevice(path string) (string, error) + RemoveFileDevice(ctx context.Context, path string) (string, error) + CheckFileDeviceDirectory(ctx context.Context, directory string) (string, error) } type commands struct { @@ -681,7 +682,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito data []internal.LVData cmdStr string } - lvsRes, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (lvsResult, error) { + lvsRes, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (lvsResult, error) { data, cmdStr, _, err := c.GetAllLVs(ctx) return lvsResult{data: data, cmdStr: cmdStr}, err }) @@ -703,7 +704,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito if strings.Contains(tag, internal.LVMTags[1]) { start = time.Now() - cmdStr, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + cmdStr, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return c.LVChangeDelTag(ctx, lv, tag) }) metrics.UtilsCommandsDuration(ctrlName, "lvchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) @@ -716,7 +717,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito } start = time.Now() - cmdStr, err = runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + cmdStr, err = RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return c.VGChangeAddTag(ctx, lv.VGName, internal.LVMTags[0]) }) metrics.UtilsCommandsDuration(ctrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) @@ -738,7 +739,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito data []internal.VGData cmdStr string } - vgsRes, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (vgsResult, error) { + vgsRes, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (vgsResult, error) { data, cmdStr, _, err := c.GetAllVGs(ctx) return vgsResult{data: data, cmdStr: cmdStr}, err }) @@ -760,7 +761,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito if strings.Contains(tag, internal.LVMTags[1]) { start = time.Now() - cmdStr, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + cmdStr, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return c.VGChangeDelTag(ctx, vg.VGName, tag) }) metrics.UtilsCommandsDuration(ctrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) @@ -773,7 +774,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito } start = time.Now() - cmdStr, err = runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + cmdStr, err = RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return c.VGChangeAddTag(ctx, vg.VGName, internal.LVMTags[0]) }) metrics.UtilsCommandsDuration(ctrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) @@ -792,9 +793,9 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito return nil } -func (commands) CreateFileDevice(path string, sizeBytes int64) (string, error) { +func (commands) CreateFileDevice(ctx context.Context, path string, sizeBytes int64) (string, error) { args := nsentrerExpendedArgs("/usr/bin/fallocate", "-l", strconv.FormatInt(sizeBytes, 10), path) - cmd := exec.Command(internal.NSENTERCmd, args...) + cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -805,9 +806,9 @@ func (commands) CreateFileDevice(path string, sizeBytes int64) (string, error) { return cmd.String(), nil } -func (commands) SetupLoopDevice(filePath string) (string, string, error) { +func (commands) SetupLoopDevice(ctx context.Context, filePath string) (string, string, error) { args := nsentrerExpendedArgs("/sbin/losetup", "--find", "--show", filePath) - cmd := exec.Command(internal.NSENTERCmd, args...) + cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout @@ -819,9 +820,9 @@ func (commands) SetupLoopDevice(filePath string) (string, string, error) { return cmd.String(), strings.TrimSpace(stdout.String()), nil } -func (commands) DetachLoopDevice(loopDev string) (string, error) { +func (commands) DetachLoopDevice(ctx context.Context, loopDev string) (string, error) { args := nsentrerExpendedArgs("/sbin/losetup", "-d", loopDev) - cmd := exec.Command(internal.NSENTERCmd, args...) + cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -832,9 +833,9 @@ func (commands) DetachLoopDevice(loopDev string) (string, error) { return cmd.String(), nil } -func (commands) FindLoopDeviceByFile(filePath string) (string, string, error) { +func (commands) FindLoopDeviceByFile(ctx context.Context, filePath string) (string, string, error) { args := nsentrerExpendedArgs("/sbin/losetup", "-j", filePath) - cmd := exec.Command(internal.NSENTERCmd, args...) + cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout @@ -874,9 +875,9 @@ func (commands) GetLoopBackingFile(ctx context.Context, loopDev string) (string, return cmd.String(), strings.TrimSpace(stdout.String()), nil } -func (commands) RemoveFileDevice(path string) (string, error) { +func (commands) RemoveFileDevice(ctx context.Context, path string) (string, error) { args := nsentrerExpendedArgs("/bin/rm", "-f", path) - cmd := exec.Command(internal.NSENTERCmd, args...) + cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -887,6 +888,21 @@ func (commands) RemoveFileDevice(path string) (string, error) { return cmd.String(), nil } +// CheckFileDeviceDirectory verifies that directory exists on the host and is +// writable by the agent (running in PID 1's mount namespace). +func (commands) CheckFileDeviceDirectory(ctx context.Context, directory string) (string, error) { + args := nsentrerExpendedArgs("/usr/bin/test", "-d", directory, "-a", "-w", directory) + cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return cmd.String(), fmt.Errorf("directory %q is not an existing writable directory on the node: %w, stderr: %s", directory, err, stderr.String()) + } + return cmd.String(), nil +} + func unmarshalPVs(out []byte) ([]internal.PVData, error) { var pvR internal.PVReport diff --git a/images/agent/internal/utils/devicefilter.go b/images/agent/internal/utils/devicefilter.go index fc7a8e990..c77690dda 100644 --- a/images/agent/internal/utils/devicefilter.go +++ b/images/agent/internal/utils/devicefilter.go @@ -98,7 +98,7 @@ func IsForeignDeviceBase(base string) bool { // assumption that a transient resolver failure must not silently hide // a legitimate PV. // -// Each resolver call runs under runWithTimeout(cmdTimeout) so a hung +// Each resolver call runs under RunWithTimeout(cmdTimeout) so a hung // nsenter-backed readlink cannot block the scan loop indefinitely. // This mirrors the per-command timeout protection introduced in // PR #290 for every other lvm.static / nsenter invocation in @@ -124,7 +124,7 @@ func FilterForeignPVs( out = append(out, pv) continue } - resolved, err := runWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + resolved, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return resolver(ctx, pv.PVName) }) if err != nil { diff --git a/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml b/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml index eab622dcc..6861c5f7a 100644 --- a/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml +++ b/templates/agent/nodegroupconfiguration-blacklist-loop-devices.yaml @@ -23,10 +23,10 @@ spec: # limitations under the License. # Loop devices should not be queried by multipath — they are never - # multipath targets. LVM filtering for loop devices is intentionally - # NOT configured here: the agent manages file-backed loop devices as - # LVM PVs (spec.fileDevices) and passes its own --config global_filter - # for every LVM command. + # multipath targets. Host-wide LVM also ignores loop devices via + # global_filter in lvm.conf; the agent re-attaches managed file-backed + # loop devices at startup and passes its own --config global_filter + # for every LVM command on managed Volume Groups. bb-event-on 'bb-sync-file-changed' '_on_multipath_config_changed' _on_multipath_config_changed() { @@ -35,6 +35,23 @@ spec: fi } + configure_lvm() { + command -V lvmconfig >/dev/null 2>&1 || return 0 + test -f /etc/lvm/lvm.conf || return 0 + current_global_filter=$(lvmconfig devices/global_filter 2>/dev/null || true) + + case "${current_global_filter}" in + '' ) new_global_filter='["r|^/dev/loop[0-9]+|"]' ;; + */dev/loop*) return 0 ;; + 'global_filter="'*) new_global_filter='["r|^/dev/loop[0-9]+|",'${current_global_filter#*=}] ;; + 'global_filter=['*) new_global_filter='["r|^/dev/loop[0-9]+|",'${current_global_filter#*[} ;; + *) echo error parsing global_filter >&2; return 1 ;; + esac + + lvmconfig --config "devices/global_filter=$new_global_filter" --withcomments --merge > /etc/lvm/lvm.conf.$$ + mv /etc/lvm/lvm.conf.$$ /etc/lvm/lvm.conf + } + configure_multipath() { mkdir -p /etc/multipath/conf.d bb-sync-file /etc/multipath/conf.d/loop-blacklist.conf - < Date: Wed, 24 Jun 2026 13:02:53 +0300 Subject: [PATCH 10/32] changes in helm lib Signed-off-by: v.oleynikov --- charts/deckhouse_lib_helm-1.72.0.tgz | Bin 45558 -> 0 bytes charts/helm_lib/Chart.yaml | 5 + charts/helm_lib/LICENSE | 201 ++ charts/helm_lib/README.md | 1873 +++++++++++++++++ .../templates/_admission_client_ca.tpl | 21 + .../templates/_api_version_and_kind.tpl | 36 + .../helm_lib/templates/_application_image.tpl | 23 + .../_application_security_context.tpl | 263 +++ .../templates/_capi_controller_manager.tpl | 249 +++ .../templates/_cloud_controller_manager.tpl | 280 +++ .../templates/_cloud_data_discoverer.tpl | 311 +++ .../_cloud_provider_user_authz_roles.tpl | 83 + charts/helm_lib/templates/_csi_controller.tpl | 986 +++++++++ charts/helm_lib/templates/_csi_node.tpl | 397 ++++ charts/helm_lib/templates/_dns_policy.tpl | 12 + .../templates/_enable_ds_eviction.tpl | 6 + charts/helm_lib/templates/_envs_for_proxy.tpl | 43 + .../helm_lib/templates/_high_availability.tpl | 39 + .../helm_lib/templates/_kube_rbac_proxy.tpl | 21 + .../helm_lib/templates/_module_controller.tpl | 532 +++++ .../templates/_module_documentation_uri.tpl | 15 + .../templates/_module_ephemeral_storage.tpl | 15 + charts/helm_lib/templates/_module_gateway.tpl | 22 + .../_module_generate_common_name.tpl | 13 + charts/helm_lib/templates/_module_https.tpl | 201 ++ charts/helm_lib/templates/_module_image.tpl | 136 ++ .../templates/_module_ingress_class.tpl | 13 + .../templates/_module_ingress_snippets.tpl | 11 + .../templates/_module_init_container.tpl | 101 + charts/helm_lib/templates/_module_labels.tpl | 15 + charts/helm_lib/templates/_module_name.tpl | 23 + .../templates/_module_public_domain.tpl | 11 + .../templates/_module_security_context.tpl | 263 +++ .../templates/_module_storage_class.tpl | 36 + .../_monitoring_grafana_dashboards.tpl | 101 + .../_monitoring_prometheus_rules.tpl | 135 ++ charts/helm_lib/templates/_node_affinity.tpl | 262 +++ .../templates/_pod_disruption_budget.tpl | 6 + charts/helm_lib/templates/_priority_class.tpl | 7 + .../templates/_resources_management.tpl | 160 ++ .../templates/_spec_for_high_availability.tpl | 179 ++ 41 files changed, 7106 insertions(+) delete mode 100644 charts/deckhouse_lib_helm-1.72.0.tgz create mode 100644 charts/helm_lib/Chart.yaml create mode 100644 charts/helm_lib/LICENSE create mode 100644 charts/helm_lib/README.md create mode 100644 charts/helm_lib/templates/_admission_client_ca.tpl create mode 100644 charts/helm_lib/templates/_api_version_and_kind.tpl create mode 100644 charts/helm_lib/templates/_application_image.tpl create mode 100644 charts/helm_lib/templates/_application_security_context.tpl create mode 100644 charts/helm_lib/templates/_capi_controller_manager.tpl create mode 100644 charts/helm_lib/templates/_cloud_controller_manager.tpl create mode 100644 charts/helm_lib/templates/_cloud_data_discoverer.tpl create mode 100644 charts/helm_lib/templates/_cloud_provider_user_authz_roles.tpl create mode 100644 charts/helm_lib/templates/_csi_controller.tpl create mode 100644 charts/helm_lib/templates/_csi_node.tpl create mode 100644 charts/helm_lib/templates/_dns_policy.tpl create mode 100644 charts/helm_lib/templates/_enable_ds_eviction.tpl create mode 100644 charts/helm_lib/templates/_envs_for_proxy.tpl create mode 100644 charts/helm_lib/templates/_high_availability.tpl create mode 100644 charts/helm_lib/templates/_kube_rbac_proxy.tpl create mode 100644 charts/helm_lib/templates/_module_controller.tpl create mode 100644 charts/helm_lib/templates/_module_documentation_uri.tpl create mode 100644 charts/helm_lib/templates/_module_ephemeral_storage.tpl create mode 100644 charts/helm_lib/templates/_module_gateway.tpl create mode 100644 charts/helm_lib/templates/_module_generate_common_name.tpl create mode 100644 charts/helm_lib/templates/_module_https.tpl create mode 100644 charts/helm_lib/templates/_module_image.tpl create mode 100644 charts/helm_lib/templates/_module_ingress_class.tpl create mode 100644 charts/helm_lib/templates/_module_ingress_snippets.tpl create mode 100644 charts/helm_lib/templates/_module_init_container.tpl create mode 100644 charts/helm_lib/templates/_module_labels.tpl create mode 100644 charts/helm_lib/templates/_module_name.tpl create mode 100644 charts/helm_lib/templates/_module_public_domain.tpl create mode 100644 charts/helm_lib/templates/_module_security_context.tpl create mode 100644 charts/helm_lib/templates/_module_storage_class.tpl create mode 100644 charts/helm_lib/templates/_monitoring_grafana_dashboards.tpl create mode 100644 charts/helm_lib/templates/_monitoring_prometheus_rules.tpl create mode 100644 charts/helm_lib/templates/_node_affinity.tpl create mode 100644 charts/helm_lib/templates/_pod_disruption_budget.tpl create mode 100644 charts/helm_lib/templates/_priority_class.tpl create mode 100644 charts/helm_lib/templates/_resources_management.tpl create mode 100644 charts/helm_lib/templates/_spec_for_high_availability.tpl diff --git a/charts/deckhouse_lib_helm-1.72.0.tgz b/charts/deckhouse_lib_helm-1.72.0.tgz deleted file mode 100644 index bf178e500ce1f149bb181d920ba9993207c6a9ed..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 45558 zcmV)oK%BoHiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMZ%avV33C_KOQ6xgyHCVkk=CPhkK?r>r$lD2s)QT#~Rb0%gT zJzy2O8^h{C4Ny&~M^?nX!#Pj5Px1wBRk+oqFTBXZiW!St1tb!QL?Qv?B|-7cgv=Nk zrucd|LFsh+cmiqBpTlYThxI&z!C>%mZ%_O?7!2Hh2fI7Ff7sdI9qbQw2Yb8we;DlS zy?VL#2QXMu`;E_x6_Ea6uyR{&=f02!=J*#x87A2QxZQo0AQn@c7vlGiD4l{?fl~$w zG|f|3AdsLD&M^PZz=%+AqWK3*Nis_j>p#oj6deGcv7Qz4936lZUsFiu&u(?IJN;L? z{lT+`ro_MI6PW*ctl9PuCR5D#C=KHjqpTRluwUeh%7vEszw>f;V9)>EomV@9r}=*$ z&!CFgnZYqS0G~bqoWK zLBu5h%VCV7P4M|M_;wrgK7W3unz}?;f+zzq$wqiQg*o_uiwP)O0jd=sTLH&Mz-R@0 znBaHEqX+M`FpNo7ppS*@ zK>rt*&Jf!IeX)RU0aV0J*UvRwz=sKEoO*KHjf0-Po2 zBj|$xs6Lk@Hw*0W0=-`EnI+|Sp55RqIRM9^>9;U{Hbn(YU;z)F0U)J!NtT`@M=yr` zm=@~0Xh?92(XPM+W*I4Z$3IL8r0dd9vS3yN+A>iXJ`}%jId9!fTW=&8wQKo8UK)ajInn*yTOZ)R5iQxbhCG z8fvurT{+tTH6DPJy4eLKks|~x5)n-57jz7 zgjq7=jI;Ox&?5g|?7w{B$p78JtCvsm|301$@}Cd-5Pig~V1|tUBY)lF)BjH)DNj)` zqZtDQogsinV$!l4#dw5K0w`i+Mq>mx5U>$LhI%+-C}X(5w`db!1}MrYVkqOX5N85F zoX%K*Xrn;4u=T~Kcq|#(`}`R!E!Y)w(I}^dEqx8pKk<%=5;>t<{@D=l2%vv}jTB{- zggJ&eyv8XmFk=0qi!&v`*``EVNu(YaL7aj}DC-n)=nP~8#76DNXo<@i{R>c-jS+Yr z!<-}D+A=U&#Un6*>}oa|;g4V=B_9w48#%>UF#^%{*DNxqvbm{C@TN937yw3OmL=+M z4)mHLcw@Grq@wl1?8UPV{5wX4CCIMlf=L$V<#UwGV*iX30H&$<#Q>xTuvyLr5+$3Byk6V0C%$jwd(keT zU<3Wb>_hYmOmPAWoQ+Fm@`9u|o=2PA%f7~0g0pe7X(5Y8?H%m@2pNb<&4HpZW(5`6 zRR88Xp+Md4Y}X?x8f>Xv&8(E6^3{e0blLyAOkl2@;zj$?DT>K;0B3w1oWcS`YDUkua~R*iG5XI0j}a@P z<{i6_PWvudf0uYzi`8$Spm=(erXc#S$b==zo(PPp8l|{As?g|PNc8)%Fw@PLs|{b{ zEr{huTO^%{nqM=nT=C^hQ$*I?^pqiutS04a$dIMWh48#oh5LNnWVj%7?jvhKw6Tpe z0HODAWu4bxsEBIuQzLw?bcIIrL?I6!6M}<|s;Ky@9UD#Tui5t3>_3IBYW3D?%N?}` z+^pZh(_Q{E6wfFw=0iPnotJ`U`M>*e=f#dA|6jb?eeop!@8ele{^um|7-2|f*$}ef zj3GLVXRIL8YK7Pt3qYRE@!4>CQk2eNjjS%w!Udv+sVf>#sG}&Z_vv{{H})C zXkcxFXu!PpVcXk(vF{F>cy}+ObZ@iMAo2Z9(59|Fp20Tf=&GY1%S%i|yAmd6Y)BC+ zD2@x1G;%=8@E_}@TbUqFXADenhNmzU_v4tNgzuCJoQ(xX#AKSMa|x)JAaGe?2&N=K zfXJI1&T$$ma8cVFLgP~3Oq$8Xar|zVezP@fy*GaVb;%QC)+=q zoyexZ2BClwQfzL4i>oVhfNkb`CvNFs3U1Jx9XKERlmGz!=fD3ixF#gs0u;fdN3wJ- zK!fkEz(ykR6EiIVrSv4G` z=p(*PQ3N&;92X#>BvNQZe8BXkE;>JbKRh~ldv+dy=;|+5!}n)zPoqr`&2SR&z1|Lp z>>ib1Op-@_pOzIGwH?ubpHl)34&`OEekU(5e?oK6mt8gPwI~6Hg2~;pd-2s7u13Hv zA&f>GFozze1REDq=im@%#t>t&siss}O3V(Ul$`r=(o{q-CqOfKtZ_35Wr8?#=9G(n zaP>wod6%VgUg3M3A~t6Qno8RuN|aGit#tnMNu5=i%~l=o-UEq)=_HhU% zL$2hC**+YVtCig3GL+(M_Hm8$=M92NtF9UpGK}Pv7sl1;8{W3qzQF>E6>Pqa-B`yi zuU+ufXa#Ej)V^I~NDxGBV$@2WqFa<6fN$8~8)M3T!}54od~@GCb1!UimmKiu2K{aZ zu+kUTGh%d>WjGrfdgxj&`|JOwQ>jnSt76w z3b*xZ;gw1s1vN9hzaQZ{4Rz|o_ugV3et*MTHFMXZt!|L-RN8gx-ns>!H+C&-ao_5V z$%I3`ALcsSLZiH}as$)U;(f6V&ke7ryU}WDdiuy#y0+73eMs@(JR*FCV3I_jv(ucMLTWL-bO9YuBE>TfpbKCthSw*UK*q*%ML?)wP zryhnM%|kNF_A+!(wnmP9M0|OV%=h?M?Dxq-t6IL@g<#tf%w-zNot@k*Pd!cxx@C;z z>h5drLq0z6sp6vR)t%WdJaYSl70L7ks*=mG=0h7cKRunEX8Zpb=2);kC25N2a0)Xx zMsz_huonCOSFgPIFS{=XPxk-!@|d!)X0(UeTVEOP#rf<2?CwvWP0^Il`2pA;yv6oK ze;24z_)*^dh{G1Yh5f;*Eu{DsWr(q1PRX^hy_gh5{sWT7F0hy!fb9u_X)*aHe?e$* z0KOZ1xARQIxPs{kO5ywp#UxAE0od7pmLrPAEvJ05J6IemF0pYtTD;i23sxRdI{~Cx zbcFgb%J2wTfghgBh&7FYeWp{AfwHbxsc~Jg;B(9 z=c;u6l|7tEPh&${ajoF~7Z}abR5+b&Xl-Q^{Lg>?zbd|k@Q)KEL=YQkm{)b<;^g%f z_~qhA>g?C^(q2bL3YbEifl?h2kX*d~R~3?{>fM%xbO8RAeAhDr*U0dG%8l-Cb@&Q+ zepAx1(dn&x8AjO$!FlaMal?6lcDG#rDWR#5AN`L#g3sYW)5!e|RBW%HtzpB{QvFkLu@Ooa|BS2_-Ks3k^Vs7Q*W+-IsuTqiC58o_MdGih0_k|hd4h#dG1~GG(Ynh8*AZ&Ol~x+SCtH=-GIWN|TcAdl*#e$2qa~eg90(1? zSr5C)`BKkWs0&i<>Cywg13QCiJyUJ3)B~6#SmJy{$8}twhZ=n>DC7gqmVken-F5(#&@`$Hfqu8#}D3Gd#ei2?wZ~0Qt zt(jZ1!r~qKMR`qiX^Klj}8F3B6AyVOjV&lXUr=BO(KweaO)y-){BI<8|&lSft1 z#!l@TK^~WCJWsAq#oCj!ssVb93aRo`>^E>W4z}+*;2E!%2pa^pjp4M z+Z;A_C$#2<_b&}Sy3OIwc>)XcmLzq`;#qc)!Wg}ymq-Q?)e9n5M`xn8I3*22e+xez z$=!I9a=bsNRgOR6>1PzBO$(BQ z32voG81z&-hUQ4I9rnn*mOwO1g6be!b7o`}}s?`1bj!t|U7KFjw)Ru6VT zelkbJ-PJDLst+oNfmh1($|?7`nwZ7`+eUf()8V|YMu8@v zhO%ES3FWJLdFpHMMs52`1a?9Y47FTWtzI<s#b3!5YV-2U>SO~0jql3WiHoT zm33X5TlDK{Ic+NDQcT}0;gQg)O>%G41j(0rdh%9=$OpfP7>CiV+6wh@lJsD?8S2F} z5fLT51O~bc-zQaZZDme_;Jzx5-{8|HLsJlxYR+r$Y4Mx_+KV3MxuKS9dZ`9jj$)2j z0qGbOmuRHy{LLQcIn%LwWk0LqX7RGFQ*zTM2ULQcl}s$l+uZ&e_{WSCsgpah0Vo!1jQXv#qDY;TlkVmd{Btgjs=y za3qq#%}dT+c!IsOdk|<#9U~0IEq4SqidmkbstNxDVj=L>wO{~xXv4Zt1I7+D0vj>O zVpw=dh{C(vh1yY)ZjtDF0WoAW({vmnXI~sNML?eCm!X;@|6J4UvQc`JJGVNebu$XOgD5hLn*6UL_4t1GVzwgJP%^4#E-!k=Li1x(PwEE=^fH=)Cnye+Utzvpq4nsf9 zZp|VnyRFekS8hq3iWr5&z3e(#vyQk90!Y{=V{4(MeXbR1)Hx8lvk|mTb&Rs+CGk3_ zT<&U2VA_PteAhBPQ9eOaL}5DQiXa@LVM@kK#^n&#wBi`pKy25iowCWR_Urw(8@yUu6;>)Y8`B}-A*n=KtisX;j%3CIDI-ad?B|5q| z12`K|$O<}(iy1|RrVq6Hf&uTnz87t+rSITauyx^^lodFeAr}Xn=Sfq#DT z|G0<8GqbEw_w{?WOC<=j7Z279k|o=N$20Csj?^#}bypTA)- zUvUq>kMG}KTs8Q8)KjsO}v#2u@x+~X^pskjm ze$PLJifTPF+B`^Vx+VKu<4a<=oP6nSCI7m<9go{i{;$yooF*}(N%X7LEU&PrReY;| zL%?*j!M&kT9B74%&Qb)%C{tK1>=F?v=jbTt81V(T%{+wf&_lWEti$a>EZAG_dc9>Y z*PkDOufp$F z-D=-5pwe&4+8Udtj&!l?{~db`JoyYf`3&5(&pKT_ZIcG z2Gwj-?;FhfCi91e6=qf98fQrlCJ9B19hw$Lb}`Wl)XNDi4z-n%Nr-k6{CYF$HY;G@ zTiIJS%{9x1I`TVXuL&vm+AFegI5_EH{^{q}rO3gA z`cSOadc85e?@3YbOR*{i0v;80J{9Zss@MxM)dTodbl80das3zat$1>xP)~~!Mco~y z>o!qSu7jh*bUMp$3=33cayhy<6RnHGQk;!_FABxcTWa?5GAV%xxP~drVnl5dq%C0= z2~s>l@jOnEa^sLzvz+-TnH3xc$xC}f`yiyeS%d6HWg%nS7szwe>O^7GFlTrH3bQc+ z&*kJ1cdHFkOhc5V`=x=7vylv;b|??T>$Y~*Pw$7|NFS#7ZcY1Qo>vmUWJya~)_AYN zp@w0SX!QM4i8r>=x`k< z@o1#!!hF<;lZ;EocUd}jnA1 z!vv>`1^o_#>WlNTr~EL)vARvPe(q6DH>Z!@6pmIIpY0o2zheJNVpbIP<^(L7Q0+T_ z-%l`Wf@TunA{?|aQ3HG|GyV6 zUhcX1pLX|OJ^BCL$MebaTysWy)$C6}m}|`a6okKY=BL4;ettf+mU8mz)^sP2KA+Zi z?ap@s3kXii7QfI8P7Q8)oTR;qq|5v0J-OoDuPdIuP;EAiuUv3 zS8ChdlzjM|IaE4{hd$t9(hFxFTEWqBP3eaQkKgO93CpK6LTzb;{uNxg9xRp7T2l+% zk2m7)%z}@~!&4`Bp8PA!R`itB`+H zn|+sIZxwQ_`qaU(>O0d+#2?Mu91d@0*JwzuVJz(o4#9r1a9tfaw5euiwYw9CcSNL_xP8K9l1j?el+Wh6*GW3UxxzLmyGxD`i%_ z&qFhml#E$Bl$x#wJ)G{yooRW8rY9$(hjB8p-F^b@mKht$b!2$0ojq_i9v(9_W?FaPwX+^2HqPOswym zH&EugKplKH_-@DQ;L>SEzH>?SU|B|<@_u~Tc|S;&&V`#wCtsycUuxbDEpE7?(c>GC zp4fmqw76+#aylj1P?YqS8{Uh{YZzY$C$r6Zam70xmA;maV0yms)6cI@hnKI9j)xbQ z@BYtUhHuWU-k+Wik4{c5Pp_^vxxgAkF0k9fUGH5=#sy(Tf+!VEpihAab*)WBx9y#ojqR3fqiugFOIuU5P zL)KHpf?n&ySntBV{{7hxe*3-!IQl5qinro#A^0$rKe6H1k=|?Nhx%RO=>c*oRh+8UQhW><{5- zV%v^Z{jGBkz0*dY*1CXsZn*|6c82yCGGdv>Ov{@h^UdFsWLR!xI~yZkkmRkbRT?C( zWRHB`cESe!rS)yV0#KZ8pI4T?wh6BMEK^05WjGb;Z3-)HxJ2S03TLhMXNGT(4C~FC zP?lR1eW18-jaWLGd{|@O!RVvKbYp=)x?Kz}T;eZcEn)QRz#GM&<%C-(c;{!1nSbSH zd=1|C>4Wyg#GG`|BD80$ck-607dWL?o#qS?6N(7y$7E_9+AFqiKX5iyRYNaGzUx(O ztaj>Lbx7DH;4JEX68(2_Ely#D<{?}*lD87w(=5peE_aAMwM>R|RkgC(FZJ>%DyVnY z2vS7FL6JP8tVFgXoiR4Tc)B`@Me_q-zA92CO`9qZIFmbxs~epEf+!x%Ehq8`$p}^F z-sXE2Q<$Uoc#6oZsFC>z1Wss#7 z1QdstK^OC7uT@GZaODFKg0o@{kp|k4sMP|Y9jHDnpCj@2{B-gE)OwaYpywOFe-0^0 z5$os)+U);1*x%dTcl{(RzTw?y^m}yy^I7QBpE|v;bWnVrVUJ_fD5?a zWvqZ%jKpo>r8d5HS~<~^wk)pvnM!MI(fpV~;Zda=yo5gQw;015Up9N}mAivbrd+E{ z*0IrQiQ!{*DRj^-P6={P0`r{nQR&Db#@!%pQ+p*wzy`}U z?QL4IIZGRvtAgf>VI^7x8qg@HEU7p!0-YEKQl3A5gIv)`Ol?j4&w{!Fw#(nE}*E1e8c--34dG;lhe77U*%8({9qRxSsx8;s7`R<^q zbs!9-Oiug!S$tk@7wB#6#C$$@)+QX?B-k_OAX93ZVLp=)k`TRCCC8``|8gwE zA0N2lT3Wx^jK#;oMZ^}eI&NkEV^0o=i`%CvI*$iiVcrkKeCmd=aR%sa5GuwN1*(04 zeP;OZtY%J$1z1jdML<>WOW?iIq>Y12k@aUxQofzSBzvYp9mSJp@_s1cl^!vJ261g* zOlZ6BSsHbuNx3jaex97(H8NAwm%Kv{aPnWwKhNTsQ z41=od8b&ZpuVH*s?fRlWQuv`V_Uv&P`=VHDQ4{X20qCc)9AIb)vjWFl*z)Ds zfhg#7g2#vnDnxdoKWQQR8nQB9VBoSrjtD#BkXT<=>Wj;;5*Vi@o%5{~SJOLl0r+;i zvPd7*VGzFVQ1E*d{K!1Sw7Yi*{!VZPqU~~lw?0RzlTjG#rUh7)_`WP4>C39o>O{#z z)rvfL5%^7SW~CjI*?K&q6lKL*oDszzh0i#B`Pr;j6?qH)-dI(9HmYua3(g8e`9zU3 z)x;nnA2L4dg9EP10Wumf%>V-tF4ic##rNGAh+!JfWY`A&R_T@C&<5PouoR0}T8R2| za0q5ufm5)-yRre3#G%iQdA)BK6j<}ZG5vdPVWr5l!DiPEY=y+8S3h<|gJivm^IYi> zeNX65-(CH18u`OyVdJ-zlv7)AUM+JvXjL2*MV{U7s08fUE?AU143!I7IMfBPEjjFa zCB8+%YCa{M09#_`5}HhOARsuA5$<(42CNeTHFCukhe5h6BmgB9qIMSb$jLc1FrxgEj<@hN z`%e*X!IJlWr6zM3t%1Oz;LK%(JJ3N7cZ{HiyZc7aLpSoE*rW<3UJVXK4F7XQ7;K)! ztAfa2&L*TNR)nL?>x6s!N|21)jV)lT;FMj%G+LXg1>!RX)z~nkRt5H1)5&pF+qAB6 z!j`uB8?dPA*PTrtw zTuj1Mjqm=b1dw+>)ncwF9{}NesfA$BJOCx|wH8v$a2199N2p_BlzbofT)}eERhZ|b zW<5W!N=dR#6*@IoR@S-118vx4LSU|xgslT%d~e1=hs9~aqH^o21!IJIX@X=KX14&V z0=~3EG(Fhr!CoA-Lc2I>0dhfTJ5sVx9iS@4P8f#i#~R>$BWF(S1Te}NGfLRNHNm|& zJ86W;zZ>ubmzw}B9}i9h6Po|A0`>3T4Zf>`qlER9w4bbn=oG79@7DoVp>s}93g>Om zLuJEFU@wwV88R;5r>u^oSV5-ehBLc03jWs+XBlq&Vb(GW7&FpsY1OdtTI#@K&zJqd)2w3o9jz!TKICUX)YPSx9Lz>9_rU^(@O_yBQAtK4>JH8 zN<^vaEqeh``BugZpMES5<2D!WaloN=n33TK;?(P|i@e>jO53^{0$)`Pv=P}ltynKM zZ%oiFc!*d`?kuzqJ-_r|Wlh-;))30bJ^NTHep#eqHA(yYHAXOt&a#UX#^@c@v5uk@ z1DBb{kJ_D}E?ikj*P0{CfVX2y6f>DSVlXXVSSN8|gtcVvjBV(Y_AXG>6VN$;Qc(Sl zsB)eA5jfzzt8O^gF}zpy!B}mlWUM`QEtiZrQ&7rVZGfAd59zTVsHE2LTrEf0^wXba7{C zKQvA@Z!r#qSbX%fa!BRnAr74$HsDeP&lr+yDC2w&b%ezs%#xuF*eFt|PhfCH?SjA! zbQSk;r#)(;#qr^&kh_l|xMNA^2S|IZG0?5rX43j?SzBY%+!fC}1CIcqU}FNAI>%Su z_oX*iO~QhWZb#8(@I616r9xoA#?3y-^75pl7f(=p!)8YKmW+op_^# z{?P!}JckBeEX(KSS)dab+UrbRK+|sFOhcytZ+N(m{Va1?>=lwBgSY*ULSl|%7aV}|uY)EEWUh@Md&JJmQB3GYtU5y((U zOG=a(NW@SalWBfI$q4fT{Ox9k$g18!p(mdFV5?EDr{aGcIv}(O!spMCUH(MUQl%7C3gCTXz3@|xyFdvf#C z)4Q2+6zazHgD_0#6%&eJ!rWP`%~s4=Y(?eNqRtP#sX||F#|-oT_Zf+AylNNkPKJEo z-Fkk=&bok|jZq;=>t^wj*NzW&#^*fMS9j_y6MV0}w_Oh`W4)y&DkZYE$%;I9rv8sNO zYRgD9W|HzrnmtLgR%zDFD4~NeYmjk4(*t9S#OE(S!kMOV-2|L7haZ;=+0n(DF2dam zI?Z9-AZVm$rp!TmSg*urLEhdE8jf4ydRF(M> z#E@$@ySTCya6NlcxK`dq2HeL?ss=bR;63Y}_vzB!S$P{6yJ2U-Jn4b!(F1=^GPtv~ zvv!Mu}I?T(aB0uR=h=U)A zs1C5XA+`Oe6b&N#k!*cK4)*eC+WtDV7TjkpmH#f3=}>0bsT2H0QSAFH2kM19n=LSf zwK4`EcuVKARQq0?DS>D1xdd~q5Mh3AR`pO49v2djC{xdMUasU++GQJ_l@T)e_U~`o#WepLNH7&9aq}wdq%rl?QtO-Ux(2?t(j7Ip>=x6!reEfl zHJsTzozARj5$x=fQ?mkw2hYup9vCiFq{*_?Xvz#Bw?GE=PgN93>$7myuLIP3a7HkQ zW~1Gip>5IuKPzQ}Ktj{Jn4e&J06v-HclVNLRXG%rLN%SSY11x&77;!Rm}_fY@d12Q zTt79-RpoXEruhVFpS7znW^47ul8z0ZO8=IQ>a$}lz0qO6xCu5gRO4Hq!pv-10(og{ z1Q0!}$FkeDZo1GOq0;F~XDaC$`HS2_8+OpH0~G)XVllI_tf^^H33C&>A#Bfl{dl1vN%3IcLOM}$^!rlq%zNZBMaVG7rnVgd`0A(SWyVF2K} z?@BIjDVdA2F-XuYjxC6MHc4>@QanQOJWi3`pvhjuFo)MT#RaY;qDf2B)htO&iQfZP zf4LeSoxD9eUzXG*fC`QI3iJ+wGe~ZXM>F-=ES>nxnGZ*oL5~;Mubb$FM+hn0zh<`p8)}9~48hsy%{5l=9KWS$exkLG~`5%384wj%0W~fgTiq_yKsd_?frN@ zflN>t784_!qZ7ZaqfTz@@@`2g_I8~#dBfXCobC(PVpq=(AbKbdH0zw%TBS<}ruc5{ z(TaKAs+{zI#(Ndk6a7}xjkmfY*BRg>GfBSV!kB+n+*0atLbRi#!$MGIDOE`!9T9@3 zc`EA2G-uV6g~gE_q>u|{oI*(0O+Q){jtK&D&sSI zBg0orUkSpB#@X zcCfuY#>HfI-H*w1yNWv4iy7WVDnw$>fVO>|lIv~h`Mxbf${5wQMV81poTf$`9y2Xu z=Zup6P$b_Sc$7bg89u0}cM!t(Zbv7O84~<8ihi|{DT2R7Iqx+qP*$h~O%##262V`i zF)E@h5OEavzao(*|3k(9+X9hH1{wXjcvEo-@w8)mG7e#vR%CdTE?_-iN=~WK4uF>5 zMRbWygzA7Ux<#3bZ3DFx9kxJ}*YrW{8TxIVQNyi@H}qy*O^8!budmH5wKNbjoRMT* zL(v`7SKz2zU}Yz(Z5!i@BH>H9Qu{A6g84Ph5}b{HfAX2luK$kWLWt<#n!ngePf3cDwUxiPt$`AH z{2Qom4B)CuYB!u7RQplu>sLT%?Ic5hjKqW<5w3yry`W6=&lTY6+=oygY`>YUIzxsM zK(c3Ffe(CY^SOxR`QzVw({8?l{aYWLmOvpzAVE1ru{b-#85qG^LODinL}T6z4y;2O zT5ztGE0LbGYSp(^B(+L;lD0fksfOKeUs7cajei9$xMUMogXQtkm6TY=Dk6rUm?@*h z6(b)rK7#q|I>l_l#m-lphI;#=<7oh#jR>8p{m9}=mR2JqIJ~&Z|#(YxnvQ;EU zaEl?==nGh&(JZ|}1=uL*&2(bc3zBS#Q@1K2CfG1b@WCXBC?axoUdnim99%)b2+Q|I z_00~Po|7a%agS?Zo-@C)jwl(_Fn@JK;2IXMFF+FB^-5+fVe4D1jP2*Muc{Cn*4xE* zq#Di_vK=)dk9@g%(e^#j9d0A;LKj{~pS*UrrMbP`eEyhp@B3lKN3VDrWOy7}w}H@O zP`MveyS%pr{;?4KQLOF@PKjy}fA&=wgT{Qf_`0s3d?Cwv6?f;$UDa#Wy)_;ycYz(; zD=Y;mh-LGleD`O^k73!r7iN9*tA0g=$Fb;F_&lxo_qFCf5}H4XC4aH`VC~w?fG8_& z=M7LTOFyToS(@Y8b6!3M)xeGx^9a;~bzgukaU;C88_wk~hSz$hT2ZV0c%hPf>%CCc zm<2VQN2-b3+uV5c+K5ew$DxtfI6Y}4_obCQBDy~c&1CVpVHg?y;dlI$4whoT`N(hU zq-Shf*g$3^LH#06mv3V*7z|$S?TLQ}gMst!i&umF7k}8<-yQ4^b_aXA`+pehyxQN{ z{{tATv!iv-j1`dnVX$&rZs)#`$25HvDT<`A(@-Ugf|djR=g-eXmdl;p=`)?(a(DkN ze&#w4TM(B4(7~ctz@EYkj}R*i>7>%#Z+5ZobzxMVAUf25$jP0ckDw0*hReTd#QeNt z)GnQkjr9BFJ0rh87oIXBQ&pG$F7tuw%GJx?Dhrgz|7sYhm@ws_e*gcSoSQ(6w;`y; z=NhnfdL$FH^Tp`0S--M#KPL(L<>JT$;eV{FC}Ppn)~w>&1geTaI<-}+cswC=D+i$( zlugHsS^&yxq6LcSTM+>D#tcToj;BdIJillK)Q_fPfT;r~ z0HFf)0E1yjf~0eUgZKeCtcsy{BA@DW`URWx5(CO{lB9c6x;UDn2LUsIe6RqD(tfYPStEVx)0m)5{eu8)*1gR`-*ot68yUlcGt z%}P*@F`>3l)B^ikmHW1Ubc~8JdZyX^NTmsv@mN(tafqfQyF!I{SJy>(hM;o8ijwlp zTausy5S?WgDU8uOs#9V|#?ggYs|=MWUSd=mjL_1xYSpv_lF<7(`Z*nYasu?qZARyG z{jHmnWi$8h`jYTCl!jqIsm!Cy}r;P9N z`64pD`zqE`uJ?9@wIT3Zjn=yM{O%8yU|VO0uORdGj=Z;^JhV8OQJjsh;t5J-Tuhve zGg7`j{fOe3;D5_|K6AII_UE1yDyJWFiloDa8#TM99Oel<0B4!=5r7*sKM;-Bvy+Q6 zuk5W8VWVo|m~-K&j_6od-~!Pp%urU;6_A>x_tgyuSFeXU@f#%Q2B?83{&ng|;GDp% z)L0|HUORhXI8AflUnkYRP|!N%*cevL1e6eCQ7kS?&Z`XHr8?7cwrCKy z4C;2^gd1uF5?YD|3u%K5NBA1KnjrhZSoklD0_bC<N_mhso#aEE;af+l8;}x?PB6}3o5yC7h@D!cM!)c>HC@!yj z5=wGk?LT_+#uNMD`ygjvoT$E`xW2*pV=Ro-RdCHJ7#1llnMcYZOKc12Odp{+HgdeQ6X30^5mayVAfAbO2uU}wP-zOJy; z5+Or$H&jZ{tv8R`F6S*aYAD>y&Ow4dbpD5jTAUVMi)w{WD6dA-{d@O}2~}%Pi9nW0 z1Y#QuskF1M5s|aKv%jC0HyCa8nu8!3dP7Zil3Lx;jq3_3vF+}P)Q(82T=Q*H2n%)O zM5iNhS`SIq?hSWtCLmH5=w9=tuO5ka^jO?udM9h1sBS>Ul-qtc8 `yeej?r;IAg zW>l%!6+A>P5k8axVvx(@F(@?l_oy|gsADoT1XPN#}OPehq167H4*C~l_ z6#J`u3uS7HY^<_fr1L7rfQXuYgJ_0Q8QuxA8zYAAgQjFyhSFwIhUWAk29-%x(xN>l z%c$HlmZKuEtKFR59cuzqw$g=aTs=~P8cd-K)WxZ!Q*W9XDH3tlCA+u<6x*qsDUvR3 zatn$?kUYRpF~bR%pfu--2%%zv}CB5@ew3dS%5n^ z(Sir-nr}n-R@`JmW*(J5<^yZa+qqc8(20ZwolX}XZ(5;c1i}J|g^ZcqAcD)uQ05R4 z)b^yWDOLKO4*SVV(GtGysgkppH%7H-Sr)lLda!gX%M4@Jge-n^`YkixM;R4ghs^TQ z6R~*~PbkUoKdS^#y7vIalrW|u*b5`ceoc0a`av_v|3GwWdvE?A)BwdKqR{D{GXj%)>8gh#7U?hce%4u5>*i>YySz_fk zE7x4U;1!t9i_BC|EpgU46A3V~WGu9et&(M!+GKbv8pi_!QefI-;RFhIMsU^~- zb^;4X77i(-hnP&JTM@#wOmD{8Nzwhl;6AykmQL?&Mo|-wIT7Pk8e2i2DwLEY4mBz5 zuVO-4>W!hBFU%sOeb{lQ)G_qZMdq23f)yN&v%yWamZgX#bUCqkHI^757b`+4aav{e z>TsU(sU@#c!y!;(P9@MOuTr4>P-dm(s|8CdTqvuOi{eswm1^ww0x$msaw|1Jt4XtD z&J7V}{$qi%Bz%ssRFRwwFg_Mm|%EKNWlsUbDWKbtbhfo;x0tWa3-qvCUs*nY<~+VDrQt^ zPHM|j8NLcjKaijti9k8}1V(MuL5U5>Cy=4WXb&x&P}OR~K8TM39qBXhx@_8w08y1Z z8Wo5J3-0QI!a^l@+oSq;#!3J=Dh0CeP8o%p(K1_U9$s-5|0i+(Fa0#jf4Th~CTz&H zbD?D{u@1Dz|G~>wuN?Wmv%f!hlK=Pdbddi!72zIDE*MsRd449%?Qg+Rk}&W$CBqDp zAHz$(kIA-VuU^7>8t~r?k3VCJSo*O7U9eQC)ml)wj`tm0gr3g-H_ZtxU?$Sgs6Gi0 zB}g(d8p=shUYh9zN4v0t#-*l>UcBNGq&d(B&TAFYZS((@4M&6yb4orgtpm2q|GgJ` z`|kW7?Cd_x|ND44%zv8+)O;tjTj4Thp!hcuW@S5(`eQfR45)-hNdkU+|Ndflary55 z{AEl0eIIcehF(sMIT?HxTGe#Z~RVU##XsJqytGvGo1OY?B(1QKCp(6}i z1~NG%$t*6I8ElwOjuD~L?S#a$a{Ik~yR%Kv2vL;9XdC8uTWNBWVw!%HLkg#;K$P{Q z@g3`l_BR_PiA&7#VuXt|HB0qnx;%eYTI80Bnz#i}5$h((s&DvEkdb+KVbHXo7%{Lx za`|b~u5N0`ee=B-&~tJjsCsY+#CZ3r%hBy`!Py8v`5rTnL&i|DC3A#{9DIDY0FXS< z;3f2x2KuP%6!;4!ER*$Tt5FWQZ)1x`BHG3o0})Gcj3o1xhahEVOl`ks7O)yHHT_=T zlB$zNe=OY2XYz;#*dbpQsM(k7>)X382%*o}{zAjj_IB0udscdOnI>IJKsT%~bu0X% zWbM0XrQs}WX+p6erLNXSD%MCVwl-qAa=B1VWj1WoUY%()>0Eo# ztQ09R$zoVkZ`RO8EzeuhYyp(Q^^9gTm+EE2W>F32H}H3YGZ1Y>Ar8+-h`pQZdcA!* z%2@De*8c#KKyAM#csv=xTZmJg%X*nDK#TtWVsC%X)&KVfPv^h)@^sMuF&j?cupmP* zEi09Nju^;D!zU-tRnf@Whxcril>O+>8 z3NRi^o&>yEA`9uKrtt}4EyLa z&;+Ve-y{P4hRKbld7*A-w!nu8jwfPMCcJmVh4+}0!PEd`%D|v$LYpnu?>Cp9#J{WA zrF5_;BZV3YR#2RcBTFZG&`Grvigz`a7DmRILcQOq8riJ>-OR4hkY2-hr32u$`~Us@ z9asO`89bf;-pkWL|8tmN7{g(VXn{w7TPYZ{=`U_{AV!jtMh4bA9eZq z=veH(kB|DCQf|ComAOjv?=8$NM^P_yr+vW{G}=4XhP05=?Ru6aDU!};G0ZsOW=wjx zb>M{MLehGnXMy2%Z8X!9bhRB{#eqdRNMFPmM6>)iBEd=Ln>*P5?+;$Ra_zr%_Fg{a z|GAf^gZ%fHz>LTdxct9VI|OZ7Tve1vDh zDAZV}KO(62a{V!g);%}U_P5Uf_yJ{z@@a1l*onxQIz?Hr1%A0W64!7SC$B*XyU({f zIvY}T%gH5v59kO12SP}Sn^`WglM-lYe|`WWGl`t2Eq6aZq4@#0L36`tN_JI!2OFqA z?r(|k;Xmdw-5$eNFBA|(aur`NYa3qKz7X)%Q#!STh<<*mNrOU{c%qq~E_R5fXVc&jE9hax9e z3!t6B;4QXUhlNln=f8)@)UREGMI8(ihfr7|i-=X^SLNKGeFbuoI1rRui&fq9NNlQG zx!;<8DFIO0Da<2v=Z=LN4nFkTRf< z1X9YcVD&~ZriZy6X2}Q!82!!xav>tU<%AY?&5-Q+Zg1~}U7HJ?w^e{M<)6#Nm;p@4 zF$$dpnV0EnGgzA8yHMluMcKTurTrOgXnq3)ux8qH3R8Snd6fvMoWr50+ zWea3PmP8+O!gmrXoEbyK=Cj2#l7*56R-eN;0~bykSn-$vzFaTO>eb6oiofb>O@=>v z`Q(5TJTAHOR3FzW?=jUuuex3{O~l+!M)w+4Oar#LY?g?bFIBKG;y*&7#P*9 zlqOJbxwIhrUk&hj8LgnX3uib%nfg{i64AG9el(X;D~*`@&A>-2r*&}TYE}zMeKw%U z^~~oA<|E~Nu~3@UfbjA0KQ>{rV&*+>eGS)bN(LETTVhDJIFuUtX{(A?yf*82(uUqS=5t#vd6#x7vs6j(|_@YOOJf~-D; zi0*dE`C@cy@ZC;$dRJ>|%>*|JShk%%pu*Xg*+-l5+b`vvTlmV&Si>ynuV*+-gr2N6 z4L0!G99Ck|NGaunsMB{!*2-a8dsc7`!4y&9q+PyJsSQ35dXo~E^zUo=xI`e5 zd$?%RvrFpImM{OUd&Q@0eO@N9ig;^F&{v*Ph?h%e=o7Ke7R<}%=bJNxqYh}(mUXV% zdD@b>==AtmDuiKDsE@L|a%x3(u6Z3jQ+@TxP|EaWi4Z!1TY&_Q!cs#3Umv2Ku8r49LVKhO=wK`rN8Rt6lOa zR`K3zS5M_V=_XjL%LL%(vh_3NRjpb#45q}q{}g$fHHl_`V;bq5!|2CI3FMXwq{$1h zh7F~b1fqz)9SfZGYzA6%qllHPp)(<#jm}ANK@qc)mr?{19L$|llRc+Q4!~>9|LJ}! z6X-Z+CKhsn)8Yxzet*`d*3>!xj37>D6uqBN#3m$7q(Ww3fhZONLss4~KGvoRYU^$& zUC>(k+kkASr!N{A*g00R5%m)9%Gs)}cN~0;4Yl9J+Ga#kloTW;=>d3ue9^|I!4xcY zNmyEhNMo9~P{x}*P=yb=<7_iObP7llzNNNFL#p9 zyn*o28M%k|cKQ0kElCmPB@|b)~7N76hB_zB{)TYLOh*D*E9kmbBmYG}nG<Ft3K}yCz)CMC;rs2E!j<#+FBAspBEMqjs+-+%t097?5<2Ryb?%8<1{9>3~oSlFF zF0u>&&_4_q(dp&oyG!Z&HE^6|?Sg+kx;&TVb{fmc26?&N#&Rd8uYdkQ7JSiI@cre{ z@u@7h=laU-Hn@B*?glYb2Wg9X3~19)l~A*E4_L%IfQ(dtFA1XF7FM4#ALY5YvXhqv zGA)fZmTxl1m39Df^^<=~qfm(jSP`3@z6?! zu_2?zMeSrm7GMVVnAOeKtu|>Dp-m9RTLXgRouGl9V4$#Fp8OQ{?9{nB1(hG$)wDs! z`rB!ll2wInJ>xp1p~Grk-`mqluF2ONd4tg9${SKrT^-oCFbz-|*!K@k0*z9`@8?## z4sQKs+(dmn#|$Rnb1Y~RVqadDx1lDtE-&uJk*j#a|9)p<+_ zTiH>)Jh+Zt&H^g8vbu4BjNgOpiXOTjly9M`isGlen)IZ7#WBg`^kN4zk8qoEkCVj4_<57UmpQAN}!_f1|T&>%YFvZa4 z&$g+&AhTRrj8l~7dOFo-*q4DmB!v!aLy+qD$WVP7(TuDbTFWv;ig!U+B{x$7#3>QO z*CK$4ZxMI>at~#(Tcc!=OLU9T2gN$<0Q@z&-HEn9bh~qn3b+&fDvDc6r$E=3{R*VJ zTng>kp5u-oc2MR3Z&^1-%B|8gW~(2NmCH4v1slqf!DWvt-oLprqYW`24U$$nz7P-s z%9nXyxhC~89P8jt^h}uN!fMt%acS=JVh0EIkgiVwTlBcV8mTW>Q_^E*9Wm&4bvja) zMSdO5UF%t|9U85D**1<_I9EQl(-u9N{!xoOVzEPR6)E$=f>}#bKZZ4{le(7o%8RqE zx(YXF?pC-F^Te;P`fUllhVcr4ZZ2OR9am9pz{ZtjQMv^#Ns6}gk(KX$wH{()G;?+yGO9W=_#yqYN^r{-jo7EPK(If_-Et*}gS#BGj zmkEYoR4-{{?a|7Y!F^i!)e_bw$_^1NCt3Gfs0pu-yDYSw_bP^bY=WrXBZ+rN_JDWV z>Ko^nKAaU3Lh(P{4bd0V!E)>3uw6h6b_iljyH)6te0NaW^#h-1_eP4qgs7)T=l)l` zq~z?E!zpZb{dcil69PFntwDpQara&bCY1K3#?-5a40HSgC9}MN^vevAza~Cv?tQr7 zF$KYrbm=mLmXf6cl6H#!X(c)v&M00i3}|co=Uq4d-;2TSQ~c+9dAjS`9upX5Nq@{? zF&P%5EcB?V8(DZ2Knqd#j#BOR0d)u6qx6RAQ7Kg2PHLtd8N`R zJ(3q1<`j+aN3PUGp=^Lb?N&T%&omp>95Z8p=x~hH{uanI2h#8f|VmG;`7l zG$POO4)H-PL}hCweoAm(M6Bn~(@Fk&*17AI|9dY7j{M(!wg2)-{@=&bUHdl(_a3u7Cp*Xo$vz7#2ta8LPd+D$=3<1}3KlaGZfDy2jV@qB6bp zFcidxNdo#{pObOv2+)nD^nuXUYa1YC_>Yps2ZduOTU}}(j1I1p!XYpyZYb`Dp0DeE zI?4YrEYJsdSNXqpcRl@o@an}={=a*9T)EP)yH;Fa9I~r7o6dxXag6e!v~w2uJ1|pm z(Gocg;~Aw~T~#3DDLBlMY7^Z>6qG1+ICfrHC#Q(b(!$rqHQ=L6WtNh?CaS}BfAz6- zd%sny<1(hOr=arlP?Zw{H`hP!9_kBf( z3Ctrv{=26ROB#>=iU|S|hB(eHZ^p$=V8+CLiI+Jx^1!L&4*Gh0Lqrpp!VHcP-Rz_( z1hK!b<2>G4>v5i>hAQObES0BVvg%UjWflsX0ocP7Wn*c8l%lL`(K^?)l$f@5R&o&%Hd|<-d7;A2>#m(p?)2 z&L~#FF632;Oc$kmAv=g`v$62rZpCEK)jkRt7bN$#y=vQCDM~rnkb{N$qAZA}BB*H3 zibWv%=WWE~d=}r#7W~kVW&P%@lmX8$%(Bt9!(1CJt*ssFdfRmNk4`XCiE5r(o#yad zhkE^M8{&gdTizvkKSeJ{4D$V@+{i4&-la$|D4@qvr@;Y0arx_8Bk3)l?A~lzeux@cxx;ajy46bLaTIgCCQ&jg(Ef{v z>ttk>j8aJX>XYCxE?`Px1T+0D$ygQbFcOkktof{ z_D0$=dvE!HZfoXlhxd*+zZ&n)pP%)BC?>tJb+(Z8h|7{B-k!ZD)+^mA&c+lmrV@@f zhSQZLbvhP95E*WRpM3xDrDcn5ut?dkR5_Z~VRr`bMv9aHHqjqp+pavVdwl#xgig0D6ZP%eoo$Lnh@vb;+c3x5Y6mtcrs-Fed$7uIQwAXrm>7t- z##T7f2CNuf=u zKV#g}l)FBw)uZNOasmrzgk8{TzY|=-t>B#I@cCW?+CI?{Tm22~bjir#6qcu3Um^EUe4)v(Vh?WYI4}>w*=lpA z2W1s*E=Q%}YNbp)bjL(apEV5DgLKTS&|w)eO8Ztnb!{&f=f*_Lb*tC0=H%kMtgrTh zacuNnyEMBE3nv-u#>~b3JcgZwqN-^x?!t0b0`R#{yjLw*L3Tv>GWJc~ZdbgwqU|W* zY1g#E>nPzAhP|@VZ4P>=XLr-B%L7(_wg_;Yl~Rkx??Vx7Rws8JP@N22nbo1(vk!0T z@8RloR9U+6mGw6yK?&D(t0|CIt$!>Ldv z=lT2@xY~v~43PKq?(hDEf?%!(Iz{qp$jTUMfja$5RjyA_P8b&EB!L}}1uh`VH5hQs z3t)M|%E~~Vws~HOr0TjE%sYhRxe;Z@M9Qolz)6!u2UhQdlHjouJVvYt#Ike{ljWGp zFacCeunij%nWFh?CawoF0uoUpavIod>Or+wZ-34HQv|oL2G63Ji&w?2Z_pUVb1;Hw zdJW?n$+61mO`TB+WSPmgb@rVUevp*&M%sJjgu1w~tB5Xm{^!m}OK9TPW#4rE|2lOZ^;Yxw>KUBl~UzIWhTV}1z! zPH-mMko$hoHG#yYOTN0ydfrdP4*WM6x!fR`1ZQJ#os#&*W{FBvhUBC-kHY))1$NW| z;xHpaK4Y46scsq*9)S^=Wl3k%scQO5QlVCr&8Epj!}8)bvPc`o3eFE;5&a}`7ZbS+`q2Ff(}xj^$}6{ZkJ(Kwij1cc&D{u!0qwB zUb*p~cV6s0-T&Ulv%oTC`|mXPQ);gmQBrPVlmOGtSAbBxwxf@jiyl!*-7cxbP+b-$ z8>hSWuXx~)o2IW-lNEX2Ea(~mb&s*x&|I_geO>cU<~fh=SxEk~4Cgs27F_~b<^M}J z{`2n3S5NUD@8z-Oe>TS1M?HW0w$-hbAHSzq;Q;*BZP73*Yd*3)eP1>>>v6QoWd$ZO z(=bfI1VQ1vwSoHMKKSwK{naKHa6i6(e{lscD}DGDSxUl~{@z{pV3G_aHNjOuaa{D? zQ<$-w(4u#R;u*!o97I$2u?NTK@WsylixGE`~(_rXS#({l}Bk{}P^# z@*fw&a$C3X2B5|MbFlMj*OmXfulAqh|9w14c765LSKuukOQ{ETcKSQJ{oTRt-~e2r zDY-?Um|zAbD9sU-at)>_zy$*{CTzgqs6bSfipg|}3qTQ!Cn6bbd~-!o?4||jwB<1Q zZWvF=NP_4>0ESggj1FD zI>8jh1)&%*!{Ju8*llrX??~I3ny~`ODu#oQJ{j{aO+>ssnC3V`juT6#O@Mtq)K^zg z5$}$(Bap2f+TTo>-&1}>qVGL5Sf4)Goe{MJsKu{~E2k&%TD#zo=@e$k0r+dgCeap% zda+2G&#?f#OYm}k|Ha;c_%GNlrulXP3mEt;1YQGC-Aedx(P ztYKD00_sl!oN^~N9c>$cYSQWnIaDBrIb%bLnAnD*WH?IEM|_>~rez{$N+K}b&rjbBkKdhNy}vvgBI4|IJirDZ2 zg?TRG<#nEDoGo?zo~pmZbn#KRlLM4$q+OhQ9HA9>kPCh}V3d$zI4WiTForq2#wji^ zV#9=zdHFc)$=kE@NdA3!^ykP=J>IwkwXHB6W^j1wf=E;C z&yr|oe2sS&SV80MYn*LgLpBk=dMFnE{wC1bo-r!E$2r*cSZAB@PoPI{{lY`>?c2sO z6s07F1=_ADD}L%$L9eG)1RD^YIt5GgdbhzP+FXTsMH$&jFq-a@dfFyLE63mn6=my@IiX9Zy~Ohshj(x5obX9r+s zFjYUNXiDguzj=#m&D|dQml>h5qo<{!>$6l!n zALZE6MpfE%Jn}+1 zeFDFMSytd8MZg8qxcs;t&>=J*HBHJ6xcz=;R(yIM;OQj)t<<9HxBvCyKkW`)y?m1Y z_wjVs|AYBu7fSp~MW}6*00~MO43)~cIH;$$I7VRfk0k4|6>w1UcLd^jblH6MF^oEX zWXcVq*{C)|Gb0R;ig~?cbbPJ$zq|V{U%2}J?yH>_PxAjho(1GTNdng-vup_2aK;c- zNdu~tVP`NHi2pP!2^S=}vhU-oeDF+MH9o%NXaK*rbIxBp__9W3$guE*xyW}-pedz*|+y?i`F zY|h6O#=OZoBuEq6ZmDtHTL#@R$Y~h*Je@Hx#TlN$v`Q>e`q_$n71KPOOF+d0fy)v@ znR1ql#B0uHrK8VP0kZ_sq!ZVyOs92o4op+>;ez5@oT4#0Wid>JUqfY8W7_aYql<)+ zT>jPL^=tU5D^D^MbFyDhGQug6aAhWo%UM?7DLRoc5iHJpR@(8|yulgbq`WBo`Rru- zhqDve6xbjXP(q5$EpTylWsa@QcV#4r8#HGJ&c{9_0D%Ab@Ba&~2}!qv(QA)n>0E#Y z-(P_Z9m7GI-8t1`m@`rRufNLk?JbMIeV8N_1d$ug12Myi!~kc-Rz=b6Y7i<2qURn2 zb7<_|%rXF(K|wK_M8b65n=RQ%@msJ*OK0t&H5*D~HgIR~J403y>8=bWB?b{6t0-ig zcJ-I5;rp|mCQ?~S%n75Eobht1 zR75e^zL`AMkeP%sK^!_W$-7Sqt<*I}@}EA5Q4lTF%j8d=RDgN2*{TEHdjN6`Ceaez zI*)7;?p(Ydy?Ijt7Zg{As4z)%s-B#I-#|hBH}g&;EpaanUPZKFI+x=$2;pn4Bn#9i z*BL>_8tORgCTcIU`h*sub?C)ZaNC_|z=O4@w6vTKcc;8+wZk#xwGDaKyD{OzJXDfn zUG!Ic1#-n~HaD#@8aB4;9*m{5v9;_IGExkM&qIc&Km@kqFgm7q?C-yCo5i^zTs_)%@{1OSf1u9?yWlZcI~3A z*(I)h^Tm*$h1`;;6*@(?C_Mn*u)#OR^!$e9@v!*j9y#S+nB$Jd{m~8e+J~)D{MIv0 zbe3f}8ygDcs+$-C!5W&H%B!H`+Nq>OGfIYR&W13V;_N;sCm5_+7K8{a5m=F~ZCzS; z1=2@1%?$4EB={adof7d4v)DY}%b->*+_mtk8``^)XWe?XZqeqASPL`Uqp}ioFkEPC z7gkkZnp(^*cFZ|Sz+J6Xb)9@BpU&RAQ73yB?=O#zPaA!4 z)}-BR31-s#BNhQNMO<*A#OASkWvz-z>#*{8!|6*#!Tvl97n%oHR;29EA=Y~M^bv96 z{jl8QW31mD=d5Z^b{9fyOR$t_es*?pXPopn?b4PpL94qhy$@OVa3_9?E=zZ2Gw^tA z0#>BZ7pNjG$B_48|J%*}TLqf9oB01PUhaGGpLbq7#ecq+r@PRzm|zIAj1*!bl;#A_ z!E+T6UlkPr;QN`%J`|K{Ix)bNQ3DvTyU3ab#Cjcq1c23Jbj3pnvOYAIL{Vi#kYzj= zZrHzF2_g)mHo?suCMZztn8R~$C<7uG-c35H9r~EVjA4?AxYa{48l`;r4i43wR0UCm zD=?hRc`3PTh*L64dO0PxI6<@r|2d;5@?Os&@2UU(mgN%^s&E@8k)5Y>0-23Mw-`BA zPR*9CE9;DwI0{BWx@K@@l!zgx7(<0UI+2-sL{r~K}|8gTY zk}!CG>nZS+y|I!%l9JkOdsgFdoGi(VR zyU8mLuN6BJi=wEV5as9Zs!==ygeIa*Xj(h&)e7N0g0OTK)UCQ0Hg8V2PlcrHf$#kj zUZl0lSC6p&ud@F#DjBEQY&_$5iZeV;u$WJn@&kGsf-CaBAAV7*|Mck>AM(FH&0i1u zuQu~zP%4l>NV#T^Uzr6@!oP%QN<#Tp=;yfLoMiCnmyWm%PE+tW9gd;Kk(nzLoGk*d z`xPhw5~WyeK}7UF|NVbZAxHxLu_F{s3W=7iKrznL&xq<1lN@tNahfjCjAVpklV$&(*)$C9f?x`PN=ded4#Uc*22B-;#oh3=z9VE#{SK*oDaSbd+K-|RKx&+&F9K@R{|%$?n^EnXF?`dFK%_J#bv!*7 zqbvNcg72$o4ojINg1kVbK6}O(X;AipX0w!dJyy@cUP5jsKMv2YH@@jKfyXo9BGvpp^x1!jjmn>wHh(fpuL=QPJ5k> z9xjxR$~?%Cs_eccZ>r`hHHGB8oltUi%JhAs88~PuHQDvAdqyGzPH5tm#l&$dY-Og}0+`&0D1wn$W$Pjxz+_|M$a)iFs zg6jry>c+9NAQC57;-ej#ct80Qloe{GwrAFACom=m=ND=@I$m2*k#cV%HW7s1>BHV- zxrex=xoA-)joXhWXdUeHLc=(nQ6(@MWQW*nC>9Nwx_ife{ZbioNaol78o-M9?`JPxysXB5 zKmU;b`Dy-oSOV3VZw&dWr6Y12ldoFCzvyRpHL7~_2?cw!2++RAAh5Z2%Og=ioY%KA|1#)eOC?@=!utv`&H^#N`+go)x-3U#hevsa>*HF z;5w?`_6R1-P0}8$(C{)t8O&ew#T1Bd_?fMrGlxhjAhsVfjiUN}jwWa$3vCZYGEaSJ z(K^(%0kS12+Ynbep)lEO3^7 z*Z`0Ip8Xq$Qlnx2qHT#@OJ2mVt)bnD1S=Z~u9obT7jUij2r`M67Z;=$-)(F97D;vu z>A&_G@4=K*wz3LLO`;00D|@XjKx{fCB%#WQORj8h1!ly{7xBU|1?@-SZJ(fV=_b_f zOsR!?sW6dC9Tn_(GZ>h-35M8~7N-Gn`MDN)w-68LVxa#2#0SAN%G(=r#W<9LcYae)>~*FM!np`Ox8i=*euA zqUI><&lch4vDV;-IXKaV2Kz4LnJ!IWj3j8X^pTQv!kNGKAZFCTsjw_nL&UN5!fZKa zQ;FxjXt`*!{#6B*6OJY0!!_n#b1`HaZ108f4_>4LCAjs=NXmV`^i z{PI=T>PY^p5E7Y|NLRy^3~AVi+Au-)y_Np7^;rZDT>^G6Z42NANOWVS88YP6-Ydcw z91rwW$o3aH>jqmYE5p%UL#+8;|Nh-O5kB3r%`BYY=YQ$)#ogz>8B53*Pr*=GK2rVn z7oWd)QGNe&@Y!b{-hV&IpIP>QBb-i`NX<7ABqgzA90|#>B(vqHQ@_EA%Wyi5=On(> zB4a^}EeNmNT=he>{kY^nQ?ArTmPFxR8llLhe~T7axqJ~S+nhQgXfHFgrO>ocnZOU*WP(r|HWm!)~l{mAqe*G360y#dE zl>OU={j=vBdYbHa1H>{&7t((QBb+UVnSO12PtzoZD?KZovV&U|?g?r6nAKZL<*wU( z1n-5Y@*22?b(Nv^F<=?TYUgrBu% z-O)Zw6jcf~7?ZY<{SB|Z|8$*`m`-VY!4h)9icEgJykaS7V3m?!dvIE!bqB$M9deeE zay06<-x@-%@7^7uD0(ES-8^hYt?mApG`h990f{zCPDoqH<+hYEZr+$XxpbOhulkFj zXYYf!{!zrWQ6wsA%IpjEixT}tg`!x;gI@t47*$`9F2D0vh&W4!d5Sagh|A87cmq~H z6^W0c8CFKUtkor13RmxJ_Wa&9*NQMf05?LrvdB|{MiqlZ&WA+?fmxi=f0D#}$qG3% zDI$hBXOomHM6K3&&EzW~d;;%&V~Z~!w9($1U_mx8`g)jpI7z6EBwlOqR9k#{c02|4 zUZrnu>7hi7&<#_+c++3s%ru*D}rPTRKCqpUd= zm!(;TuGNFA7UnDpA&<=wL}Qa($?2jFf3Nklrj0I;rgcgCQFrW1o5RsgUYk-5)WDI? z2Jxd=x<_b6?}$J-;R`DCX*+~N=LF8vA?d#fBIvZd*%C=iGbzw+A?j_~* zMQ9jUVH-PXv+c^dw(@MdX{A60Cr#~eSixbD?u+__1{fc+Y)WTGNNww!CPbhKN!fj%O$Lfcmw5{2#lb>CTLJYet=6Wf z7}VkRT)utT-mc*c`=C0!S8fYALdqi1o4?;!kUMV`N5PTR0Ga1n1ElP^p^#XT;X+>s zfL(IbHn5crbL+$fFJpR5-8W{Z3`a(@87D#vjZv2pUK6cmLjAV8icH(GACcF_bn_&d z8UbyLPckt;G2wE!z!{!}uNb@Nx8}hr>P=qQR@&=shF{u^duoL`yT>%C)nOxXF4h<< zVb{B4MZv|*Q8m%SINEEu%G}dLK2heHQ%`c4ld1viLzkx5D$+$+wFaPISU@&J<(kiz z6HXHcufdk}1BFgw#>cwT*mNqV{i!$`rp*Z`sKV%*n(FM)$$ojW%n6E46e)rHMAgAY zp)C9e8dA`NaFnrZs7G%A3RAd7pN^tRp2*QBR>xw2zAG`Sd95~G9=n5==>5`2HW>~T znj!_UbgQdn_^ozH4@;mSCT`9MO0Gav_`bn3g%=ZUu`AMOrqd{sMBCYG949n8Cf$b# z)*4?OlWz1+9{Y&F^6Kl5k)UnWzFcH6;|a@1;tUcaaNG2B+dS7*mdU$!J419!mPf5Y z6dl}H)v6T72pgAU2}H)p5c3HQK)TD#rKURw*{cjkC^~yXg%nY(OJ}Q7k`gt{6PzV9 z!IHe5nwy5lg=8YesVCxweNCV}NxojzTeYb}txLoRc7z4VWx7i~ACNvCn-Qi?l8^N9t zHH%_++cAO*$gK^nBGP`v7T6SEUd^aB8`jH9kF|F8j{>JVNZ*bDV#CYrfuL;+G91(c zpcr01%7pukW@|GUoRX}X$7Un{xy_;*(4)-2)>l$+7S+xDCMV^&@xDp+cUnEArLuEM zA@HGpL4R^_{_g^`8aUfIXK$8BGHu|YIHQ5H=t4sfu?KSoPnDl>a`Fl}1lqoT>@a>0 zi>KL6bn_U0g+G1A~+xB7IW6=Am) zW@T&~3+lSpY>0IrC1mWw)T0*DqfvUn>hygB=@lka8FMOWjMGq9K}545CpcLeox`~K zN0BM#5ng&0yykLmL0HQqZ!D}8u2qD!Y=dV^>cMujVTByB=`f+~ZZp7GmZk>NW`Q)t~+lqwK^w-P={kcBZqH*+c) zPO_FR3^7@Tbpw=NSaQ1&c@vXd24?a0MRKhZ&y|8{i?lS!ree-0rP+i^covj0A&37c z7%vuWO{R>;WJuYNaL#yJEB#b=s9g9VyCZxJ1u_IR7R+_|WFuqfoW^r> zj|B?!w4O(!#RrgcW{&TO>K+RU+(rqfcZ4HOW>n~B6f8qjK8lWkQ9VGpXyxu_E^vcy zl;PFsU)T3E%>y}{FeVlA^L)*%t?ztRB53W_jh|(hPAW|Nsx+e~C$Ao1b~E|}1MNjO9EBNpGulxRg&XRFL1ux*S*LYGS zGa}c12e6|4+ri6&gKGV^PY(}1-2Z=yzaIK4PbT98lLgBJk&T&u?UB>v>DOq1-=M(I z>V6tkhTM2TY82K{TmNFz0>AmeQ*(q45B}}3OX~ct{#LAiPMI#=xmo2${kLlUe|}JV z|MSHcpMF^XPx04h{g;FwG-2#7x-6E?1`U?V{wGcEDQ$C8!m#_%WEtm*yOR%65%&4I zK?!xkI#WJhmd6q1eXaqqnJ=oaAHT9!zrXJFZ&NOavA*8{XG~kJ|9bfOr=QmL|HDr| ztp6wZ>#_cuga9l<_k(>_{iXZHTuC#RH@-qYlYCAVkP6B~0^o8h5mI#JC(t?LNrF6J zL1I}-wpPLfVNOWX*ri?ijHvt=JHVj+>P2Mj9f`|hM{9+$pUE4%$W!vm?p(@T9PRIy zd|`8zyHADG=i|ZMm$_V|ANg|M_QDt(o(~<(ek^;nf?j8>?5`H}aua1kHjv=pvx8SO zs$JLg7k5jEFzX;fMLH}+hC|e)8EAPB*En@LaW+n?b6A-%Q6*_{0iK8@=lY8vh;_Exr+Zhq_ItDcji8*e5*x^J@x1#q_`MFwEEAn8WH z+^?5%b1~9zyd+H&utp*B<7||p8r2UFCC6?rO;LTw>xfcCgLP=da{m}h6rEjNy}mj^ z_6n)nLp&SN0v@B(_y*eat-gvntX$l!TNh22Jl5Xk?{YjAIf$Q0@b1qmx5S5$T0VZ=V4X)u}ai%(SjM`>z(V|Gv;laT{ zWh>p*)U=pJUmB}(y}1cHsp$b=>(5LJO`A^3?aFEKEzn!d{${=#hcbZuY9;Jf9k5q| z|8^z#Z@a;-#Bj3`!_7(zE0J8TL~_{&$x1wDEAgE5#Iq9BU+GF*L6Zuu7cW;x)oS3c zx`6k^^X+Op-}c1Q7v;@rlsD_6?2G+!HTKH~Ved=G*=j=0wna!MO;8cz71WA}{@R!ImeylOQ~uCxF(G3$x^&;aTGYAYjJXq|sG3$MwtNvd8y3hYbPU7*D@$sC_ z=3{e>N*DWB;}*b*^Z%DGzWA(K|MkV^FF&0BKgC}UoszR;j5A4HB4{lU*e6pjRn3q@ zNL?DDC8U?L1X)lzq}O>9s1$`)2&r8HGC=i)ZrVWMSe6`Tk{;WE!B0MQ za5dRcLtuT5$L9G*CBds5x!Nl#X!Q1B^Msui%1txZ<7qZKhxZTW>$5i`hRjR4&rnks zCiF9b5;d~#VPHoM&(H_xW1R^9=r4{(m9V^iWB%@)4z}plm8i;Z)wC3ns(2 zmnlIEeYn`F*E)*Ko z+10KM(eLT0v;;FAVsyPLO3WTw`j#-Zt?9N7?e=i}wyB2OK*xoga*RYJ&F~N=g$Z46 z2ww=#rlDrjnJaE8LKd3mKjk4PIWDouKlVA#N>n!d9PU0pGrucvZcN`l#O+68<&S@jI(6SNuJUei@+>tv3oo$|1wQ!d%??65{Cs4O#I6gj#O9!Uz2t$TdJ&wtmENtq2yY_dzh3pM*QegRAT6h?XaSmhPuA2;Xiyl{9D$_I`!2GO;@q|eMUL=&#@X~ zr~ex0ah=c>5I?<6?=|2~8{GbAeU4*3D^*+ht(sFjDowqE-flFPAa|S2I(!dTuuHw7 zFdum>(fMD>tw-*MI)X>t-bxO8qw0Bh9fJlnyBW|%1q(&g0`8SGXgPmtT?%^Y<88BL zr7rHTjT7PU zr}xw{CO6&LAJ0dt=#*M55TUj*$#mX!ZzR1yhkL>9+{?jxBx|u*8s7Ke$sgda;r#FW z^OLiS>j42V{?4=Q;pjg9d-3^U_5JVR%Y)B8y#IZYziN1`_II4)cuvswG$xrKwQyYU zXKzWr7#$4I|AVsv^Cfz5aPYFV(_G5@Xn+6y{(gi3&4}^YeyUN4eRxuFb9VLW8XaGp zqLbGbr{_24uP?6AH?ObI57%b{bai%l_4@RO6ZQ1~TAiL>-&~!4{e$`j0343csVlWF zM)o;SWJVMrF~@0&76j|kACmBePH-8sOy^Y63LjE!RVyVaygwxc3 zVRB9kfs_b)J_dKG2MBW#T&zZuTM3-XoMs54dc3eeZg}k zi11u>6K>}yGEgEIia@(`Pj_Yagbz?cIf*5Lq8$3)P$eqHg&?Y(fuer`4)E~Z4jkyj zb3@nw-OmXOf3nonV}KoU=AJ5PVjS&Kx~Hc{%xSJ5P3crFQBHWQK<|Eb@Nav7j&Wib zZvhojNSq~V4#XUDwfk#`bPr8PMy52TI4wgD==4+m6DuOL%Q*atM|(rllro*Kijog4=6VNz36G57&v=7P=G+zJ*KBEK$CJ z&TwWYSQ{tm=Sd;K41;H2B!Ln|Y%1@Sc)6D+2^qMx2Y?J1-C$tXfABtKN>dW3kJpp` zBC)KKjx&834{;)j6xN3>)kESr&S;G70+AeNLSNf60tMfs=HnD$q}c=D2IT>|6P(g8 zl|4!!+b$4ihB+e{BvMYw0}i#9zG3bvI_b5$APL1_#0A6tk@4G_Qn+XQ7AOWiq$JHR z0h-xixCNkDXNI-F2|+$xlwSXVUK$`ssc2FjD^bEuFIerCF-c?M)SIzW6NOu;6)Dx; z0vxi`8W6iUL+06$epPM`!%Y<^T@#GH(dCr(JQq$0r*~M=JAxELL{yakg`=4VX7mP< z=7A;LJ#AA4KVwkf6{j1Cl_Fbetki?kG&9lH!a>vrZ66@q}^v ziE%s4;c7C#)Gh}P?1B3j-OpJHivrpA^9^%e*T+^*O-svvfU4{>+|#KRwU2QR-veVf zSzzkcC&@7fNl|=(VJrwI=@O+hy9LHgXa;hU;RV^VlMZD?rZ|R8Vi4?+j;S@Gpj;N4 z`dL3ws@&}HjWb?d18&I%_#6Z613|NwIyw}v(wu+{o9vFlU6;|kfnJQa5ng<6^5c#Z-FA*DhlOKru0UGNht5LOmmPGo^oD}MKkr^!84 zwnoOXAckpo-&oQ4V$B8X*Dwo44CuD#q=j4vXkczvyn;ZX6S83JW78xjWFjsP1 z({x^)=fxzYak^B3nx}X<@ZaQw>#bJ6H^#6C{kG7|ce)siC@?HqKAo}f>I3A)5e z>HYy`XV)Yg#{xllMPnjSm$7FL<$A2a>@Dyu8R&%9%F43@fgjDLQ)TBdBuJVL%zx=3 zXI$!uaeBZQ%El@LHDgBx^(Ct-%w&!FUtuC5jX0|+QRq5S9Em1XlR#vdE23vxq6F$dXD!@-nv?_QxLRTb=42=Nw z0x!L0t7@T`<<#0XC4JOk<=Gkm%3~`Q11(2PSFdGmM=ra~dJAvSWCQOC0YkiKEeO%G zJY{Lh?)Cm|b=T33^Q-psNDCpMXfTkc{S8iHnp0gPAuxTNcc}hq1I4gER6U75!!~Mh zP6C`daN|vB-DkiIqT!S=#{p(eB&ONG z+RcFr58gqxtPL(e>=1f^7^oHIcgKN|?txN^2~l>_ATTCCM5SMDW^6hH(LhgCrIsx> ztvxLaATDA0+Bo5Am{9l!Rvedp<3h%QuMV`#v}Dg-P>1<9j-Q(9=;HOw`N>&?;B-sz zTP-_-1L=ed_AgvrfeL6?DK*Z*dxmI3H~}Bi3ms`>t9|Ddvfn5*&7PO0ZDwR2Zc-$&-kFuWo&J|Aj)NYg9a{5D=P&Ih&#gdWXU<@$xsb)>E zWmpi*N9bm5&Rw&T7#gkw^9{y>cBNgA52u0isw|W;Ic9Bv##)wQ^KRR~NuvJEmA?~8 zWB|pIZaCCuB@8t81x-q#1O61w6jG8U#lo7cC5f?$Mf-ntI;%Pj7#tM5XA zOmJ4jrVZ2#{IaoCNt#`eFgoX!62McagJ_;7pmthS+J&9f9kYctrIj3&!ZOZNsdxgV z3Hc119di<(eN}G-iV^xDONkILY2;0w(wHju7a#@)Brd+OtXh#lFe?aSwT7;|rLQnn z13B8%n}pHlbzC!JW&xRjz-SQKbWQB(qwasfr0V2OZ(x6!FzuA7Wjh1+PHiJVnkaI@ z1xbiL2vW-+m=c3XTNgS&Dv5WAW}IljUz!yH9vOkxAa03);&lw+WQMsuSE~9R<}Bdz z5xTJkh%IPk?dXIN{b&qK?%+g7G2R@gY4gP%W8ei5fkh*fL&)zaB*%k~j3Xn%y0H~n zOYy)*os64mpTvMnaDoMciz<3b;pQynOQ42kC?O)|bYiB$IURIH>+xB;n(Vxp-JoF$ z(46{Wgifgd*NJe|>PO5KUzTnWI8rA|?ZATPtz0ax24EJ!dGS%OfuAt5_Jt>PSJ6ta zdy^A7x2`pnqCB~K2vX^x==d6)Uq|Tc zkL6!~gN`qLLVrEKI2|BD^$Dx<#}#Wz>y7vG+}I=i?TpjT&CC*K`k+#G*>{{8vQPas0yoZnoWUF+)u#|Fmb z@zu@w$q(NjU!lt%t}b6+pXuFJpBAPhRlb_YStcl)I>5<{_6p15g7ciSoKt0h9QHD%u+fdSxoi@?zDSQ3gx=U z&q{gU)rNvV2}$V$3_2ig#+eZA+`^)i2**;u>3H+{(Au-KeK3wDcFIx;h)qNQrWP;o ztQ^Bvz3r6%e>ngi-TELK&0?A;!&V<9C<8{vbSX|PP^+QiIaVwp9AU1{%+)UL_G(e2 zvg+spHw&j@3jGbu%wz_7HH;YScAkk_s?@MkrUiD!SaMI(Fm`y0gk*V+XJi02L7@nr zVwx6Q??yPC7MVA0U=wV(2@|Jy#dIfGSI7mI0M54({XY~;gED{u5SoI z4HN9;1_NHNfd3ew<5=wpiVIfrD$GHhxL{HII9H~7dAU{(3pu0s#^?C>1OdW3$G9_7}d(2syG=$DEUw|gE#Op z%d?Bq^Zz_jQwY(5JWrSA`a*bhPyGg@+`F?RgwRdT4g+(Iq8v@IW;LTJ;rddt_T~oO zlb=$OCIXQxPMOe3f5LHmOQb;2&%Z?8Nl9^Rce|w(CeW(J>kB;T5!yXv*}uE{0>K)! z!2j1C0&gAMR552onkWt9+}Hv^;C6wTncL0S9<5G@zp?@+q?Qt)I!2Janj|QP`VUxCPCm`$7eyw)g1Yl~ z!#R8I>cTTV$N`xL?yRQthlz@7XLk^xJ$nOL3b?)204xw8Hpm#14Iky%(&5Q>MTG0uUvu*;1pG_c8hD*^pE5le?Nndr_)LeTWz8D=q zmp;aMo7rKmo9%=F$`ZdUc~q1dv>D$Iw7jj&4Q#t6q(p+1Z<9tOMk&r_1)dQ!V|RpS z)k}RQP~uH>Q6Jan!`1xX$zQ|$|Esg((^qGsMY1g%o%jC_zj%50#TV84|DV46kpJsR z{yswAk#vDlI^kHSkwm9fB#_CuzO(buN9YFB9-I2;_s-k;m+0-z+u?BdcIPen>9{K+S(1^PKCTWs2l@5yA&*zGumw&NI&W5Ue_Mt<4-Xx%P*$WW0}w;dg4 z31rl-k#S42WK5tYafmATdau%BM&uahbZoBijVDWJ9rkW%;t7M@1a{|h<+9Y7<6DfW zYEfcC^~WZ-*CZ~WRN;w!E7pl3XUSNY4rBAUY|M*njK#PR?!|c<5k0$C8F{z5W5y;d zS%!FfcdBAri(n06tgMf*+T4}2kEdc|2DHF?5RFcOZ%mbgupK(K$F6peU6lIVqdkG= zLX0_4Mp!IK(njqA!EQ)#8OV5=k~egcw$AnXkRCMWoH4oa?5fV2V3%}3#tE6?B3&DK z*muJj$(S5Nj;e9a=^ahUjEsqhv3|F)!Nha}+*HyDPR1-tm%!#R+*yGe!FnkPlG3br zvx%(t0=#Xiw@K)xE74BQCK|$b-VnPH2$GD&Qh)`NJ&dAmfDh);R;RKL*sZUV80R_; zM~N6GoaJhfZgs&lVtATyK#BpJY_!>JhH1lbZJYY1rLn*5ye05YBs~wem>z`CIyxsf zr#kq~(v)!Y3ZA&|nvWAJC(J!J=B5x1{4br7ip#gMvBRFSBI$)*HSCXnW$Q8lr&waR zV8HGOZ^fQqiN}d+)rGpXX;rwIn=8|6!7?ftUky8GTprb>zM;U;4}x%XT*&!9(G^RH zXc?cm(ZKa$z^eV&kf#f-nDK!F@+(b95Y3C8$9xr?v?pmVc?r zwohym_VM3Z3jSJ52)df!xMf7u@)zwn*@X(kmD~u?bu_E;!ZbC( zHqi;M-lzpyfH-BdaZ2t;+D)HpR^i2Ic&8Zsz-b#xN|%qv1*Z-8YHcdWd>#F3U3|R_ zS*=RDqF?BRPu@t58<4N;)56PoKiFDT=U!M`>&jri#gg3Px=O)g+|6^xw zwZ`2RZ?_+K%gfsa-LGROR+`N?5n?P;(WK{Awe5t>Ss_U;$hDT;oMQRX<~(iDp?%tS z;t(X}M0PViXj2ij<{-Z5T#Mg_-?tE@jfISjrIv zu7u8rkgdbgKlgyJl0oHW)k)LhjZRW5TJ?lo18|?C2~YTY3xey6<~fn8afDfC#zOb% z!r9cmMgx_|)%9(sfXcvYi|6b zPyLvFoIs-n$fe&a2v(j$S{UmeL9W-cWwJInf?TiX!|-;VsXY+Xrj#`vJg(jO!PvLL zn*Jw^fnTzvL&ohuT%XP!M~fQ+*>qN`2a4-1Y6Xa7UNhr)UYKejblo16kZL_${{0B#-lqf_5>aQ0|O)*sCl zX?`-wdfBFrbh@|wse6!>vXwJE>o4fgWVcstM*e4-W!-@;I@>#kK+OJFPo@rON-tenX1SV&vJa)hsXiU5<1Dobp_P89_L3|myuNIL zuT77!`%RypGCrO!C!8iW`uxl8M=RS@u~!BfVC}<~rsg$jakkVPD37$eK@G}1O_NwH zsx~N14QfypZV9!*X>L-3H05yPzMYFMfkb>JmS&?yzX z&<`BHE|M9k2m114GEOjAuuPQ0+NB>HOir1O8#WpOx%s&sm@ofg!Cl!?X>(a;L9#l9 z?&x~#GyFq5pz+{)b?d6mYwTguLa&Laxmz!ccR3!5oWv_}*oM8J#(ANnLyECv;{{Dq z8Z+KwAT9lR;ardf<4fOtvXnhAHukDLPRViYW=7-ebT+KroN`ulFW5brTLHh$NemZk zR$q;QTJ4#nu3)<ogH*VvV?G9s$(p0u3u+60LTD9u|ai=0stnZZ6S0n z&O5?wt~*EL6}hA29v)s3A;ky*sUPW@Z#z4`Hr6ofNxv^{7C|1V9LFSrTCcwvEXRBX zH-N;>j%s~_?6^{Ok{3{r zGI+sF5@huPQk~Flj0G9eOpr`aN$Ae^pkP|TVOZD|ovG%L2vsLj# z=TgWS2NPqb!lIT3mjSyVtw$yi&Z%X$kQ^TEyW$(Z7>NRm!sx}or^UC)_B4kxCLvF& z`XqGjD4oScnMZxt*+TkzWGzT#<%MchX;hz}2Aqb^8Yaz5-V79RGNVHB<$jLiTRbEG zx4xSZ^;_drbRl=(iwb+HW2l}&cMPZJadI%=OO{yp20Y3!#|t8rwp4PX4CJt4w1Q)^ zTNU}*Ea*w1XVT%p!2$ed+u{S4v}pqW{uaifK2*5d`^oNRJgqP@(y-PSte!+hXw<^Q zHQDzN8EiSx-!22;M+Kcg5+eco_5u>ZCrID65s$^?+R&Z_+ua%bb{OrzJ%X?Xu%8p- z9!X?_(Y_z4Z6DxM@c)5VNP;<7F#+x-;mT>q(?XyH&FBIOlbKXXf=EfTnXonM(xuPv zbmizl3zpc5hcM@E`@WVqOE6E?pTP&R=snM3%Wa#To!W$z^SJzQe!Bnd`6>GR2#C})SmQ6cIJ+4ipT0W3h){I>)Ajh~{MA{shoXWek+KO7 zQB)p_0~0bhGNVzp^#k?+#rdsh{DbYQa1O(Kf171^!E7@gnR!$$DhTD326}4baa$Y< z34PCcvHL3A#`_w8ytEL{96+-&wz&6n55RMG!_B|fExe7{q`)J;C3x^UXKOB91?}sz z?-eH4^RYCEV7Ix2_ZsN?HpSYAe1r*j3f{hNo5)=Y+=PF;$aI2B2W6W)^H3kUguAb2 zZfWE&H(u@N3WyfnHu=&$nuqeL&qdK~a_mW#U!OreSH;$R0kC5V*4|{GL|T9ZDI~c) zU3v7~rEz?}ui3D7W4FHv=>m3kLO`dm1yoidNqN%V-(1LpA3HA{WVGH9Y;U)#F}sFWNakgFrK zgZ?*~7HJA6^t)D;_R#@MGU$_KXh5&AXx)9|<7(8p$e^@NTt6dmv0{n|z&z|-LM<%7zo^q0+~ z>268{Z050x2EIV5CoF)1vq0zb9d(BMv zEzXN0^y0I{0O`AmN9ePISJX0~!du4=dv8il8APQ2c6n?nO+cCzjosdmg`2f{v`th* zdesiiXooSUAfk~WLunEVDyVm&ahNK8$1xNTZ%6X6meY|HNCW%2)IAVHQ{wfKnX%wP zGC15T_qbuVBx}Yfd10}bH7{I|Gq_7->JMUrTLLI5+!-RxOchB-1SfY?P?mj1g=BmQ zh1Z&qzc84yVlmM*0a==$D}8tGnn+=)NQcF2?W)tbY*}*3vTr$#$tB?wBE&39TEyw_ zAegbBKbk23C#=Pyb30L<1C+F3m^380^??ZB6I{TpD>w@1VNiC1;bCwe;Iu z%KTd*kI-*9mh&UDKPNbq^M4NDa@P_1)4`t(-wlu<=9Hv(X$0p89ey@2lHk62abPu$ zMXq)_U8OhJTe|eBvIXecWUF4wUR2kKK0NUIgu%Tw)&42OF`u=~u$@N8&!#3*^n`hq z-Sq-VvOCII1{JC9FsFL4^}}(=xa>!`-$!!j(+~c6Mx`@O`yry`E;S|Lf&UFIX}nVE zwjb!butUEA{LNeh=>uMAC3A74&gVpD<7pA?cO^jD1Tg-y*+g>C$;PZ*5Q!5kaRZuT z|8TVzuHd0-Z-jX=(Wya`m0PFP{Y@q*2V0qDGY9n*e)B`-Z`U^OQqH<5we^-)11gQ+ zF2|i@H|<%?_I8ExyBzkK%7JR$QO>i=6vyN>zasi#x7}Qn=~X~)@V^|ySNP3w znzH*gYPIqA*+HxE_l7Qtg}3^ELB{5z+TzpOr!~a1d!{$wGpMmfS++LAuAAX_mg#V8 zlISXRoIsf0TfY4+^RINpe4xNsj#~^9j372}QrafqZiQ7TEneNoIGZhLp{w5J$~4a# zxSK7gH}^LOE1XU3__YbeL!Hvbl?IF(dd3Y6yU*F7{o{L> z&(G@rIkHc|qhCs>REIt`{z~twVFT`PzB-qv>4JEX5@fRJLV4954qIey6h)10+cR>^ zpDGIRlmUQkID&@VpILJRQK5-qHK@09zcbk_%WV%+u)qwbt6%)cLC*eTFBE z9ZsF%#?W(PbC2$WSw`qeR_-dU+S!}uWN06>XhC8PTlFsA(~R! z!{(E;HQnqj9i<&SPk*V4vVZL(u|JY?@T5m^4?ZS)JiqjJ>Lk5ihXpj~b8=I9I|i+0 z^E2=YDi-*tcDZzACCtF~zhsHYy6f!!wax#hFwp}#oe^v{ddQyZGh$UNVf{0xIB$rn z5thvbuS2c~=w!I=+O0x-9}_cDV2wiOP3sw7C9AT!v^tyTyyYOyfB zR3;?OVgg35P32jI?BDy_=Elf%DhN={I0n<&Soa_y{u1a_xBKN}432_nHsx4IUc|EC zq>M*&Ao2pL_zrord+mx^nsJ`1`FBTbQU5G z(-`WMdO6|2siU@_5~75UW;4XfSqdFMl+SU`_xPRk1dO*wneb?hv=(XC_sh_Zbxg2O}%Etp>ezX4JlK_n8LLL%jB zB9U!q6_6PZ^46o$6gP7!koy0HQtcRm5Xevs-5eM;4S?q?5hy7j?rWz-0fn8l*MxL) zCU^=AY!0hOS6E!G5Up7Xe((75>h*vAG|)b<{s4R2#cT7S-JSFyv>hXcLJ8r|fWkey z`DGUhgC6bgyR<>(H9%6tQtbOF9ICZAWqh$;sx!QQceu~VlyH*8WFO~rf0nWdPDgXO zNI&x1>QJi~F*HT6`|!8S7Q;`*QLo08M|Q9!97g6JFCOoSU0}Kevm|*bMx@7IPjp<7 zi*lXsK;>xd5-rt6L#IxBsX%i&o2N^k2Xuf)Du@Z4=+05b`ekb?qxURb5`i+dZ3g~C z?6@EgEwdjgJn?nR1AagvCf#9Rvh-4cq)AgdS)v*9VF)bRlAsI~Gk5EZ@HithY8&?k zkGm=r*sDo%JFBXwqgX{$+)(*GAS){tgwq(MY=+=BZHpQs)I?b} zGE+aRVyL>gu#kpczNuev5S7 zS+T~LYcz$HcaG&;>22HStER!9*;@LxU{&+rR~Gi)*@388#pqCt(61?*4OuS5Xo24h zA)(hNsPvtn|0KVndsd{0TAeO-Yz{%p7U1=~C1z;79R)x~M-)Px1qQr7tqIoP=*odc z=-F&TVqk9GgOd#2#FPC-;EqL7a0#+3H(pBe>sU zN$zoVw=}fO?2nK-HJ1Wm9qE*FER|891H~?xwN9ySdTFl_XJJ6ES5%;kJaGQaH+$RM z=Z-UD@91$c8~ZMe$g2JoBXc~}*JssM2mt}Lm$oq2w6vao1}#g@u>J)?XU+it>C%xc z?aBofiAmsKx0hXIIM}Q4{XB&Z+CsgkJ`YWn`0WQEE^0aG4RgO!E><%)3r@%Spwsct z?t^Lz(LbR`IcbsFHQ-;OpZQ)x@mN`X3?d*+eF)_NzSQX(lmxT~ni>u78ssh7-yLI6 z0_d)7*|;7Rq9D8~@80!Nzv^YS&g}F<_#`t&vl%BsjAbesT&#?le7C0P1QhN_1rl?+ z)tEl+Ov%7&WXI~+ZlLCACZDrHlC3cTpl!&48YNF8{VZSFQb5fPrywyW2L8`Rdj~C_ zKBxrvhMJOqkUg}c_8VhN`kfC5H{a%*OzE2qmTWVZ8a-XePt1vgwLI1sAJP3)pRV;3 z>AEO(h#iw25LhZXQ=MBASYMZ00eH`#Ctyk=8~o1PR2at4-BA-}jnUUiy)+z%=%)I! zj}SiM`G^@r`>@9u8&5G!TT0?g^%>LJe6V1<`0;*5ZS2L4Ip6TKtO5S>9)0)JH*jGS zX=pe5F!Jeg;W2LW2I(Sox|OzpOiX(%4u_OHMnuMP<2s{xPUI?vgg0--LiY)c z>>178I5q|6X>)?X{M;p@*$91ieRI91_ymWehze(WS6Bn^Mh0HxP`X6R(9m*(-Zq~x z_8Exc?^~47EFqahpFaYxsq~WB#%U^p{D$$I-DkEcmA;p{vx8K}S9E4BKph^A4quF3 z9K1LiyT;gbkKnF|u0w?LJVKEKJ1tTo$1;*h^A*iz+rK<=>^C_x z3ZNup$|am5B7Kn9%o}~wI8IEH`Wa>8StKkZ{fKM&kECC%geHW~Ake;|MyjRyd6q|A zkGkiQZn`hk@2foOdh|VVoc#}06!n3ldf&``zok&urt;aa>vvQD_2FZCWUhB5q5l^y zd%7L&yuZ2oQ0(-NTkLe>*?;eKPWx)ZCoXZi4q5MM6MPnhP1m6D{b|1ETh8>MwD-C6 z{D&=Kx~(z3M|0|#l`ee%#qZIsc^?H!*CFcxi}ZI=ru3l$c>srgU&TqcGoELtu%3HK z(yb5ecQXgSmqMhEHt^?L+;a7hpMUw$2ae&7IXSga6)47;ez#|` zw_z?RPSd5UxRK2~5&b5&=V-l#^8-OY2gCSf+f(f42U(2O??WzDRbl72!YeBsbGC3& zvv0r+J$LsX&fdqer3KAqDe*XAxcapaM*2rT?q0*FLJaPgcn0dnanY2e3E>g?*!}Wo zqTw@3eFI1#x_CVlP-9Ta|cW$1hWz% z80xRs)ZAi}%uVm^l%}v@`#X+wYBTC3{T}3@X&x$grG!G=;?!G+0uAIF457iko8xq8sL<(k6eGdX z-(>!pXYFST^NHuCosBx*=UBuX=VUDDg0Mm=vyNp+iM`2x6+$A@z~r!y4C#)DW|Htb zoT7f(ye|O(7tJqUZJTj7I^n4u7}N;YMVmp@T`0r(cnbQjmdQ#DI>V{CXIsjuKDPPS zcXrU$12!-+82&=AW?fVagY3v+;227W{| z<*H+rL@4rWD?*VCFxqVN$R~?Oz7AVWB7f1PQR@$q~);WV*n0d{uKiCRN0^MG9DkTCVC>y5fek*E{s zGLAG%2cx#GD(f|ng@)Bynh8b9H<+fdvAfYLqG+a%aY>??u~Qo|&X#&bmu39x5I#sZ z&X%aiV#X7ekwmX{n%UuON50E%9z2m&YC4I8Y%`_-fgSfjL^(-qBg%{VI%KJFnxsil zLK1l3os7aq`$s5TL^w;@8`=dXG3}i-u4T9Z&b9&DWxnm_lap6lINs1aRRBc1-#X^#^&Ye*Jy#? zpdu(9)61(3;cfLO)~l6iI(?Z2U4T*HsZi_kt(w$e zstH{G40FBK9{KqkuY#ED=~(EMR<3zLr^+9p{bKg3X|Vx_FFgSzd|#oTU7|==tiV?F zTOH*2+vTw@Pf;Go&*TkWp9)!h@n1_Q73);8d5SYKbOZ52*5SCb)^XIl z^+vuiPG?kUphuWYwY`&`PodW!F6x-q*0uX==B&@8+~s&IauPR9!(EP%`mK%oP_$D= z$6VgB&1b+;;#QOajdN-q#((9$yQNv;KS4$m_sN|-f7LO0tHUxH*zx7Lubbh&{wvLr z@biT|9l5E{;LlBdS5gt@g)#=D7)v%@&@`nn z@}+~Jm5#jz(24M|PEq}Bk)C6XYlU(*A8`-lTeZ}#l|uc7@s(vT)JB9eNype6E1i9A}c+lN9}vKg_6|P^m$-QahO-hzb3&c_%p6QfVYttp-bz)MLL!tQ@u^~ z=kc6M0;R^p2%YOYRxa)y{%Udd#x1K1Xr^dVO)w)U>WI8Q<5tv}%T30c}Ok9EWifdr==sy2HVN4y*K8YipAl zJ&J~HB*Ua}U7vAb%8&QP#2(bwWZX86{6EKXnG-MK?R") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Kind name portion + + +### helm_lib_get_api_version_by_kind + + returns current apiVersion string, based on available helm capabilities, for the provided kind (not all kinds are supported) + +#### Usage + +`{{ include "helm_lib_get_api_version_by_kind" (list . "") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Kind name portion + +## Application Image + +### helm_lib_application_image + + returns image name in format "registry/package@digest" + +#### Usage + +`{{ include "helm_lib_application_image" (list . "") }} ` + + +## Application Security Context + +### helm_lib_application_pod_security_context_run_as_user_custom + + returns PodSecurityContext parameters for Pod with custom user and group + +#### Usage + +`{{ include "helm_lib_application_pod_security_context_run_as_user_custom" (list . 1000 1000) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- User id +- Group id + + +### helm_lib_v_pod_security_context_run_as_user_nobody + + returns PodSecurityContext parameters for Pod with user and group "nobody" + +#### Usage + +`{{ include "helm_lib_application_pod_security_context_run_as_user_nobody" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_pod_security_context_run_as_user_nobody_with_writable_fs + + returns PodSecurityContext parameters for Pod with user and group "nobody" with write access to mounted volumes + +#### Usage + +`{{ include "helm_lib_application_pod_security_context_run_as_user_nobody_with_writable_fs" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_pod_security_context_run_as_user_deckhouse + + returns PodSecurityContext parameters for Pod with user and group "deckhouse" + +#### Usage + +`{{ include "helm_lib_application_pod_security_context_run_as_user_deckhouse" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_pod_security_context_run_as_user_deckhouse_with_writable_fs + + returns PodSecurityContext parameters for Pod with user and group "deckhouse" with write access to mounted volumes + +#### Usage + +`{{ include "helm_lib_application_pod_security_context_run_as_user_deckhouse_with_writable_fs" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_container_security_context_run_as_user_deckhouse_pss_restricted + + returns SecurityContext parameters for Container with user and group "deckhouse" plus minimal required settings to comply with the Restricted mode of the Pod Security Standards + +#### Usage + +`{{ include "helm_lib_application_container_security_context_run_as_user_deckhouse_pss_restricted" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_container_security_context_pss_restricted_flexible + + SecurityContext for Deckhouse UID/GID 64535 (or root), PSS Restricted + Optional keys: + .ro – bool, read-only root FS (default true) + .caps – []string, capabilities.add (default empty) + .uid – int, runAsUser/runAsGroup (default 64535) + .runAsNonRoot – bool, run as Deckhouse user when true, root when false (default true) + .seccompProfile – bool, disable seccompProfile when false (default true) + +#### Usage + +`include "helm_lib_application_container_security_context_pss_restricted_flexible" (dict "ro" false "caps" (list "NET_ADMIN" "SYS_TIME") "uid" 1001 "seccompProfile" false "runAsNonRoot" true) ` + + + +### helm_lib_application_pod_security_context_run_as_user_root + + returns PodSecurityContext parameters for Pod with user and group 0 + +#### Usage + +`{{ include "helm_lib_application_pod_security_context_run_as_user_root" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_pod_security_context_runtime_default + + returns PodSecurityContext parameters for Pod with seccomp profile RuntimeDefault + +#### Usage + +`{{ include "helm_lib_application_pod_security_context_runtime_default" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_container_security_context_not_allow_privilege_escalation + + returns SecurityContext parameters for Container with allowPrivilegeEscalation false + +#### Usage + +`{{ include "helm_lib_application_container_security_context_not_allow_privilege_escalation" . }} ` + + + +### helm_lib_application_container_security_context_read_only_root_filesystem_with_selinux + + returns SecurityContext parameters for Container with read only root filesystem and options for SELinux compatibility + +#### Usage + +`{{ include "helm_lib_application_container_security_context_read_only_root_filesystem_with_selinux" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_container_security_context_read_only_root_filesystem + + returns SecurityContext parameters for Container with read only root filesystem + +#### Usage + +`{{ include "helm_lib_application_container_security_context_read_only_root_filesystem" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_container_security_context_privileged + + returns SecurityContext parameters for Container running privileged + +#### Usage + +`{{ include "helm_lib_application_container_security_context_privileged" . }} ` + + + +### helm_lib_application_container_security_context_escalated_sys_admin_privileged + + returns SecurityContext parameters for Container running privileged with escalation and sys_admin + +#### Usage + +`{{ include "helm_lib_application_container_security_context_escalated_sys_admin_privileged" . }} ` + + + +### helm_lib_application_container_security_context_privileged_read_only_root_filesystem + + returns SecurityContext parameters for Container running privileged with read only root filesystem + +#### Usage + +`{{ include "helm_lib_application_container_security_context_privileged_read_only_root_filesystem" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all + + returns SecurityContext for Container with read only root filesystem and all capabilities dropped + +#### Usage + +`{{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add + + returns SecurityContext parameters for Container with read only root filesystem, all dropped and some added capabilities + +#### Usage + +`{{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- List of capabilities + + +### helm_lib_application_container_security_context_capabilities_drop_all_and_add + + returns SecurityContext parameters for Container with all dropped and some added capabilities + +#### Usage + +`{{ include "helm_lib_application_container_security_context_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- List of capabilities + + +### helm_lib_application_container_security_context_capabilities_drop_all_and_run_as_user_custom + + returns SecurityContext parameters for Container with read only root filesystem, all dropped, and custom user ID + +#### Usage + +`{{ include "helm_lib_application_container_security_context_capabilities_drop_all_and_run_as_user_custom" (list . 1000 1000) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- User id +- Group id + + +### helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted + + returns SecurityContext parameters for Container with minimal required settings to comply with the Restricted mode of the Pod Security Standards + +#### Usage + +`{{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + +## Capi Controller Manager + +### helm_lib_capi_controller_manager_manifests + + Renders common manifests for provider-specific CAPI Controller Managers. + Includes Deployment, VerticalPodAutoscaler (optional) and PodDisruptionBudget (optional). + Supported configuration parameters: + + fullname (required) — resource base name used for Deployment, PDB, VPA, and by default for the main container name. + + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. + + image (required) — image for the main container. + + capiProviderName (required) — value for the cluster.x-k8s.io/provider label in selectors and pod labels. + + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. + + priorityClassName (optional, default: `"system-cluster-critical"`) — Pod priority class name. + + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. + + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. + + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. + + terminationGracePeriodSeconds (optional, default: `10`) — Pod termination grace period. + + hostNetwork (optional, default: `false`) — enables host networking for the Pod. + + dnsPolicy (optional, default: `nil`) — Pod DNS policy; if not set, the field is omitted. + + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. + + tolerationsStrategies (optional, default: `["any-node", "uninitialized"]`) — arguments passed to helm_lib_tolerations. + + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 8081}, initialDelaySeconds: 15, periodSeconds: 20}`) — liveness probe configuration for the main container. + + readinessProbe (optional, default: `{httpGet: {path: /readyz, port: 8081}, initialDelaySeconds: 5, periodSeconds: 10}`) — readiness probe configuration for the main container. + + additionalArgs (optional, default: `[]`) — extra args for the main container. + + additionalEnv (optional, default: `[]`) — extra environment variables for the main container. + + additionalPorts (optional, default: `[]`) — extra container ports for the main container. + + additionalInitContainers (optional, default: `[]`) — extra initContainers for the Pod. + + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. + + additionalVolumes (optional, default: `[]`) — extra Pod volumes. + + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. + + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. + + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. + + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. + + vpaEnabled (optional, default: `false`) — enables VerticalPodAutoscaler rendering. + + vpaUpdateMode (optional, default: `"InPlaceOrRecreate"`) — VPA update mode. + + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. + + securityPolicyExceptionEnabled (optional, default: `false`) — enables SecurityPolicyException rendering and adds the related pod label. + +#### Usage + +`{{ include "helm_lib_capi_controller_manager_manifests" (list . $config) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc. +- Configuration dict for the CAPI Controller Manager. + +## Cloud Controller Manager + +### helm_lib_cloud_controller_manager_manifests + + Renders common manifests for provider-specific Cloud Controller Managers. + Includes Deployment, VerticalPodAutoscaler (optional), PodDisruptionBudget (optional), and SecurityPolicyException (optional). + Supported configuration parameters: + + fullname (optional, default: `"cloud-controller-manager"`) — resource base name used for Deployment, PDB, VPA, SecurityPolicyException, and the main container name by default. + + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. + + image (required) — image for the main container. + + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. + + priorityClassName (optional, default: `"system-cluster-critical"`) — Pod priority class name. + + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. + + tolerationsStrategies (optional, default: ["wildcard"]) — strategies passed to helm_lib_tolerations. + + hostNetwork (optional, default: `true`) — enables host networking for the Pod and SecurityPolicyException network rule generation. + + dnsPolicy (optional, default: `"Default"`) — Pod DNS policy. + + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. + + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. + + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. + + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 10471, host: 127.0.0.1, scheme: HTTPS}}`) — liveness probe configuration for the main container. + + readinessProbe (optional, default: `{httpGet: {path: /healthz, port: 10471, host: 127.0.0.1, scheme: HTTPS}}`) — readiness probe configuration for the main container. + + additionalEnvs (optional, default: `[]`) — extra environment variables for the main container. + + additionalArgs (optional, default: `nil`) — extra args for the main container. + + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. + + additionalVolumes (optional, default: `[]`) — extra Pod volumes; hostPath volumes are also used to build SecurityPolicyException rules when enabled. + + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. + + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. + + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. + + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. + + additionalPDBAnnotations (optional, default: `{}`) — extra annotations added to PodDisruptionBudget metadata. + + vpaEnabled (optional, default: `true`) — enables VerticalPodAutoscaler rendering. + + vpaUpdateMode (optional, default: `"InPlaceOrRecreate"`) — VPA update mode. + + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. + + securityPolicyExceptionEnabled (optional, default: `false`) — enables SecurityPolicyException rendering and adds the related pod label. + +#### Usage + +`{{ include "helm_lib_cloud_controller_manager_manifests" (list . $config) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc. +- Configuration dict for the Cloud Controller Manager. + +## Cloud Data Discoverer + +### helm_lib_cloud_data_discoverer_manifests + + Renders common manifests for provider-specific Cloud Data Discoverers. + Includes Deployment, VerticalPodAutoscaler (optional) and PodDisruptionBudget (optional). + Supported configuration parameters: + + fullname (optional, default: `"cloud-data-discoverer"`) — resource base name used for Deployment, PDB, VPA, and the main container name by default. + + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. + + image (required) — image for the main container. + + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. + + replicas (optional, default: `1`) — number of Deployment replicas. + + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. + + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. + + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. + + priorityClassName (optional, default: `"cluster-low"`) — Pod priority class name. + + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. + + tolerationsStrategies (optional, default: `["any-node", "with-uninitialized"]`) — strategies passed to helm_lib_tolerations. + + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 8080, scheme: HTTPS}}`) — liveness probe configuration for the main container. + + readinessProbe (optional, default: `{httpGet: {path: /healthz, port: 8080, scheme: HTTPS}}`) — readiness probe configuration for the main container. + + additionalArgs (optional, default: `[]`) — extra args for the main container. + + additionalEnv (optional, default: `[]`) — extra environment variables for the main container. + + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. + + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. + + additionalInitContainers (optional, default: `[]`) — extra initContainers for the Pod. + + additionalVolumes (optional, default: `[]`) — extra Pod volumes. + + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. + + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. + + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. + + vpaEnabled (optional, default: `true`) — enables VerticalPodAutoscaler rendering. + + vpaUpdateMode (optional, default: `"Initial"`) — VPA update mode. + + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. + +#### Usage + +`{{ include "helm_lib_cloud_data_discoverer_manifests" (list . $config) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc. +- Configuration dict for the Cloud Data Discoverer. + + +### helm_lib_cloud_data_discoverer_pod_monitor + + Renders PodMonitor manifest for provider-specific Cloud Data Discoverers. + Supported configuration parameters: + + fullname (optional, default: `"cloud-data-discoverer"`) — PodMonitor base name. + + targetNamespace (required) — target pod namespace for selector. + + additionalRelabelings (optional, default: `[]`) — additional rules for labels rewriting. + +#### Usage + +`{{ include "helm_lib_cloud_data_discoverer_pod_monitor" (list . $config) }} ` + + +## Cloud Provider User Authz Roles + +### helm_lib_cloud_provider_user_authz_cluster_roles + + Renders user-authz ClusterRoles for provider-specific cloud resources. + Includes User and ClusterAdmin ClusterRoles. + Supported configuration parameters: + + providerName (required) — provider name segment used in ClusterRole names. + + instanceClassResource (required) — Deckhouse instance class resource name granted by the rules. + + capiResources (optional, default: `[]`) — CAPI infrastructure resource names granted by the rules. + + additionalUserRules (optional, default: `[]`) — extra rules appended to the User ClusterRole. + + additionalClusterAdminRules (optional, default: `[]`) — extra rules appended to the ClusterAdmin ClusterRole. + +#### Usage + +`{{- include "helm_lib_cloud_provider_user_authz_cluster_roles" (list . $config) }} ` + + +## Csi Controller + +### helm_lib_csi_image_with_common_fallback + + returns image name from storage foundation module if enabled, otherwise from common module + +#### Usage + +`{{ include "helm_lib_csi_image_with_common_fallback" (list . "" "") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Container raw name +- Kubernetes semantic version + +## Dns Policy + +### helm_lib_dns_policy_bootstraping_state + + returns the proper dnsPolicy value depending on the cluster bootstrap phase + +#### Usage + +`{{ include "helm_lib_dns_policy_bootstraping_state" (list . "Default" "ClusterFirstWithHostNet") }} ` + + +## Enable Ds Eviction + +### helm_lib_prevent_ds_eviction_annotation + + Adds `cluster-autoscaler.kubernetes.io/enable-ds-eviction` annotation to manage DaemonSet eviction by the Cluster Autoscaler. + This is important to prevent the eviction of DaemonSet pods during cluster scaling. + +#### Usage + +`{{ include "helm_lib_prevent_ds_eviction_annotation" . }} ` + + +## Envs For Proxy + +### helm_lib_envs_for_proxy + + Add HTTP_PROXY, HTTPS_PROXY and NO_PROXY environment variables for container + depends on [proxy settings](https://deckhouse.io/products/kubernetes-platform/documentation/v1/reference/api/global.html#parameters-modules-proxy) + +#### Usage + +`{{ include "helm_lib_envs_for_proxy" . }} or {{ include "helm_lib_envs_for_proxy" (list . (list "extra1" "extra2")) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- List of additional NO_PROXY entries (optional) + +## High Availability + +### helm_lib_is_ha_to_value + + returns value "yes" if cluster is highly available, else — returns "no" + +#### Usage + +`{{ include "helm_lib_is_ha_to_value" (list . yes no) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Yes value +- No value + + +### helm_lib_ha_enabled + + returns empty value, which is treated by go template as false + +#### Usage + +`{{- if (include "helm_lib_ha_enabled" .) }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + +## Kube Rbac Proxy + +### helm_lib_kube_rbac_proxy_ca_certificate + + Renders configmap with kube-rbac-proxy CA certificate which uses to verify the kube-rbac-proxy clients. + +#### Usage + +`{{ include "helm_lib_kube_rbac_proxy_ca_certificate" (list . "namespace") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Namespace where CA configmap will be created + +## Module Controller + +### helm_lib_module_controller_resources + + Returns default controller resources + +#### Usage + +`{{ include "helm_lib_module_controller_resources" . }} ` + + + +### helm_lib_module_webhooks_resources + + Returns default webhooks resources + +#### Usage + +`{{ include "helm_lib_module_webhooks_resources" . }} ` + + + +### helm_lib_module_controller_log_level + + Returns numeric log level from module values + +#### Usage + +`{{ include "helm_lib_module_controller_log_level" (list . "csiHpe") }} ` + + +## Module Documentation Uri + +### helm_lib_module_documentation_uri + + returns rendered documentation uri using publicDomainTemplate or deckhouse.io domains + +#### Usage + +`{{ include "helm_lib_module_documentation_uri" (list . "") }} ` + + +## Module Ephemeral Storage + +### helm_lib_module_ephemeral_storage_logs_with_extra + + 50Mi for container logs `log-opts.max-file * log-opts.max-size` would be added to passed value + returns ephemeral-storage size for logs with extra space + +#### Usage + +`{{ include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 }} ` + +#### Arguments + +- Extra space in mebibytes + + +### helm_lib_module_ephemeral_storage_only_logs + + 50Mi for container logs `log-opts.max-file * log-opts.max-size` would be requested + returns ephemeral-storage size for only logs + +#### Usage + +`{{ include "helm_lib_module_ephemeral_storage_only_logs" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + +## Module Gateway + +### helm_lib_module_gateway + + accepts a dict that is updated with current gateway name and namespace + +#### Usage + +`{{- include "helm_lib_module_gateway" (list . $gateway) ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- An empty dict to update with current default gateway name and namespace + +## Module Generate Common Name + +### helm_lib_module_generate_common_name + + returns the commonName parameter for use in the Certificate custom resource(cert-manager) + +#### Usage + +`{{ include "helm_lib_module_generate_common_name" (list . "") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Name portion + +## Module Https + +### helm_lib_module_uri_scheme + + return module uri scheme "http" or "https" + +#### Usage + +`{{ include "helm_lib_module_uri_scheme" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_https_mode + + returns https mode for module + +#### Usage + +`{{ if (include "helm_lib_module_https_mode" .) }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_https_cert_manager_cluster_issuer_name + + returns cluster issuer name + +#### Usage + +`{{ include "helm_lib_module_https_cert_manager_cluster_issuer_name" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_https_ingress_tls_enabled + + returns not empty string if tls should be enabled for the ingress + +#### Usage + +`{{ if (include "helm_lib_module_https_ingress_tls_enabled" .) }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_https_route_tls_enabled + + returns not empty string if tls should be enabled for the route + +#### Usage + +`{{ if (include "helm_lib_module_https_route_tls_enabled" .) }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_https_copy_custom_certificate + + Renders secret with [custom certificate](https://deckhouse.io/products/kubernetes-platform/documentation/v1/reference/api/global.html#parameters-modules-https-customcertificate) + in passed namespace with passed prefix + +#### Usage + +`{{ include "helm_lib_module_https_copy_custom_certificate" (list . "namespace" "secret_name_prefix") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Namespace +- Secret name prefix + + +### helm_lib_module_https_secret_name + + returns custom certificate name + +#### Usage + +`{{ include "helm_lib_module_https_secret_name (list . "secret_name_prefix") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Secret name prefix + +## Module Image + +### helm_lib_module_image + + returns image name + +#### Usage + +`{{ include "helm_lib_module_image" (list . "" "(optional)") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Container name + + +### helm_lib_module_image_no_fail + + returns image name if found + +#### Usage + +`{{ include "helm_lib_module_image_no_fail" (list . "") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Container name + + +### helm_lib_module_common_image + + returns image name from common module + +#### Usage + +`{{ include "helm_lib_module_common_image" (list . "") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Container name + + +### helm_lib_module_common_image_no_fail + + returns image name from common module if found + +#### Usage + +`{{ include "helm_lib_module_common_image_no_fail" (list . "") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Container name + + +### helm_lib_module_image_digest + + returns image digest + +#### Usage + +`{{ include "helm_lib_module_image_digest" (list . "" "(optional)") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Container name + + +### helm_lib_module_image_digest_no_fail + + returns image digest if found + +#### Usage + +`{{ include "helm_lib_module_image_digest_no_fail" (list . "" "(optional)") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Container name + +## Module Ingress Class + +### helm_lib_module_ingress_class + + returns ingress class from module settings or if not exists from global config + +#### Usage + +`{{ include "helm_lib_module_ingress_class" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + +## Module Ingress Snippets + +### helm_lib_module_ingress_configuration_snippet + + returns nginx ingress additional headers (e.g. HSTS) if HTTPS is enabled + +#### Usage + +`nginx.ingress.kubernetes.io/configuration-snippet: | {{ include "helm_lib_module_ingress_configuration_snippet" . | nindent 6 }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + +## Module Init Container + +### helm_lib_module_init_container_chown_nobody_volume + + ### Migration 11.12.2020: Remove this helper with all its usages after this commit reached RockSolid + returns initContainer which chowns recursively all files and directories in passed volume + +#### Usage + +`{{ include "helm_lib_module_init_container_chown_nobody_volume" (list . "volume-name") }} ` + + + +### helm_lib_module_init_container_chown_deckhouse_volume + + returns initContainer which chowns recursively all files and directories in passed volume + +#### Usage + +`{{ include "helm_lib_module_init_container_chown_deckhouse_volume" (list . "volume-name") }} ` + + + +### helm_lib_module_init_container_check_linux_kernel + + returns initContainer which checks the kernel version on the node for compliance to semver constraint + +#### Usage + +`{{ include "helm_lib_module_init_container_check_linux_kernel" (list . ">= 4.9.17") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Semver constraint + + +### helm_lib_module_init_container_iptables_wrapper + + returns initContainer with iptables-wrapper + +#### Usage + +`{{ include "helm_lib_module_init_container_iptables_wrapper" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + +## Module Labels + +### helm_lib_module_labels + + returns deckhouse labels + +#### Usage + +`{{ include "helm_lib_module_labels" (list . (dict "app" "test" "component" "testing")) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Additional labels dict + +## Module Public Domain + +### helm_lib_module_public_domain + + returns rendered publicDomainTemplate to service fqdn + +#### Usage + +`{{ include "helm_lib_module_public_domain" (list . "") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Name portion + +## Module Security Context + +### helm_lib_module_pod_security_context_run_as_user_custom + + returns PodSecurityContext parameters for Pod with custom user and group + +#### Usage + +`{{ include "helm_lib_module_pod_security_context_run_as_user_custom" (list . 1000 1000) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- User id +- Group id + + +### helm_lib_module_pod_security_context_run_as_user_nobody + + returns PodSecurityContext parameters for Pod with user and group "nobody" + +#### Usage + +`{{ include "helm_lib_module_pod_security_context_run_as_user_nobody" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_pod_security_context_run_as_user_nobody_with_writable_fs + + returns PodSecurityContext parameters for Pod with user and group "nobody" with write access to mounted volumes + +#### Usage + +`{{ include "helm_lib_module_pod_security_context_run_as_user_nobody_with_writable_fs" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_pod_security_context_run_as_user_deckhouse + + returns PodSecurityContext parameters for Pod with user and group "deckhouse" + +#### Usage + +`{{ include "helm_lib_module_pod_security_context_run_as_user_deckhouse" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_pod_security_context_run_as_user_deckhouse_with_writable_fs + + returns PodSecurityContext parameters for Pod with user and group "deckhouse" with write access to mounted volumes + +#### Usage + +`{{ include "helm_lib_module_pod_security_context_run_as_user_deckhouse_with_writable_fs" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_container_security_context_run_as_user_deckhouse_pss_restricted + + returns SecurityContext parameters for Container with user and group "deckhouse" plus minimal required settings to comply with the Restricted mode of the Pod Security Standards + +#### Usage + +`{{ include "helm_lib_module_container_security_context_run_as_user_deckhouse_pss_restricted" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_container_security_context_pss_restricted_flexible + + SecurityContext for Deckhouse UID/GID 64535 (or root), PSS Restricted + Optional keys: + .ro – bool, read-only root FS (default true) + .caps – []string, capabilities.add (default empty) + .uid – int, runAsUser/runAsGroup (default 64535) + .runAsNonRoot – bool, run as Deckhouse user when true, root when false (default true) + .seccompProfile – bool, disable seccompProfile when false (default true) + +#### Usage + +`include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" false "caps" (list "NET_ADMIN" "SYS_TIME") "uid" 1001 "seccompProfile" false "runAsNonRoot" true) ` + + + +### helm_lib_module_pod_security_context_run_as_user_root + + returns PodSecurityContext parameters for Pod with user and group 0 + +#### Usage + +`{{ include "helm_lib_module_pod_security_context_run_as_user_root" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_pod_security_context_runtime_default + + returns PodSecurityContext parameters for Pod with seccomp profile RuntimeDefault + +#### Usage + +`{{ include "helm_lib_module_pod_security_context_runtime_default" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_container_security_context_not_allow_privilege_escalation + + returns SecurityContext parameters for Container with allowPrivilegeEscalation false + +#### Usage + +`{{ include "helm_lib_module_container_security_context_not_allow_privilege_escalation" . }} ` + + + +### helm_lib_module_container_security_context_read_only_root_filesystem_with_selinux + + returns SecurityContext parameters for Container with read only root filesystem and options for SELinux compatibility + +#### Usage + +`{{ include "helm_lib_module_container_security_context_read_only_root_filesystem_with_selinux" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_container_security_context_read_only_root_filesystem + + returns SecurityContext parameters for Container with read only root filesystem + +#### Usage + +`{{ include "helm_lib_module_container_security_context_read_only_root_filesystem" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_container_security_context_privileged + + returns SecurityContext parameters for Container running privileged + +#### Usage + +`{{ include "helm_lib_module_container_security_context_privileged" . }} ` + + + +### helm_lib_module_container_security_context_escalated_sys_admin_privileged + + returns SecurityContext parameters for Container running privileged with escalation and sys_admin + +#### Usage + +`{{ include "helm_lib_module_container_security_context_escalated_sys_admin_privileged" . }} ` + + + +### helm_lib_module_container_security_context_privileged_read_only_root_filesystem + + returns SecurityContext parameters for Container running privileged with read only root filesystem + +#### Usage + +`{{ include "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all + + returns SecurityContext for Container with read only root filesystem and all capabilities dropped + +#### Usage + +`{{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add + + returns SecurityContext parameters for Container with read only root filesystem, all dropped and some added capabilities + +#### Usage + +`{{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- List of capabilities + + +### helm_lib_module_container_security_context_capabilities_drop_all_and_add + + returns SecurityContext parameters for Container with all dropped and some added capabilities + +#### Usage + +`{{ include "helm_lib_module_container_security_context_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- List of capabilities + + +### helm_lib_module_container_security_context_capabilities_drop_all_and_run_as_user_custom + + returns SecurityContext parameters for Container with read only root filesystem, all dropped, and custom user ID + +#### Usage + +`{{ include "helm_lib_module_container_security_context_capabilities_drop_all_and_run_as_user_custom" (list . 1000 1000) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- User id +- Group id + + +### helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted + + returns SecurityContext parameters for Container with minimal required settings to comply with the Restricted mode of the Pod Security Standards + +#### Usage + +`{{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + +## Module Storage Class + +### helm_lib_module_storage_class_annotations + + return module StorageClass annotations + +#### Usage + +`{{ include "helm_lib_module_storage_class_annotations" (list $ $index $storageClass.name) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Storage class index +- Storage class name + +## Monitoring Grafana Dashboards + +### helm_lib_grafana_dashboard_definitions_recursion + + returns all the dashboard-definintions from / + current dir is optional — used for recursion but you can use it for partially generating dashboards + +#### Usage + +`{{ include "helm_lib_grafana_dashboard_definitions_recursion" (list . [current dir]) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Dashboards root dir +- Dashboards current dir + + +### helm_lib_grafana_dashboard_definitions + + returns dashboard-definintions from monitoring/grafana-dashboards/ + +#### Usage + +`{{ include "helm_lib_grafana_dashboard_definitions" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_single_dashboard + + renders a single dashboard + +#### Usage + +`{{ include "helm_lib_single_dashboard" (list . "dashboard-name" "folder" $dashboard) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Dashboard name +- Folder +- Dashboard definition + +## Monitoring Prometheus Rules + +### helm_lib_prometheus_rules_recursion + + returns all the prometheus rules from / + current dir is optional — used for recursion but you can use it for partially generating rules + file list is optional - list of files to include (filters all files if provided) + +#### Usage + +`{{ include "helm_lib_prometheus_rules_recursion" (list . [current dir] [file list]) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Namespace for creating rules +- Rules root dir +- Current dir (optional) +- File list for filtering (optional) + + +### helm_lib_prometheus_rules + + returns all the prometheus rules from monitoring/prometheus-rules/ optionally filtered by fileList + +#### Usage + +`{{ include "helm_lib_prometheus_rules" (list . [fileList]) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Namespace for creating rules + + +### helm_lib_prometheus_target_scrape_timeout_seconds + + returns adjust timeout value to scrape interval / + +#### Usage + +`{{ include "helm_lib_prometheus_target_scrape_timeout_seconds" (list . ) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Target timeout in seconds + +## Node Affinity + +### helm_lib_internal_check_node_selector_strategy + + Verify node selector strategy. + + + +### helm_lib_node_selector + + Returns node selector for workloads depend on strategy. + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- strategy, one of "frontend" "monitoring" "system" "master" "any-node" "wildcard" + + +### helm_lib_tolerations + + Returns tolerations for workloads depend on strategy. + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "any-node" "with-uninitialized" "without-storage-problems") }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- base strategy, one of "frontend" "monitoring" "system" any-node" "wildcard" +- list of additional strategies. To add strategy list it with prefix "with-", to remove strategy list it with prefix "without-". + + +### _helm_lib_cloud_or_hybrid_cluster + + Check cluster type. + Returns not empty string if this is cloud or hybrid cluster + + + +### helm_lib_internal_check_tolerations_strategy + + Verify base strategy. + Fails if strategy not in allowed list + + + +### _helm_lib_any_node_tolerations + + Base strategy for any uncordoned node in cluster. + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "any-node") }} ` + + + +### _helm_lib_wildcard_tolerations + + Base strategy that tolerates all. + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "wildcard") }} ` + + + +### _helm_lib_monitoring_tolerations + + Base strategy that tolerates nodes with "dedicated.deckhouse.io: monitoring" and "dedicated.deckhouse.io: system" taints. + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "monitoring") }} ` + + + +### _helm_lib_frontend_tolerations + + Base strategy that tolerates nodes with "dedicated.deckhouse.io: frontend" taints. + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "frontend") }} ` + + + +### _helm_lib_system_tolerations + + Base strategy that tolerates nodes with "dedicated.deckhouse.io: system" taints. + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "system") }} ` + + + +### _helm_lib_additional_tolerations_uninitialized + + Additional strategy "uninitialized" - used for CNI's and kube-proxy to allow cni components scheduled on node after CCM initialization. + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "any-node" "with-uninitialized") }} ` + + + +### _helm_lib_additional_tolerations_node_problems + + Additional strategy "node-problems" - used for shedule critical components on non-ready nodes or nodes under pressure. + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "any-node" "with-node-problems") }} ` + + + +### _helm_lib_additional_tolerations_storage_problems + + Additional strategy "storage-problems" - used for shedule critical components on nodes with drbd problems. This additional strategy enabled by default in any base strategy except "wildcard". + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "any-node" "without-storage-problems") }} ` + + + +### _helm_lib_additional_tolerations_no_csi + + Additional strategy "no-csi" - used for any node with no CSI: any node, which was initialized by deckhouse, but have no csi-node driver registered on it. + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "any-node" "with-no-csi") }} ` + + + +### _helm_lib_additional_tolerations_cloud_provider_uninitialized + + Additional strategy "cloud-provider-uninitialized" - used for any node which is not initialized by CCM. + +#### Usage + +`{{ include "helm_lib_tolerations" (tuple . "any-node" "with-cloud-provider-uninitialized") }} ` + + +## Pod Disruption Budget + +### helm_lib_pdb_daemonset + + Returns PDB max unavailable + +#### Usage + +`{{ include "helm_lib_pdb_daemonset" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + +## Priority Class + +### helm_lib_priority_class + + returns priority class + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Priority class name + +## Resources Management + +### helm_lib_resources_management_pod_resources + + returns rendered resources section based on configuration if it is + +#### Usage + +`{{ include "helm_lib_resources_management_pod_resources" (list [ephemeral storage requests]) }} ` + +#### Arguments + +list: +- VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) +- Ephemeral storage requests + + +### helm_lib_resources_management_original_pod_resources + + returns rendered resources section based on configuration if it is present + +#### Usage + +`{{ include "helm_lib_resources_management_original_pod_resources" }} ` + +#### Arguments + +- VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) + + +### helm_lib_resources_management_vpa_spec + + returns rendered vpa spec based on configuration and target reference + +#### Usage + +`{{ include "helm_lib_resources_management_vpa_spec" (list ) }} ` + +#### Arguments + +list: +- Target API version +- Target Kind +- Target Name +- Target container name +- VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) + + +### helm_lib_resources_management_cpu_units_to_millicores + + helper for converting cpu units to millicores + +#### Usage + +`{{ include "helm_lib_resources_management_cpu_units_to_millicores" }} ` + + + +### helm_lib_resources_management_memory_units_to_bytes + + helper for converting memory units to bytes + +#### Usage + +`{{ include "helm_lib_resources_management_memory_units_to_bytes" }} ` + + + +### helm_lib_vpa_kube_rbac_proxy_resources + + helper for VPA resources for kube_rbac_proxy + +#### Usage + +`{{ include "helm_lib_vpa_kube_rbac_proxy_resources" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_container_kube_rbac_proxy_resources + + helper for container resources for kube_rbac_proxy + +#### Usage + +`{{ include "helm_lib_container_kube_rbac_proxy_resources" . }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + +## Spec For High Availability + +### helm_lib_pod_anti_affinity_for_ha + + returns pod affinity spec + +#### Usage + +`{{ include "helm_lib_pod_anti_affinity_for_ha" (list . (dict "app" "test")) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Match labels for podAntiAffinity label selector + + +### helm_lib_pod_affinity + + Returns affinity spec that combines: podAntiAffinity by provided labels when HA is enabled and optional nodeAffinity that schedules pods only on specified architectures. If the list of architectures is not provided or empty, node affinity is not rendered. + +#### Usage + +`{{- include "helm_lib_pod_affinity" (list . (dict "app" "test") (list "amd64")) }} ` + +#### Arguments + +list: +- Template context with .Values, .Chart, etc +- Match labels for podAntiAffinity label selector + + +### helm_lib_deployment_on_master_strategy_and_replicas_for_ha + + returns deployment strategy and replicas for ha components running on master nodes + +#### Usage + +`{{ include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc + + +### helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha + + returns deployment with custom strategy and replicas for ha components running on master nodes + +#### Usage + +`{{ include "helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha" (list . (dict "strategy" "strategy_type")) }} ` + + + +### helm_lib_deployment_strategy_and_replicas_for_ha + + returns deployment strategy and replicas for ha components running not on master nodes + +#### Usage + +`{{ include "helm_lib_deployment_strategy_and_replicas_for_ha" }} ` + +#### Arguments + +- Template context with .Values, .Chart, etc diff --git a/charts/helm_lib/templates/_admission_client_ca.tpl b/charts/helm_lib/templates/_admission_client_ca.tpl new file mode 100644 index 000000000..b599179fe --- /dev/null +++ b/charts/helm_lib/templates/_admission_client_ca.tpl @@ -0,0 +1,21 @@ +{{- /* Usage: {{ include "helm_lib_admission_webhook_client_ca_certificate" (list . "namespace") }} */ -}} +{{- /* Renders configmap with admission webhook client CA certificate which uses to verify the AdmissionReview requests. */ -}} +{{- define "helm_lib_admission_webhook_client_ca_certificate" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +{{- /* Namespace where CA configmap will be created */ -}} + {{- $context := index . 0 }} + {{- $namespace := index . 1 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: admission-client-ca.crt + namespace: {{ $namespace }} + annotations: + kubernetes.io/description: | + Contains a CA bundle that can be used to verify the admission webhook client. + {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} +data: + ca.crt: | + {{ $context.Values.global.internal.modules.admissionWebhookClientCA.cert | nindent 4 }} +{{- end }} diff --git a/charts/helm_lib/templates/_api_version_and_kind.tpl b/charts/helm_lib/templates/_api_version_and_kind.tpl new file mode 100644 index 000000000..4de8a8a36 --- /dev/null +++ b/charts/helm_lib/templates/_api_version_and_kind.tpl @@ -0,0 +1,36 @@ +{{- /* Usage: {{ include "helm_lib_kind_exists" (list . "") }} */ -}} +{{- /* returns true if the specified resource kind (case-insensitive) is represented in the cluster */ -}} +{{- define "helm_lib_kind_exists" }} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $kind_name := index . 1 -}} {{- /* Kind name portion */ -}} + {{- if eq (len $context.Capabilities.APIVersions) 0 -}} + {{- fail "Helm reports no capabilities" -}} + {{- end -}} + {{ range $cap := $context.Capabilities.APIVersions }} + {{- if hasSuffix (lower (printf "/%s" $kind_name)) (lower $cap) }} + found + {{- break }} + {{- end }} + {{- end }} +{{- end -}} + +{{- /* Usage: {{ include "helm_lib_get_api_version_by_kind" (list . "") }} */ -}} +{{- /* returns current apiVersion string, based on available helm capabilities, for the provided kind (not all kinds are supported) */ -}} +{{- define "helm_lib_get_api_version_by_kind" }} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $kind_name := index . 1 -}} {{- /* Kind name portion */ -}} + {{- if eq (len $context.Capabilities.APIVersions) 0 -}} + {{- fail "Helm reports no capabilities" -}} + {{- end -}} + {{- if or (eq $kind_name "ValidatingAdmissionPolicy") (eq $kind_name "ValidatingAdmissionPolicyBinding") -}} + {{- if $context.Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" -}} +admissionregistration.k8s.io/v1 + {{- else if $context.Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1beta1/ValidatingAdmissionPolicy" -}} +admissionregistration.k8s.io/v1beta1 + {{- else -}} +admissionregistration.k8s.io/v1alpha1 + {{- end -}} + {{- else -}} + {{- fail (printf "Kind '%s' isn't supported by the 'helm_lib_get_api_version_by_kind' helper" $kind_name) -}} + {{- end -}} +{{- end -}} diff --git a/charts/helm_lib/templates/_application_image.tpl b/charts/helm_lib/templates/_application_image.tpl new file mode 100644 index 000000000..c1aa9ef87 --- /dev/null +++ b/charts/helm_lib/templates/_application_image.tpl @@ -0,0 +1,23 @@ +{{- /* Usage: {{ include "helm_lib_application_image" (list . "") }} */ -}} +{{- /* returns image name in format "registry/package@digest" */ -}} +{{- define "helm_lib_application_image" }} + {{- $context := index . 0 }} + + {{- $image := index . 1 | trimAll "\"" }} + {{- $imageDigest := index $context.Application.Package.Digests $image }} + {{- if not $imageDigest }} + {{- fail (printf "Image %s has no digest" $image) }} + {{- end }} + + {{- $registryBase := $context.Application.Package.Registry.repository }} + {{- if not $registryBase }} + {{- fail "Registry base is not set" }} + {{- end }} + + {{- $packageName := $context.Application.Package.Name }} + {{- if not $packageName }} + {{- fail "Package name is not set" }} + {{- end }} + + {{- printf "%s/%s@%s" $registryBase $packageName $imageDigest }} +{{- end }} diff --git a/charts/helm_lib/templates/_application_security_context.tpl b/charts/helm_lib/templates/_application_security_context.tpl new file mode 100644 index 000000000..fd2108359 --- /dev/null +++ b/charts/helm_lib/templates/_application_security_context.tpl @@ -0,0 +1,263 @@ +{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_custom" (list . 1000 1000) }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with custom user and group */ -}} +{{- define "helm_lib_application_pod_security_context_run_as_user_custom" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +{{- /* User id */ -}} +{{- /* Group id */ -}} +securityContext: + runAsNonRoot: true + runAsUser: {{ index . 1 }} + runAsGroup: {{ index . 2 }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_nobody" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with user and group "nobody" */ -}} +{{- define "helm_lib_v_pod_security_context_run_as_user_nobody" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_nobody_with_writable_fs" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with user and group "nobody" with write access to mounted volumes */ -}} +{{- define "helm_lib_application_pod_security_context_run_as_user_nobody_with_writable_fs" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_deckhouse" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with user and group "deckhouse" */ -}} +{{- define "helm_lib_application_pod_security_context_run_as_user_deckhouse" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + runAsNonRoot: true + runAsUser: 64535 + runAsGroup: 64535 +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_deckhouse_with_writable_fs" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with user and group "deckhouse" with write access to mounted volumes */ -}} +{{- define "helm_lib_application_pod_security_context_run_as_user_deckhouse_with_writable_fs" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + runAsNonRoot: true + runAsUser: 64535 + runAsGroup: 64535 + fsGroup: 64535 +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_run_as_user_deckhouse_pss_restricted" . }} */ -}} +{{- /* returns SecurityContext parameters for Container with user and group "deckhouse" plus minimal required settings to comply with the Restricted mode of the Pod Security Standards */ -}} +{{- define "helm_lib_application_container_security_context_run_as_user_deckhouse_pss_restricted" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + runAsGroup: 64535 + runAsNonRoot: true + runAsUser: 64535 + seccompProfile: + type: RuntimeDefault +{{- end }} + + +{{- /* SecurityContext for Deckhouse UID/GID 64535 (or root), PSS Restricted */ -}} +{{- /* Optional keys: */ -}} +{{- /* .ro – bool, read-only root FS (default true) */ -}} +{{- /* .caps – []string, capabilities.add (default empty) */ -}} +{{- /* .uid – int, runAsUser/runAsGroup (default 64535) */ -}} +{{- /* .runAsNonRoot – bool, run as Deckhouse user when true, root when false (default true) */ -}} +{{- /* .seccompProfile – bool, disable seccompProfile when false (default true) */ -}} +{{- /* Usage: include "helm_lib_application_container_security_context_pss_restricted_flexible" (dict "ro" false "caps" (list "NET_ADMIN" "SYS_TIME") "uid" 1001 "seccompProfile" false "runAsNonRoot" true) */ -}} +{{- define "helm_lib_application_container_security_context_pss_restricted_flexible" -}} +{{- $ro := true -}} +{{- if hasKey . "ro" -}} + {{- $ro = .ro -}} +{{- end -}} +{{- $seccompProfile := true -}} +{{- if hasKey . "seccompProfile" -}} + {{- $seccompProfile = .seccompProfile -}} +{{- end -}} +{{- $caps := default (list) .caps -}} +{{- $uid := default 64535 .uid -}} +{{- $runAsNonRoot := true -}} +{{- if hasKey . "runAsNonRoot" -}} + {{- $runAsNonRoot = .runAsNonRoot -}} +{{- end -}} + +securityContext: + readOnlyRootFilesystem: {{ $ro }} + allowPrivilegeEscalation: {{ not $runAsNonRoot }} +{{- if $runAsNonRoot }} + privileged: false +{{- end }} + capabilities: + drop: + - ALL +{{- if $caps }} + add: {{ $caps | toJson }} +{{- end }} + runAsUser: {{ ternary $uid 0 $runAsNonRoot }} + runAsGroup: {{ ternary $uid 0 $runAsNonRoot }} + runAsNonRoot: {{ $runAsNonRoot }} +{{- if $seccompProfile }} + seccompProfile: + type: RuntimeDefault +{{- end }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_root" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with user and group 0 */ -}} +{{- define "helm_lib_application_pod_security_context_run_as_user_root" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_pod_security_context_runtime_default" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with seccomp profile RuntimeDefault */ -}} +{{- define "helm_lib_application_pod_security_context_runtime_default" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + seccompProfile: + type: RuntimeDefault +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_not_allow_privilege_escalation" . }} */ -}} +{{- /* returns SecurityContext parameters for Container with allowPrivilegeEscalation false */ -}} +{{- define "helm_lib_application_container_security_context_not_allow_privilege_escalation" -}} +securityContext: + allowPrivilegeEscalation: false +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_read_only_root_filesystem_with_selinux" . }} */ -}} +{{- /* returns SecurityContext parameters for Container with read only root filesystem and options for SELinux compatibility*/ -}} +{{- define "helm_lib_application_container_security_context_read_only_root_filesystem_with_selinux" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + seLinuxOptions: + level: 's0' + type: 'spc_t' +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_read_only_root_filesystem" . }} */ -}} +{{- /* returns SecurityContext parameters for Container with read only root filesystem */ -}} +{{- define "helm_lib_application_container_security_context_read_only_root_filesystem" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_privileged" . }} */ -}} +{{- /* returns SecurityContext parameters for Container running privileged */ -}} +{{- define "helm_lib_application_container_security_context_privileged" -}} +securityContext: + privileged: true +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_escalated_sys_admin_privileged" . }} */ -}} +{{- /* returns SecurityContext parameters for Container running privileged with escalation and sys_admin */ -}} +{{- define "helm_lib_application_container_security_context_escalated_sys_admin_privileged" -}} +securityContext: + allowPrivilegeEscalation: true + readOnlyRootFilesystem: true + capabilities: + add: + - SYS_ADMIN + privileged: true +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_privileged_read_only_root_filesystem" . }} */ -}} +{{- /* returns SecurityContext parameters for Container running privileged with read only root filesystem */ -}} +{{- define "helm_lib_application_container_security_context_privileged_read_only_root_filesystem" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + privileged: true + readOnlyRootFilesystem: true +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all" . }} */ -}} +{{- /* returns SecurityContext for Container with read only root filesystem and all capabilities dropped */ -}} +{{- define "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} */ -}} +{{- /* returns SecurityContext parameters for Container with read only root filesystem, all dropped and some added capabilities */ -}} +{{- define "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +{{- /* List of capabilities */ -}} +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + add: {{ index . 1 | toJson }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} */ -}} +{{- /* returns SecurityContext parameters for Container with all dropped and some added capabilities */ -}} +{{- define "helm_lib_application_container_security_context_capabilities_drop_all_and_add" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +{{- /* List of capabilities */ -}} +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + add: {{ index . 1 | toJson }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_capabilities_drop_all_and_run_as_user_custom" (list . 1000 1000) }} */ -}} +{{- /* returns SecurityContext parameters for Container with read only root filesystem, all dropped, and custom user ID */ -}} +{{- define "helm_lib_application_container_security_context_capabilities_drop_all_and_run_as_user_custom" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +{{- /* User id */ -}} +{{- /* Group id */ -}} +securityContext: + runAsUser: {{ index . 1 }} + runAsGroup: {{ index . 2 }} + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault +{{- end }} + +{{- /* Usage: {{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . }} */ -}} +{{- /* returns SecurityContext parameters for Container with minimal required settings to comply with the Restricted mode of the Pod Security Standards */ -}} +{{- define "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault +{{- end }} diff --git a/charts/helm_lib/templates/_capi_controller_manager.tpl b/charts/helm_lib/templates/_capi_controller_manager.tpl new file mode 100644 index 000000000..97d993500 --- /dev/null +++ b/charts/helm_lib/templates/_capi_controller_manager.tpl @@ -0,0 +1,249 @@ +{{- define "capi_controller_manager_resources" -}} +cpu: 25m +memory: 50Mi +{{- end -}} + +{{- define "capi_controller_manager_max_allowed_resources" -}} +cpu: 50m +memory: 50Mi +{{- end -}} + +{{- define "capi_controller_manager_liveness_probe" -}} +httpGet: + path: /healthz + port: 8081 +initialDelaySeconds: 15 +periodSeconds: 20 +{{- end -}} + +{{- define "capi_controller_manager_readiness_probe" -}} +httpGet: + path: /readyz + port: 8081 +initialDelaySeconds: 5 +periodSeconds: 10 +{{- end -}} + +{{- /* Usage: {{ include "helm_lib_capi_controller_manager_manifests" (list . $config) }} */ -}} +{{- /* Renders common manifests for provider-specific CAPI Controller Managers. */ -}} +{{- /* Includes Deployment, VerticalPodAutoscaler (optional) and PodDisruptionBudget (optional). */ -}} +{{- /* Supported configuration parameters: */ -}} +{{- /* + fullname (required) — resource base name used for Deployment, PDB, VPA, and by default for the main container name. */ -}} +{{- /* + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. */ -}} +{{- /* + image (required) — image for the main container. */ -}} +{{- /* + capiProviderName (required) — value for the cluster.x-k8s.io/provider label in selectors and pod labels. */ -}} +{{- /* + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. */ -}} +{{- /* + priorityClassName (optional, default: `"system-cluster-critical"`) — Pod priority class name. */ -}} +{{- /* + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. */ -}} +{{- /* + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. */ -}} +{{- /* + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. */ -}} +{{- /* + terminationGracePeriodSeconds (optional, default: `10`) — Pod termination grace period. */ -}} +{{- /* + hostNetwork (optional, default: `false`) — enables host networking for the Pod. */ -}} +{{- /* + dnsPolicy (optional, default: `nil`) — Pod DNS policy; if not set, the field is omitted. */ -}} +{{- /* + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. */ -}} +{{- /* + tolerationsStrategies (optional, default: `["any-node", "uninitialized"]`) — arguments passed to helm_lib_tolerations. */ -}} +{{- /* + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 8081}, initialDelaySeconds: 15, periodSeconds: 20}`) — liveness probe configuration for the main container. */ -}} +{{- /* + readinessProbe (optional, default: `{httpGet: {path: /readyz, port: 8081}, initialDelaySeconds: 5, periodSeconds: 10}`) — readiness probe configuration for the main container. */ -}} +{{- /* + additionalArgs (optional, default: `[]`) — extra args for the main container. */ -}} +{{- /* + additionalEnv (optional, default: `[]`) — extra environment variables for the main container. */ -}} +{{- /* + additionalPorts (optional, default: `[]`) — extra container ports for the main container. */ -}} +{{- /* + additionalInitContainers (optional, default: `[]`) — extra initContainers for the Pod. */ -}} +{{- /* + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. */ -}} +{{- /* + additionalVolumes (optional, default: `[]`) — extra Pod volumes. */ -}} +{{- /* + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. */ -}} +{{- /* + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. */ -}} +{{- /* + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. */ -}} +{{- /* + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. */ -}} +{{- /* + vpaEnabled (optional, default: `false`) — enables VerticalPodAutoscaler rendering. */ -}} +{{- /* + vpaUpdateMode (optional, default: `"InPlaceOrRecreate"`) — VPA update mode. */ -}} +{{- /* + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. */ -}} +{{- /* + securityPolicyExceptionEnabled (optional, default: `false`) — enables SecurityPolicyException rendering and adds the related pod label. */ -}} +{{- define "helm_lib_capi_controller_manager_manifests" -}} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc. */ -}} + {{- $config := index . 1 -}} {{- /* Configuration dict for the CAPI Controller Manager. */ -}} + + {{- $fullname := required "helm_lib_capi_controller_manager_manifests: fullname is required" $config.fullname -}} + {{- $namespace := dig "namespace" (printf "d8-%s" $context.Chart.Name) $config -}} + {{- $image := required "helm_lib_capi_controller_manager_manifests: image is required" $config.image -}} + {{- $capiProviderName := required "helm_lib_capi_controller_manager_manifests: $capiProviderName is required" $config.capiProviderName -}} + {{- $resources := dig "resources" (include "capi_controller_manager_resources" $context | fromYaml) $config -}} + {{- $priorityClassName := dig "priorityClassName" "system-cluster-critical" $config -}} + {{- $serviceAccountName := dig "serviceAccountName" $fullname $config -}} + {{- $automountServiceAccountToken := dig "automountServiceAccountToken" true $config -}} + {{- $revisionHistoryLimit := dig "revisionHistoryLimit" 2 $config -}} + {{- $terminationGracePeriodSeconds := dig "terminationGracePeriodSeconds" 10 $config -}} + {{- $hostNetwork := dig "hostNetwork" false $config -}} + {{- $dnsPolicy := dig "dnsPolicy" nil $config -}} + {{- $nodeSelectorStrategy := dig "nodeSelectorStrategy" "master" $config -}} + {{- $tolerationsStrategies := dig "tolerationsStrategies" (list "any-node" "uninitialized") $config -}} + {{- $livenessProbe := dig "livenessProbe" (include "capi_controller_manager_liveness_probe" $context | fromYaml) $config }} + {{- $readinessProbe := dig "readinessProbe" (include "capi_controller_manager_readiness_probe" $context | fromYaml) $config }} + {{- $additionalArgs := dig "additionalArgs" (list) $config -}} + {{- $additionalEnv := dig "additionalEnv" (list) $config -}} + {{- $additionalPorts := dig "additionalPorts" (list) $config -}} + {{- $additionalInitContainers := dig "additionalInitContainers" (list) $config -}} + {{- $additionalVolumeMounts := dig "additionalVolumeMounts" (list) $config -}} + {{- $additionalVolumes := dig "additionalVolumes" (list) $config -}} + {{- $additionalPodLabels := dig "additionalPodLabels" (dict) $config -}} + {{- $additionalPodAnnotations := dig "additionalPodAnnotations" (dict) $config -}} + {{- $pdbEnabled := dig "pdbEnabled" true $config -}} + {{- $pdbMaxUnavailable := dig "pdbMaxUnavailable" 1 $config -}} + {{- $vpaEnabled := dig "vpaEnabled" false $config -}} + {{- $vpaUpdateMode := dig "vpaUpdateMode" "InPlaceOrRecreate" $config -}} + {{- $vpaMaxAllowed := dig "vpaMaxAllowed" (include "capi_controller_manager_max_allowed_resources" $context | fromYaml) $config -}} + {{- $securityPolicyExceptionEnabled := dig "securityPolicyExceptionEnabled" false $config }} + +{{- if and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $fullname }} + updatePolicy: + updateMode: {{ $vpaUpdateMode | quote }} + resourcePolicy: + containerPolicies: + - containerName: {{ $fullname | quote }} + minAllowed: + {{- toYaml $resources | nindent 10 }} + maxAllowed: + {{- toYaml $vpaMaxAllowed | nindent 10 }} +{{- end }} + +{{- if $pdbEnabled }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +spec: + maxUnavailable: {{ $pdbMaxUnavailable }} + selector: + matchLabels: + app: {{ $fullname }} +{{- end }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +spec: + {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" $context | nindent 2 }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + selector: + matchLabels: + app: {{ $fullname }} + cluster.x-k8s.io/provider: {{ $capiProviderName }} + control-plane: controller-manager + template: + metadata: + labels: + app: {{ $fullname }} + cluster.x-k8s.io/provider: {{ $capiProviderName }} + control-plane: controller-manager + {{- with $additionalPodLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $additionalPodAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + imagePullSecrets: + - name: deckhouse-registry + {{- include "helm_lib_pod_anti_affinity_for_ha" (list $context (dict "app" $fullname)) | nindent 6 }} + {{- include "helm_lib_priority_class" (tuple $context $priorityClassName) | nindent 6 }} + {{- include "helm_lib_node_selector" (tuple $context $nodeSelectorStrategy) | nindent 6 }} + {{- include "helm_lib_tolerations" (concat (list $context) $tolerationsStrategies) | nindent 6 }} + {{- include "helm_lib_module_pod_security_context_run_as_user_deckhouse" $context | nindent 6 }} + automountServiceAccountToken: {{ $automountServiceAccountToken }} + serviceAccountName: {{ $serviceAccountName }} + terminationGracePeriodSeconds: {{ $terminationGracePeriodSeconds }} + hostNetwork: {{ $hostNetwork }} + {{- with $dnsPolicy }} + dnsPolicy: {{ . }} + {{- end }} + {{- with $additionalInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ $fullname }} + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" dict | nindent 8 }} + image: {{ $image }} + imagePullPolicy: IfNotPresent + args: + - --leader-elect + {{- with $additionalArgs }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $additionalEnv }} + env: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $additionalPorts }} + ports: + {{- toYaml . | nindent 10 }} + {{- end }} + livenessProbe: + {{- with $livenessProbe }} + {{- toYaml . | nindent 10 }} + {{- end }} + readinessProbe: + {{- with $readinessProbe }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $additionalVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 10 }} + {{- end }} + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- if not (and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd")) }} + {{- toYaml $resources | nindent 12 }} + {{- end }} + {{- with $additionalVolumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + +{{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} +{{- if $hostNetwork }} +--- +apiVersion: deckhouse.io/v1alpha1 +kind: SecurityPolicyException +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} +spec: + network: + {{- if $additionalPorts }} + hostPorts: + {{- range $additionalPorts }} + - port: {{ .containerPort }} + protocol: {{ .protocol | default "TCP" }} + {{- end }} + {{- end }} + hostNetwork: + allowedValue: true + metadata: + description: | + Allow host network access for CAPI infrastructure controller manager. + The CAPI infrastructure controller manager requires host network access to continue infrastructure reconciliation even if the CNI or pod network is unavailable. +{{- end }} +{{- end }} + +{{- end -}} \ No newline at end of file diff --git a/charts/helm_lib/templates/_cloud_controller_manager.tpl b/charts/helm_lib/templates/_cloud_controller_manager.tpl new file mode 100644 index 000000000..e8820a778 --- /dev/null +++ b/charts/helm_lib/templates/_cloud_controller_manager.tpl @@ -0,0 +1,280 @@ +{{- define "cloud_controller_manager_resources" }} +cpu: 25m +memory: 50Mi +{{- end }} + +{{- define "cloud_controller_manager_max_allowed_resources" }} +cpu: 50m +memory: 50Mi +{{- end }} + +{{- define "cloud_controller_manager_liveness_probe" -}} +httpGet: + path: /healthz + port: 10471 + host: 127.0.0.1 + scheme: HTTPS +{{- end -}} + +{{- define "cloud_controller_manager_readiness_probe" -}} +httpGet: + path: /healthz + port: 10471 + host: 127.0.0.1 + scheme: HTTPS +{{- end -}} + +{{- /* Usage: {{ include "helm_lib_cloud_controller_manager_manifests" (list . $config) }} */ -}} +{{- /* Renders common manifests for provider-specific Cloud Controller Managers. */ -}} +{{- /* Includes Deployment, VerticalPodAutoscaler (optional), PodDisruptionBudget (optional), and SecurityPolicyException (optional). */ -}} +{{- /* Supported configuration parameters: */ -}} +{{- /* + fullname (optional, default: `"cloud-controller-manager"`) — resource base name used for Deployment, PDB, VPA, SecurityPolicyException, and the main container name by default. */ -}} +{{- /* + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. */ -}} +{{- /* + image (required) — image for the main container. */ -}} +{{- /* + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. */ -}} +{{- /* + priorityClassName (optional, default: `"system-cluster-critical"`) — Pod priority class name. */ -}} +{{- /* + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. */ -}} +{{- /* + tolerationsStrategies (optional, default: ["wildcard"]) — strategies passed to helm_lib_tolerations. */ -}} +{{- /* + hostNetwork (optional, default: `true`) — enables host networking for the Pod and SecurityPolicyException network rule generation. */ -}} +{{- /* + dnsPolicy (optional, default: `"Default"`) — Pod DNS policy. */ -}} +{{- /* + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. */ -}} +{{- /* + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. */ -}} +{{- /* + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. */ -}} +{{- /* + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 10471, host: 127.0.0.1, scheme: HTTPS}}`) — liveness probe configuration for the main container. */ -}} +{{- /* + readinessProbe (optional, default: `{httpGet: {path: /healthz, port: 10471, host: 127.0.0.1, scheme: HTTPS}}`) — readiness probe configuration for the main container. */ -}} +{{- /* + additionalEnvs (optional, default: `[]`) — extra environment variables for the main container. */ -}} +{{- /* + additionalArgs (optional, default: `nil`) — extra args for the main container. */ -}} +{{- /* + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. */ -}} +{{- /* + additionalVolumes (optional, default: `[]`) — extra Pod volumes; hostPath volumes are also used to build SecurityPolicyException rules when enabled. */ -}} +{{- /* + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. */ -}} +{{- /* + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. */ -}} +{{- /* + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. */ -}} +{{- /* + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. */ -}} +{{- /* + additionalPDBAnnotations (optional, default: `{}`) — extra annotations added to PodDisruptionBudget metadata. */ -}} +{{- /* + vpaEnabled (optional, default: `true`) — enables VerticalPodAutoscaler rendering. */ -}} +{{- /* + vpaUpdateMode (optional, default: `"InPlaceOrRecreate"`) — VPA update mode. */ -}} +{{- /* + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. */ -}} +{{- /* + securityPolicyExceptionEnabled (optional, default: `false`) — enables SecurityPolicyException rendering and adds the related pod label. */ -}} +{{- define "helm_lib_cloud_controller_manager_manifests" }} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc. */ -}} + {{- $config := index . 1 -}} {{- /* Configuration dict for the Cloud Controller Manager. */ -}} + + {{- $fullname := dig "fullname" "cloud-controller-manager" $config }} + {{- $namespace := dig "namespace" (printf "d8-%s" $context.Chart.Name) $config -}} + {{- $image := $config.image | required "image is required" }} + {{- $resources := dig "resources" (include "cloud_controller_manager_resources" $context | fromYaml) $config }} + {{- $priorityClassName := dig "priorityClassName" "system-cluster-critical" $config }} + {{- $nodeSelectorStrategy := dig "nodeSelectorStrategy" "master" $config -}} + {{- $tolerationsStrategies := dig "tolerationsStrategies" (list "wildcard") $config -}} + {{- $hostNetwork := dig "hostNetwork" true $config }} + {{- $dnsPolicy := dig "dnsPolicy" "Default" $config }} + {{- $automountServiceAccountToken := dig "automountServiceAccountToken" true $config }} + {{- $serviceAccountName := dig "serviceAccountName" $fullname $config }} + {{- $revisionHistoryLimit := dig "revisionHistoryLimit" 2 $config }} + {{- $livenessProbe := dig "livenessProbe" (include "cloud_controller_manager_liveness_probe" $context | fromYaml) $config }} + {{- $readinessProbe := dig "readinessProbe" (include "cloud_controller_manager_readiness_probe" $context | fromYaml) $config }} + {{- $additionalEnvs := dig "additionalEnvs" (list) $config }} + {{- $additionalArgs := dig "additionalArgs" nil $config }} + {{- $additionalVolumeMounts := dig "additionalVolumeMounts" (list) $config }} + {{- $additionalVolumes := dig "additionalVolumes" (list) $config }} + {{- $additionalPodLabels := dig "additionalPodLabels" (dict) $config }} + {{- $additionalPodAnnotations := dig "additionalPodAnnotations" (dict) $config }} + {{- $pdbEnabled := dig "pdbEnabled" true $config }} + {{- $pdbMaxUnavailable := dig "pdbMaxUnavailable" 1 $config }} + {{- $additionalPDBAnnotations := dig "additionalPDBAnnotations" (dict) $config }} + {{- $vpaEnabled := dig "vpaEnabled" true $config }} + {{- $vpaUpdateMode := dig "vpaUpdateMode" "InPlaceOrRecreate" $config }} + {{- $vpaMaxAllowed := dig "vpaMaxAllowed" (include "cloud_controller_manager_max_allowed_resources" $context | fromYaml) $config }} + {{- $securityPolicyExceptionEnabled := dig "securityPolicyExceptionEnabled" false $config }} + +{{- if and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $fullname }} + updatePolicy: + updateMode: {{ $vpaUpdateMode }} + resourcePolicy: + containerPolicies: + - containerName: {{ $fullname | quote }} + minAllowed: + {{- toYaml $resources | nindent 10 }} + maxAllowed: + {{- toYaml $vpaMaxAllowed | nindent 10 }} +{{- end }} + +{{- if $pdbEnabled }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} + {{- with $additionalPDBAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + maxUnavailable: {{ $pdbMaxUnavailable }} + selector: + matchLabels: + app: {{ $fullname }} +{{- end }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +spec: + {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" $context | nindent 2 }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + selector: + matchLabels: + app: {{ $fullname }} + template: + metadata: + labels: + app: {{ $fullname }} + {{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} + security.deckhouse.io/security-policy-exception: {{ $fullname }} + {{- end }} + {{- with $additionalPodLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $additionalPodAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + imagePullSecrets: + - name: deckhouse-registry + {{- include "helm_lib_pod_anti_affinity_for_ha" (list $context (dict "app" $fullname)) | nindent 6 }} + {{- include "helm_lib_priority_class" (tuple $context $priorityClassName) | nindent 6 }} + {{- include "helm_lib_node_selector" (tuple $context $nodeSelectorStrategy) | nindent 6 }} + {{- include "helm_lib_tolerations" (concat (list $context) $tolerationsStrategies) | nindent 6 }} + {{- include "helm_lib_module_pod_security_context_run_as_user_deckhouse" $context | nindent 6 }} + automountServiceAccountToken: {{ $automountServiceAccountToken }} + hostNetwork: {{ $hostNetwork }} + dnsPolicy: {{ $dnsPolicy }} + serviceAccountName: {{ $serviceAccountName }} + containers: + - name: {{ $fullname }} + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" dict | nindent 10 }} + image: {{ $image }} + args: + - --leader-elect=true + - --bind-address=127.0.0.1 + - --secure-port=10471 + {{- with $additionalArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- if not $context.Values.global.clusterIsBootstrapped }} + - name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: KUBERNETES_SERVICE_PORT + value: "6443" + {{- end }} + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- with $additionalEnvs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- include "helm_lib_envs_for_proxy" $context | nindent 12 }} + livenessProbe: + {{- with $livenessProbe }} + {{- toYaml . | nindent 12 }} + {{- end }} + readinessProbe: + {{- with $readinessProbe }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with $additionalVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 14 }} + {{- if not (and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd")) }} + {{- toYaml $resources | nindent 14 }} + {{- end }} + {{- with $additionalVolumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + +{{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} +--- +apiVersion: deckhouse.io/v1alpha1 +kind: SecurityPolicyException +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} +spec: + {{- if $hostNetwork }} + network: + hostNetwork: + allowedValue: true + metadata: + description: | + Allow host network access for Cloud Controller Manager. + The Cloud Controller Manager requires host network access to communicate with the API for managing infrastructure resources, including load balancer configuration, node lifecycle management, and routing operations. + {{- end }} + + {{- $hasHostPathVolumes := false }} + {{- if $additionalVolumes }} + {{- range $volume := $additionalVolumes }} + {{- if $volume.hostPath }} + {{- $hasHostPathVolumes = true }} + {{- end }} + {{- end }} + {{- end }} + {{- if $hasHostPathVolumes }} + volumes: + types: + allowedValues: + - hostPath + metadata: + description: | + Allow hostPath volume type for Cloud Controller Manager. + The Cloud Controller Manager requires hostPath volumes for accessing host-level resources needed for cloud provider integration and infrastructure management operations. + hostPath: + allowedValues: + {{- range $volume := $additionalVolumes }} + {{- if $volume.hostPath }} + {{- $readOnly := false }} + {{- range $volumeMount := $additionalVolumeMounts }} + {{- if eq $volumeMount.name $volume.name }} + {{- $readOnly = (default false $volumeMount.readOnly) }} + {{- end }} + {{- end }} + - path: {{ $volume.hostPath.path }} + readOnly: {{ $readOnly }} + metadata: + description: | + Allow access to additional hostPath volume at {{ $volume.hostPath.path }}. + This additional hostPath volume is required by the Cloud Controller Manager for provider-specific infrastructure management operations. + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- end }} diff --git a/charts/helm_lib/templates/_cloud_data_discoverer.tpl b/charts/helm_lib/templates/_cloud_data_discoverer.tpl new file mode 100644 index 000000000..c9c6eb50a --- /dev/null +++ b/charts/helm_lib/templates/_cloud_data_discoverer.tpl @@ -0,0 +1,311 @@ +{{- define "cloud_data_discoverer_resources" -}} +cpu: 25m +memory: 50Mi +{{- end -}} + +{{- define "cloud_data_discoverer_max_allowed_resources" -}} +cpu: 50m +memory: 50Mi +{{- end -}} + +{{- define "cloud_data_discoverer_liveness_probe" -}} +httpGet: + path: /healthz + port: 8080 + scheme: HTTPS +{{- end -}} + +{{- define "cloud_data_discoverer_readiness_probe" -}} +httpGet: + path: /healthz + port: 8080 + scheme: HTTPS +{{- end -}} + +{{- /* Usage: {{ include "helm_lib_cloud_data_discoverer_manifests" (list . $config) }} */ -}} +{{- /* Renders common manifests for provider-specific Cloud Data Discoverers. */ -}} +{{- /* Includes Deployment, VerticalPodAutoscaler (optional) and PodDisruptionBudget (optional). */ -}} +{{- /* Supported configuration parameters: */ -}} +{{- /* + fullname (optional, default: `"cloud-data-discoverer"`) — resource base name used for Deployment, PDB, VPA, and the main container name by default. */ -}} +{{- /* + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. */ -}} +{{- /* + image (required) — image for the main container. */ -}} +{{- /* + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. */ -}} +{{- /* + replicas (optional, default: `1`) — number of Deployment replicas. */ -}} +{{- /* + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. */ -}} +{{- /* + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. */ -}} +{{- /* + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. */ -}} +{{- /* + priorityClassName (optional, default: `"cluster-low"`) — Pod priority class name. */ -}} +{{- /* + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. */ -}} +{{- /* + tolerationsStrategies (optional, default: `["any-node", "with-uninitialized"]`) — strategies passed to helm_lib_tolerations. */ -}} +{{- /* + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 8080, scheme: HTTPS}}`) — liveness probe configuration for the main container. */ -}} +{{- /* + readinessProbe (optional, default: `{httpGet: {path: /healthz, port: 8080, scheme: HTTPS}}`) — readiness probe configuration for the main container. */ -}} +{{- /* + additionalArgs (optional, default: `[]`) — extra args for the main container. */ -}} +{{- /* + additionalEnv (optional, default: `[]`) — extra environment variables for the main container. */ -}} +{{- /* + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. */ -}} +{{- /* + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. */ -}} +{{- /* + additionalInitContainers (optional, default: `[]`) — extra initContainers for the Pod. */ -}} +{{- /* + additionalVolumes (optional, default: `[]`) — extra Pod volumes. */ -}} +{{- /* + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. */ -}} +{{- /* + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. */ -}} +{{- /* + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. */ -}} +{{- /* + vpaEnabled (optional, default: `true`) — enables VerticalPodAutoscaler rendering. */ -}} +{{- /* + vpaUpdateMode (optional, default: `"Initial"`) — VPA update mode. */ -}} +{{- /* + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. */ -}} +{{- define "helm_lib_cloud_data_discoverer_manifests" -}} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc. */ -}} + {{- $config := index . 1 -}} {{- /* Configuration dict for the Cloud Data Discoverer. */ -}} + + {{- $fullname := dig "fullname" "cloud-data-discoverer" $config -}} + {{- $namespace := dig "namespace" (printf "d8-%s" $context.Chart.Name) $config -}} + {{- $image := required "helm_lib_cloud_data_discoverer_manifests: image is required" $config.image -}} + {{- $resources := dig "resources" (include "cloud_data_discoverer_resources" $context | fromYaml) $config -}} + {{- $replicas := dig "replicas" 1 $config -}} + {{- $revisionHistoryLimit := dig "revisionHistoryLimit" 2 $config -}} + {{- $serviceAccountName := dig "serviceAccountName" $fullname $config -}} + {{- $automountServiceAccountToken := dig "automountServiceAccountToken" true $config -}} + {{- $priorityClassName := dig "priorityClassName" "cluster-low" $config -}} + {{- $nodeSelectorStrategy := dig "nodeSelectorStrategy" "master" $config -}} + {{- $tolerationsStrategies := dig "tolerationsStrategies" (list "any-node" "with-uninitialized") $config -}} + {{- $livenessProbe := dig "livenessProbe" (include "cloud_data_discoverer_liveness_probe" $context | fromYaml) $config }} + {{- $readinessProbe := dig "readinessProbe" (include "cloud_data_discoverer_readiness_probe" $context | fromYaml) $config }} + {{- $additionalArgs := dig "additionalArgs" (list) $config -}} + {{- $additionalEnv := dig "additionalEnv" (list) $config -}} + {{- $additionalPodLabels := dig "additionalPodLabels" (dict) $config }} + {{- $additionalPodAnnotations := dig "additionalPodAnnotations" (dict) $config }} + {{- $additionalInitContainers := dig "additionalInitContainers" (list) $config -}} + {{- $additionalVolumes := dig "additionalVolumes" (list) $config -}} + {{- $additionalVolumeMounts := dig "additionalVolumeMounts" (list) $config -}} + {{- $pdbEnabled := dig "pdbEnabled" true $config -}} + {{- $pdbMaxUnavailable := dig "pdbMaxUnavailable" 1 $config -}} + {{- $vpaEnabled := dig "vpaEnabled" true $config -}} + {{- $vpaUpdateMode := dig "vpaUpdateMode" "Initial" $config -}} + {{- $vpaMaxAllowed := dig "vpaMaxAllowed" (include "cloud_data_discoverer_max_allowed_resources" $context | fromYaml) $config -}} + +{{- if and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $fullname }} + updatePolicy: + updateMode: {{ $vpaUpdateMode | quote }} + resourcePolicy: + containerPolicies: + - containerName: {{ $fullname | quote }} + minAllowed: + {{- toYaml $resources | nindent 8 }} + maxAllowed: + {{- toYaml $vpaMaxAllowed | nindent 8 }} + {{- include "helm_lib_vpa_kube_rbac_proxy_resources" $context | nindent 4 }} +{{- end }} + +{{- if $pdbEnabled }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +spec: + maxUnavailable: {{ $pdbMaxUnavailable }} + selector: + matchLabels: + app: {{ $fullname }} +{{- end }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullname }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +spec: + replicas: {{ $replicas }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + strategy: + type: Recreate + selector: + matchLabels: + app: {{ $fullname }} + template: + metadata: + labels: + app: {{ $fullname }} + {{- with $additionalPodLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + kubectl.kubernetes.io/default-exec-container: {{ $fullname }} + kubectl.kubernetes.io/default-logs-container: {{ $fullname }} + {{- with $additionalPodAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + imagePullSecrets: + - name: deckhouse-registry + {{- include "helm_lib_priority_class" (tuple $context $priorityClassName) | nindent 6 }} + {{- include "helm_lib_node_selector" (tuple $context $nodeSelectorStrategy) | nindent 6 }} + {{- include "helm_lib_tolerations" (concat (list $context) $tolerationsStrategies) | nindent 6 }} + {{- include "helm_lib_module_pod_security_context_run_as_user_deckhouse" $context | nindent 6 }} + dnsPolicy: {{ include "helm_lib_dns_policy_bootstraping_state" (list $context "Default" "ClusterFirstWithHostNet") }} + automountServiceAccountToken: {{ $automountServiceAccountToken }} + serviceAccountName: {{ $serviceAccountName }} + {{- with $additionalInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ $fullname }} + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" dict | nindent 8 }} + image: {{ $image }} + args: + - --discovery-period=1h + - --listen-address=127.0.0.1:8081 + {{- with $additionalArgs }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $additionalEnv }} + env: + {{- toYaml . | nindent 10 }} + {{- end }} + livenessProbe: + {{- with $livenessProbe }} + {{- toYaml . | nindent 10 }} + {{- end }} + readinessProbe: + {{- with $readinessProbe }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $additionalVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 10 }} + {{- end }} + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 12 }} + {{- if not (and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd")) }} + {{- toYaml $resources | nindent 12 }} + {{- end }} + - name: kube-rbac-proxy + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" dict | nindent 8 }} + image: {{ include "helm_lib_module_common_image" (list $context "kubeRbacProxy") }} + args: + - "--secure-listen-address=$(KUBE_RBAC_PROXY_LISTEN_ADDRESS):8080" + - "--v=2" + - "--logtostderr=true" + - "--stale-cache-interval=1h30m" + - "--livez-path=/livez" + ports: + - containerPort: 8080 + name: https-metrics + env: + - name: KUBE_RBAC_PROXY_LISTEN_ADDRESS + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBE_RBAC_PROXY_CONFIG + value: | + excludePaths: + - /healthz + upstreams: + - upstream: http://127.0.0.1:8081/ + path: / + authorization: + resourceAttributes: + namespace: {{ $namespace }} + apiGroup: apps + apiVersion: v1 + resource: deployments + subresource: prometheus-metrics + name: {{ $fullname }} + livenessProbe: + httpGet: + path: /livez + port: 8080 + scheme: HTTPS + readinessProbe: + httpGet: + path: /livez + port: 8080 + scheme: HTTPS + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 12 }} + {{- if not (and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd")) }} + {{- include "helm_lib_container_kube_rbac_proxy_resources" $context | nindent 12 }} + {{- end }} + {{- with $additionalVolumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_cloud_data_discoverer_pod_monitor" (list . $config) }} */ -}} +{{- /* Renders PodMonitor manifest for provider-specific Cloud Data Discoverers. */ -}} +{{- /* Supported configuration parameters: */ -}} +{{- /* + fullname (optional, default: `"cloud-data-discoverer"`) — PodMonitor base name. */ -}} +{{- /* + targetNamespace (required) — target pod namespace for selector. */ -}} +{{- /* + additionalRelabelings (optional, default: `[]`) — additional rules for labels rewriting. */ -}} +{{- define "helm_lib_cloud_data_discoverer_pod_monitor" -}} + {{- $context := index . 0 -}} + {{- $config := index . 1 -}} + + {{- $fullname := dig "fullname" "cloud-data-discoverer" $config -}} + {{- $targetNamespace := required "helm_lib_cloud_data_discoverer_pod_monitor: targetNamespace is required" $config.targetNamespace -}} + {{- $additionalRelabelings := dig "additionalRelabelings" (list) $config -}} + +{{- if ($context.Values.global.enabledModules | has "operator-prometheus-crd") -}} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ $fullname }}-metrics + namespace: d8-monitoring + {{- include "helm_lib_module_labels" (list $context (dict "prometheus" "main" "app" $fullname)) | nindent 2 }} +spec: + jobLabel: app + podMetricsEndpoints: + - port: https-metrics + path: /metrics + scheme: https + bearerTokenSecret: + name: prometheus-token + key: token + tlsConfig: + insecureSkipVerify: true + honorLabels: true + scrapeTimeout: {{ include "helm_lib_prometheus_target_scrape_timeout_seconds" (list $context 25) }} + relabelings: + - regex: "endpoint|pod|container" + action: labeldrop + - targetLabel: job + replacement: {{ $fullname }} + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + - targetLabel: tier + replacement: cluster + - sourceLabels: [__meta_kubernetes_pod_ready] + regex: "true" + action: keep + {{- with $additionalRelabelings }} + {{- toYaml . | nindent 6 }} + {{- end }} + selector: + matchLabels: + app: {{ $fullname }} + namespaceSelector: + matchNames: + - {{ $targetNamespace }} + + {{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/helm_lib/templates/_cloud_provider_user_authz_roles.tpl b/charts/helm_lib/templates/_cloud_provider_user_authz_roles.tpl new file mode 100644 index 000000000..2de128a71 --- /dev/null +++ b/charts/helm_lib/templates/_cloud_provider_user_authz_roles.tpl @@ -0,0 +1,83 @@ +{{- /* Usage: {{- include "helm_lib_cloud_provider_user_authz_cluster_roles" (list . $config) }} */ -}} +{{- /* Renders user-authz ClusterRoles for provider-specific cloud resources. */ -}} +{{- /* Includes User and ClusterAdmin ClusterRoles. */ -}} +{{- /* Supported configuration parameters: */ -}} +{{- /* + providerName (required) — provider name segment used in ClusterRole names. */ -}} +{{- /* + instanceClassResource (required) — Deckhouse instance class resource name granted by the rules. */ -}} +{{- /* + capiResources (optional, default: `[]`) — CAPI infrastructure resource names granted by the rules. */ -}} +{{- /* + additionalUserRules (optional, default: `[]`) — extra rules appended to the User ClusterRole. */ -}} +{{- /* + additionalClusterAdminRules (optional, default: `[]`) — extra rules appended to the ClusterAdmin ClusterRole. */ -}} +{{- define "helm_lib_cloud_provider_user_authz_cluster_roles" -}} + {{- $context := index . 0 -}} + {{- $config := index . 1 -}} + {{- $providerName := required "helm_lib_cloud_provider_user_authz_cluster_roles: providerName is required" (get $config "providerName") -}} + {{- $instanceClassResource := required "helm_lib_cloud_provider_user_authz_cluster_roles: instanceClassResource is required" (get $config "instanceClassResource") -}} + {{- $capiResources := dig "capiResources" (list) $config -}} + {{- $additionalUserRules := dig "additionalUserRules" (list) $config -}} + {{- $additionalClusterAdminRules := dig "additionalClusterAdminRules" (list) $config -}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + user-authz.deckhouse.io/access-level: User + name: d8:user-authz:{{ $providerName }}:user + {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} +rules: +- apiGroups: + - deckhouse.io + resources: + - {{ $instanceClassResource }} + verbs: + - get + - list + - watch +{{- if $capiResources }} +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + {{- range $capiResources }} + - {{ . }} + {{- end }} + verbs: + - get + - list + - watch +{{- end }} +{{- with $additionalUserRules }} +{{ toYaml . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + user-authz.deckhouse.io/access-level: ClusterAdmin + name: d8:user-authz:{{ $providerName }}:cluster-admin + {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} +rules: +- apiGroups: + - deckhouse.io + resources: + - {{ $instanceClassResource }} + verbs: + - create + - delete + - deletecollection + - patch + - update +{{- if $capiResources }} +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + {{- range $capiResources }} + - {{ . }} + {{- end }} + verbs: + - patch + - update +{{- end }} +{{- with $additionalClusterAdminRules }} +{{ toYaml . }} +{{- end }} +{{- end -}} diff --git a/charts/helm_lib/templates/_csi_controller.tpl b/charts/helm_lib/templates/_csi_controller.tpl new file mode 100644 index 000000000..8d717d959 --- /dev/null +++ b/charts/helm_lib/templates/_csi_controller.tpl @@ -0,0 +1,986 @@ +{{- /* Usage: {{ include "helm_lib_csi_image_with_common_fallback" (list . "" "") }} */ -}} +{{- /* returns image name from storage foundation module if enabled, otherwise from common module */ -}} +{{- define "helm_lib_csi_image_with_common_fallback" }} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $rawContainerName := index . 1 | trimAll "\"" }} {{- /* Container raw name */ -}} + {{- $kubernetesSemVer := index . 2 }} {{- /* Kubernetes semantic version */ -}} + {{- $imageDigest := "" }} + {{- $registryBase := $context.Values.global.modulesImages.registry.base }} + {{- /* Try to get from storage foundation module if enabled */}} + {{- if $context.Values.global.enabledModules | has "storage-foundation" }} + {{- $registryBase = join "/" (list $context.Values.storageFoundation.registry.base "modules" "storage-foundation" ) }} + {{- $storageFoundationDigests := index $context.Values.global.modulesImages.digests "storageFoundation" | default dict }} + {{- $currentMinor := int $kubernetesSemVer.Minor }} + {{- $kubernetesMajor := int $kubernetesSemVer.Major }} + {{- /* Iterate from currentMinor down to 0: use offset from 0 to currentMinor, then calculate minorVersion = currentMinor - offset */}} + {{- range $offset := until (int (add $currentMinor 1)) }} + {{- if not $imageDigest }} + {{- $minorVersion := int (sub $currentMinor $offset) }} + {{- $containerName := join "" (list $rawContainerName "ForK8SGE" $kubernetesMajor $minorVersion) }} + {{- $digest := index $storageFoundationDigests $containerName | default "" }} + {{- if $digest }} + {{- $imageDigest = $digest }} + {{- end }} + {{- end }} + {{- end }} + {{- /* Fallback to base container name if no versioned image found (when minor reached 0) */}} + {{- if not $imageDigest }} + {{- $imageDigest = index $storageFoundationDigests $rawContainerName | default "" }} + {{- end }} + {{- /* Fallback to common module if storage foundation module is not enabled */}} + {{- else }} + {{- $containerName := join "" (list $rawContainerName $kubernetesSemVer.Major $kubernetesSemVer.Minor) }} + {{- $imageDigest = index $context.Values.global.modulesImages.digests "common" $containerName | default "" }} + {{- end }} + {{- if $imageDigest }} + {{- printf "%s@%s" $registryBase $imageDigest }} + {{- end }} +{{- end }} + + +{{- define "attacher_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + +{{- define "provisioner_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + +{{- define "resizer_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + +{{- define "syncer_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + +{{- define "snapshotter_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + +{{- define "livenessprobe_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + +{{- define "controller_resources" }} +cpu: 10m +memory: 50Mi +{{- end }} + +{{- /* Usage: {{ include "helm_lib_csi_controller_manifests" (list . $config) }} */ -}} +{{- define "helm_lib_csi_controller_manifests" }} + {{- $context := index . 0 }} + + {{- $config := index . 1 }} + {{- $fullname := $config.fullname | default "csi-controller" }} + {{- $snapshotterEnabled := dig "snapshotterEnabled" true $config }} + {{- $snapshotterSnapshotNamePrefix := dig "snapshotterSnapshotNamePrefix" false $config }} + {{- $resizerEnabled := dig "resizerEnabled" true $config }} + {{- $syncerEnabled := dig "syncerEnabled" false $config }} + {{- $topologyEnabled := dig "topologyEnabled" true $config }} + {{- $capacityEnabled := dig "capacityEnabled" true $config }} + {{- $runAsRootUser := dig "runAsRootUser" false $config }} + {{- $extraCreateMetadataEnabled := dig "extraCreateMetadataEnabled" false $config }} + {{- $controllerImage := $config.controllerImage | required "$config.controllerImage is required" }} + {{- $provisionerTimeout := $config.provisionerTimeout | default "600s" }} + {{- $attacherTimeout := $config.attacherTimeout | default "600s" }} + {{- $resizerTimeout := $config.resizerTimeout | default "600s" }} + {{- $snapshotterTimeout := $config.snapshotterTimeout | default "600s" }} + {{- $provisionerWorkers := $config.provisionerWorkers | default "10" }} + {{- $volumeNamePrefix := $config.volumeNamePrefix }} + {{- $volumeNameUUIDLength := $config.volumeNameUUIDLength }} + {{- $attacherWorkers := $config.attacherWorkers | default "10" }} + {{- $resizerWorkers := $config.resizerWorkers | default "10" }} + {{- $snapshotterWorkers := $config.snapshotterWorkers | default "10" }} + {{- $csiControllerHaMode := $config.csiControllerHaMode | default false }} + {{- $additionalCsiControllerPodAnnotations := $config.additionalCsiControllerPodAnnotations | default false }} + {{- $additionalControllerEnvs := $config.additionalControllerEnvs }} + {{- $additionalSyncerEnvs := $config.additionalSyncerEnvs }} + {{- $additionalControllerArgs := $config.additionalControllerArgs }} + {{- $additionalControllerVolumes := $config.additionalControllerVolumes }} + {{- $additionalControllerVolumeMounts := $config.additionalControllerVolumeMounts }} + {{- $additionalControllerVPA := $config.additionalControllerVPA }} + {{- $additionalControllerPorts := $config.additionalControllerPorts }} + {{- $additionalContainers := $config.additionalContainers }} + {{- $csiControllerHostNetwork := $config.csiControllerHostNetwork | default "true" }} + {{- $csiControllerHostPID := $config.csiControllerHostPID | default "false" }} + {{- $livenessProbePort := $config.livenessProbePort | default 9808 }} + {{- $livenessProbeTimeoutSeconds := $config.livenessProbeTimeoutSeconds | default 5 }} + {{- $livenessProbeInitialDelaySeconds := $config.livenessProbeInitialDelaySeconds | default 5 }} + {{- $initContainers := $config.initContainers }} + {{- $customNodeSelector := $config.customNodeSelector }} + {{- $additionalPullSecrets := $config.additionalPullSecrets }} + {{- $forceCsiControllerPrivilegedContainer := $config.forceCsiControllerPrivilegedContainer | default false }} + {{- $dnsPolicy := $config.dnsPolicy | default "ClusterFirstWithHostNet" }} + {{- $securityPolicyExceptionEnabled := $config.securityPolicyExceptionEnabled | default false }} + + {{- $kubernetesSemVer := semver $context.Values.global.discovery.kubernetesVersion }} + + {{- $provisionerImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiExternalProvisioner" $kubernetesSemVer) }} + + {{- $attacherImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiExternalAttacher" $kubernetesSemVer) }} + + {{- $resizerImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiExternalResizer" $kubernetesSemVer) }} + + {{- $syncerImageName := join "" (list "csiVsphereSyncer" $kubernetesSemVer.Major $kubernetesSemVer.Minor) }} + {{- $syncerImage := include "helm_lib_module_common_image_no_fail" (list $context $syncerImageName) }} + + {{- $snapshotterImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiExternalSnapshotter" $kubernetesSemVer) }} + + {{- $livenessprobeImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiLivenessprobe" $kubernetesSemVer) }} + + {{- if $provisionerImage }} + {{- if ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" "csi-controller")) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: {{ $fullname }} + updatePolicy: + updateMode: "InPlaceOrRecreate" + resourcePolicy: + containerPolicies: + - containerName: "provisioner" + minAllowed: + {{- include "provisioner_resources" $context | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 50Mi + - containerName: "attacher" + minAllowed: + {{- include "attacher_resources" $context | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 50Mi + {{- if $resizerEnabled }} + - containerName: "resizer" + minAllowed: + {{- include "resizer_resources" $context | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 50Mi + {{- end }} + {{- if $syncerEnabled }} + - containerName: "syncer" + minAllowed: + {{- include "syncer_resources" $context | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 50Mi + {{- end }} + {{- if $snapshotterEnabled }} + - containerName: "snapshotter" + minAllowed: + {{- include "snapshotter_resources" $context | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 50Mi + {{- end }} + - containerName: "livenessprobe" + minAllowed: + {{- include "livenessprobe_resources" $context | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 50Mi + - containerName: "controller" + minAllowed: + {{- include "controller_resources" $context | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 100Mi + {{- if $additionalControllerVPA }} + {{- $additionalControllerVPA | toYaml | nindent 4 }} + {{- end }} + {{- end }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" "csi-controller")) | nindent 2 }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: {{ $fullname }} +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" "csi-controller")) | nindent 2 }} + +spec: + {{- if $csiControllerHaMode }} + {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" $context | nindent 2 }} + {{- else }} + replicas: 1 + strategy: + type: Recreate + {{- end }} + revisionHistoryLimit: 2 + selector: + matchLabels: + app: {{ $fullname }} + template: + metadata: + labels: + app: {{ $fullname }} + {{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} + security.deckhouse.io/security-policy-exception: {{ $fullname }} + {{- end }} + {{- if or (hasPrefix "cloud-provider-" $context.Chart.Name) ($additionalCsiControllerPodAnnotations) }} + annotations: + {{- if hasPrefix "cloud-provider-" $context.Chart.Name }} + cloud-config-checksum: {{ include (print $context.Template.BasePath "/cloud-controller-manager/secret.yaml") $context | sha256sum }} + {{- end }} + {{- if $additionalCsiControllerPodAnnotations }} + {{- $additionalCsiControllerPodAnnotations | toYaml | nindent 8 }} + {{- end }} + {{- end }} + spec: + {{- if $csiControllerHaMode }} + {{- include "helm_lib_pod_anti_affinity_for_ha" (list $context (dict "app" $fullname)) | nindent 6 }} + {{- end }} + hostNetwork: {{ $csiControllerHostNetwork }} + hostPID: {{ $csiControllerHostPID }} + {{- if eq $csiControllerHostNetwork "true" }} + dnsPolicy: {{ $dnsPolicy | quote }} + {{- end }} + imagePullSecrets: + - name: deckhouse-registry + {{- if $additionalPullSecrets }} + {{- $additionalPullSecrets | toYaml | nindent 6 }} + {{- end }} + {{- include "helm_lib_priority_class" (tuple $context "system-cluster-critical") | nindent 6 }} + {{- if $customNodeSelector }} + nodeSelector: + {{- $customNodeSelector | toYaml | nindent 8 }} + {{- else }} + {{- include "helm_lib_node_selector" (tuple $context "master") | nindent 6 }} + {{- end }} + {{- include "helm_lib_tolerations" (tuple $context "any-node" "with-uninitialized") | nindent 6 }} + {{- if $runAsRootUser }} + {{- include "helm_lib_module_pod_security_context_run_as_user_root" . | nindent 6 }} + {{- else }} + {{- include "helm_lib_module_pod_security_context_run_as_user_deckhouse" . | nindent 6 }} + {{- end }} + serviceAccountName: csi + automountServiceAccountToken: true + containers: + - name: provisioner + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} + image: {{ $provisionerImage | quote }} + args: + - "--timeout={{ $provisionerTimeout }}" + - "--v=5" + - "--csi-address=$(ADDRESS)" + {{- if $volumeNamePrefix }} + - "--volume-name-prefix={{ $volumeNamePrefix }}" + {{- end }} + {{- if $volumeNameUUIDLength }} + - "--volume-name-uuid-length={{ $volumeNameUUIDLength }}" + {{- end }} + {{- if $topologyEnabled }} + - "--feature-gates=Topology=true" + - "--strict-topology" + {{- else }} + - "--feature-gates=Topology=false" + {{- end }} + - "--default-fstype=ext4" + - "--leader-election=true" + - "--leader-election-namespace=$(NAMESPACE)" + - "--leader-election-lease-duration=30s" + - "--leader-election-renew-deadline=20s" + - "--leader-election-retry-period=5s" + {{- if $capacityEnabled }} + - "--enable-capacity" + - "--capacity-ownerref-level=2" + {{- end }} + {{- if $extraCreateMetadataEnabled }} + - "--extra-create-metadata=true" + {{- end }} + - "--worker-threads={{ $provisionerWorkers }}" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "provisioner_resources" $context | nindent 12 }} + {{- end }} + - name: attacher + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} + image: {{ $attacherImage | quote }} + args: + - "--timeout={{ $attacherTimeout }}" + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--leader-election-namespace=$(NAMESPACE)" + - "--leader-election-lease-duration=30s" + - "--leader-election-renew-deadline=20s" + - "--leader-election-retry-period=5s" + - "--worker-threads={{ $attacherWorkers }}" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "attacher_resources" $context | nindent 12 }} + {{- end }} + {{- if $resizerEnabled }} + - name: resizer + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} + image: {{ $resizerImage | quote }} + args: + - "--timeout={{ $resizerTimeout }}" + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--leader-election-namespace=$(NAMESPACE)" + - "--leader-election-lease-duration=30s" + - "--leader-election-renew-deadline=20s" + - "--leader-election-retry-period=5s" + - "--workers={{ $resizerWorkers }}" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "resizer_resources" $context | nindent 12 }} + {{- end }} + {{- end }} + {{- if $syncerEnabled }} + - name: syncer + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} + image: {{ $syncerImage | quote }} + args: + - "--leader-election" + - "--leader-election-lease-duration=30s" + - "--leader-election-renew-deadline=20s" + - "--leader-election-retry-period=10s" + {{- if $additionalControllerArgs }} + {{- $additionalControllerArgs | toYaml | nindent 8 }} + {{- end }} + {{- if $additionalSyncerEnvs }} + env: + {{- $additionalSyncerEnvs | toYaml | nindent 8 }} + {{- end }} + {{- if $additionalControllerVolumeMounts }} + volumeMounts: + {{- $additionalControllerVolumeMounts | toYaml | nindent 8 }} + {{- end }} + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "syncer_resources" $context | nindent 12 }} + {{- end }} + {{- end }} + {{- if $snapshotterEnabled }} + - name: snapshotter + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} + image: {{ $snapshotterImage | quote }} + args: + - "--timeout={{ $snapshotterTimeout }}" + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--leader-election-namespace=$(NAMESPACE)" + - "--leader-election-lease-duration=30s" + - "--leader-election-renew-deadline=20s" + - "--leader-election-retry-period=5s" + - "--worker-threads={{ $snapshotterWorkers }}" + {{- if $snapshotterSnapshotNamePrefix }} + - "--snapshot-name-prefix={{ $snapshotterSnapshotNamePrefix }}" + {{- end }} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "snapshotter_resources" $context | nindent 12 }} + {{- end }} + {{- end }} + - name: livenessprobe + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} + image: {{ $livenessprobeImage | quote }} + args: + - "--csi-address=$(ADDRESS)" + - "--probe-timeout={{ $livenessProbeTimeoutSeconds }}s" + {{- if eq $csiControllerHostNetwork "true" }} + - "--http-endpoint=$(HOST_IP):{{ $livenessProbePort }}" + {{- else }} + - "--http-endpoint=$(POD_IP):{{ $livenessProbePort }}" + {{- end }} + env: + - name: ADDRESS + value: /csi/csi.sock + {{- if eq $csiControllerHostNetwork "true" }} + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- else }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "livenessprobe_resources" $context | nindent 12 }} + {{- end }} + - name: controller +{{- if $forceCsiControllerPrivilegedContainer }} + {{- include "helm_lib_module_container_security_context_escalated_sys_admin_privileged" . | nindent 8 }} +{{- else }} + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} +{{- end }} + image: {{ $controllerImage | quote }} + args: + {{- if $additionalControllerArgs }} + {{- $additionalControllerArgs | toYaml | nindent 8 }} + {{- end }} + {{- if $additionalControllerEnvs }} + env: + {{- $additionalControllerEnvs | toYaml | nindent 8 }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: {{ $livenessProbePort }} + initialDelaySeconds: {{ $livenessProbeInitialDelaySeconds }} + timeoutSeconds: {{ $livenessProbeTimeoutSeconds }} + + {{- if $additionalControllerPorts }} + ports: + {{- $additionalControllerPorts | toYaml | nindent 8 }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /csi + {{- /* For an unknown reason vSphere csi-controller won't start without `/tmp` directory */ -}} + {{- if eq $context.Chart.Name "cloud-provider-vsphere" }} + - name: tmp + mountPath: /tmp + {{- end }} + {{- if $additionalControllerVolumeMounts }} + {{- $additionalControllerVolumeMounts | toYaml | nindent 8 }} + {{- end }} + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "controller_resources" $context | nindent 12 }} + {{- end }} + {{- if $additionalContainers }} + {{- $additionalContainers | toYaml | nindent 6 }} + {{- end }} + + {{- if $initContainers }} + initContainers: + {{- range $initContainer := $initContainers }} + - resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- $initContainer | toYaml | nindent 8 }} + {{- end }} + {{- end }} + + volumes: + - name: socket-dir + emptyDir: {} + {{- /* For an unknown reason vSphere csi-controller won't start without `/tmp` directory */ -}} + {{- if eq $context.Chart.Name "cloud-provider-vsphere" }} + - name: tmp + emptyDir: {} + {{- end }} + + {{- if $additionalControllerVolumes }} + {{- $additionalControllerVolumes | toYaml | nindent 6 }} + {{- end }} + +{{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} +--- +apiVersion: deckhouse.io/v1alpha1 +kind: SecurityPolicyException +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} +spec: +{{- if or $forceCsiControllerPrivilegedContainer $runAsRootUser (eq $csiControllerHostNetwork "true") (ne $csiControllerHostPID "false") }} + {{- if or $forceCsiControllerPrivilegedContainer $runAsRootUser }} + securityContext: + {{- if $forceCsiControllerPrivilegedContainer }} + privileged: + allowedValue: true + metadata: + description: | + Allow privileged mode for CSI Controller. + The CSI Controller requires privileged access to perform storage management operations that need direct access to host resources, including device management and volume lifecycle control. + capabilities: + allowedValues: + add: + - SYS_ADMIN + metadata: + description: | + Allow SYS_ADMIN capability for CSI Controller. + The CSI Controller requires SYS_ADMIN capability to perform privileged storage management operations such as volume provisioning, snapshotting, and direct interaction with the storage backend. + {{- end }} + {{- if $runAsRootUser }} + runAsUser: + allowedValues: + - 0 + metadata: + description: | + Allow running as root user (UID 0) for CSI Controller. + The CSI Controller requires root privileges to interact with storage backends, manage volume lifecycle operations, and access cloud provider APIs. + runAsNonRoot: + allowedValue: false + metadata: + description: | + Allow running as root for CSI Controller. + The CSI Controller requires root access for storage management operations that need elevated privileges to interact with the underlying infrastructure. + {{- end }} + {{- end }} + {{- if or (eq $csiControllerHostNetwork "true") (ne $csiControllerHostPID "false") }} + network: + {{- if eq $csiControllerHostNetwork "true" }} + hostNetwork: + allowedValue: true + metadata: + description: | + Allow host network access for CSI Controller. + The CSI Controller requires host network access to communicate with cloud provider API endpoints for volume provisioning, attachment, snapshot, and lifecycle management operations. + {{- end }} + {{- if ne $csiControllerHostPID "false" }} + hostPID: + allowedValue: true + metadata: + description: | + Allow host PID namespace access for CSI Controller. + The CSI Controller requires host PID namespace access to interact with host processes for storage operations and coordinate with system-level services. + {{- end }} + {{- end }} + {{- $hasHostPathVolumes := false }} + {{- if $additionalControllerVolumes }} + {{- range $vol := $additionalControllerVolumes }} + {{- if $vol.hostPath }} + {{- $hasHostPathVolumes = true }} + {{- end }} + {{- end }} + {{- end }} + {{- if $hasHostPathVolumes }} + volumes: + types: + allowedValues: + - hostPath + metadata: + description: | + Allow hostPath volume type for CSI Controller. + The CSI Controller requires hostPath volumes for accessing host-level resources needed for storage management operations specific to the cloud provider implementation. + hostPath: + allowedValues: + {{- range $volume := $additionalControllerVolumes }} + {{- if $volume.hostPath }} + {{- $readOnly := false }} + {{- range $volumeMount := $additionalControllerVolumeMounts }} + {{- if eq $volumeMount.name $volume.name }} + {{- $readOnly = (default false $volumeMount.readOnly) }} + {{- end }} + {{- end }} + - path: {{ $volume.hostPath.path }} + readOnly: {{ $readOnly }} + metadata: + description: | + Allow access to additional hostPath volume at {{ $volume.hostPath.path }}. + This hostPath volume is required by the CSI Controller for storage management operations specific to the cloud provider implementation. + {{- end }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_csi_controller_rbac" . }} */ -}} +{{- define "helm_lib_csi_controller_rbac" }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +automountServiceAccountToken: false + +# =========== +# provisioner +# =========== +# Source https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:csi:controller:external-provisioner + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +rules: +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +# Access to volumeattachments is only needed when the CSI driver +# has the PUBLISH_UNPUBLISH_VOLUME controller capability. +# In that case, external-provisioner will watch volumeattachments +# to determine when it is safe to delete a volume. +- apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:csi:controller:external-provisioner + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +subjects: +- kind: ServiceAccount + name: csi + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ .Chart.Name }}:csi:controller:external-provisioner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi:controller:external-provisioner + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +rules: +# Only one of the following rules for endpoints or leases is required based on +# what is set for `--leader-election-type`. Endpoints are deprecated in favor of Leases. +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +# Permissions for CSIStorageCapacity are only needed enabling the publishing +# of storage capacity information. +- apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# The GET permissions below are needed for walking up the ownership chain +# for CSIStorageCapacity. They are sufficient for deployment via +# StatefulSet (only needs to get Pod) and Deployment (needs to get +# Pod and then ReplicaSet to find the Deployment). +- apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +- apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi:controller:external-provisioner + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +subjects: +- kind: ServiceAccount + name: csi + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: Role + name: csi:controller:external-provisioner + apiGroup: rbac.authorization.k8s.io + +# ======== +# attacher +# ======== +# Source https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:csi:controller:external-attacher + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +rules: +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:csi:controller:external-attacher + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +subjects: +- kind: ServiceAccount + name: csi + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ .Chart.Name }}:csi:controller:external-attacher + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi:controller:external-attacher + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi:controller:external-attacher + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +subjects: +- kind: ServiceAccount + name: csi + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: Role + name: csi:controller:external-attacher + apiGroup: rbac.authorization.k8s.io + +# ======= +# resizer +# ======= +# Source https://github.com/kubernetes-csi/external-resizer/blob/master/deploy/kubernetes/rbac.yaml +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:csi:controller:external-resizer + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +rules: +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:csi:controller:external-resizer + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +subjects: +- kind: ServiceAccount + name: csi + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ .Chart.Name }}:csi:controller:external-resizer + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi:controller:external-resizer + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi:controller:external-resizer + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +subjects: +- kind: ServiceAccount + name: csi + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: Role + name: csi:controller:external-resizer + apiGroup: rbac.authorization.k8s.io +# ======== +# snapshotter +# ======== +# Source https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:csi:controller:external-snapshotter + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +rules: +- apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:csi:controller:external-snapshotter + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +subjects: +- kind: ServiceAccount + name: csi + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ .Chart.Name }}:csi:controller:external-snapshotter + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi:controller:external-snapshotter + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi:controller:external-snapshotter + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} +subjects: +- kind: ServiceAccount + name: csi + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: Role + name: csi:controller:external-snapshotter + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/helm_lib/templates/_csi_node.tpl b/charts/helm_lib/templates/_csi_node.tpl new file mode 100644 index 000000000..318b4bd4d --- /dev/null +++ b/charts/helm_lib/templates/_csi_node.tpl @@ -0,0 +1,397 @@ +{{- define "node_driver_registrar_resources" }} +cpu: 12m +memory: 25Mi +{{- end }} + +{{- define "node_resources" }} +cpu: 12m +memory: 25Mi +{{- end }} + +{{- /* Usage: {{ include "helm_lib_csi_node_manifests" (list . $config) }} */ -}} +{{- define "helm_lib_csi_node_manifests" }} + {{- $context := index . 0 }} + + {{- $config := index . 1 }} + {{- $fullname := $config.fullname | default "csi-node" }} + {{- $nodeImage := $config.nodeImage | required "$config.nodeImage is required" }} + {{- $driverFQDN := $config.driverFQDN | required "$config.driverFQDN is required" }} + {{- $serviceAccount := $config.serviceAccount | default "" }} + {{- $additionalNodeVPA := $config.additionalNodeVPA }} + {{- $additionalNodeEnvs := $config.additionalNodeEnvs }} + {{- $additionalNodeArgs := $config.additionalNodeArgs }} + {{- $additionalNodeVolumes := $config.additionalNodeVolumes }} + {{- $additionalNodeVolumeMounts := $config.additionalNodeVolumeMounts }} + {{- $additionalNodeLivenessProbesCmd := $config.additionalNodeLivenessProbesCmd }} + {{- $livenessProbePort := $config.livenessProbePort }} + {{- $additionalNodeSelectorTerms := $config.additionalNodeSelectorTerms }} + {{- $customNodeSelector := $config.customNodeSelector }} + {{- $forceCsiNodeAndStaticNodesDepoloy := $config.forceCsiNodeAndStaticNodesDepoloy | default false }} + {{- $setSysAdminCapability := $config.setSysAdminCapability | default false }} + {{- $additionalContainers := $config.additionalContainers }} + {{- $initContainers := $config.initContainers }} + {{- $additionalPullSecrets := $config.additionalPullSecrets }} + {{- $csiNodeLifecycle := $config.csiNodeLifecycle | default false }} + {{- $csiNodeDriverRegistrarLifecycle := $config.csiNodeDriverRegistrarLifecycle | default false }} + {{- $additionalCsiNodePodAnnotations := $config.additionalCsiNodePodAnnotations | default false }} + {{- $csiNodeHostNetwork := $config.csiNodeHostNetwork | default "true" }} + {{- $csiNodeHostPID := $config.csiNodeHostPID | default "false" }} + {{- $dnsPolicy := $config.dnsPolicy | default "ClusterFirstWithHostNet" }} + {{- $securityPolicyExceptionEnabled := $config.securityPolicyExceptionEnabled | default false }} + {{- $kubernetesSemVer := semver $context.Values.global.discovery.kubernetesVersion }} + {{- $driverRegistrarImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiNodeDriverRegistrar" $kubernetesSemVer) }} + {{- if $driverRegistrarImage }} + {{- if or $forceCsiNodeAndStaticNodesDepoloy (include "_helm_lib_cloud_or_hybrid_cluster" $context) }} + {{- if ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" "csi-node")) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: DaemonSet + name: {{ $fullname }} + updatePolicy: + updateMode: "InPlaceOrRecreate" + resourcePolicy: + containerPolicies: + {{- if $additionalNodeVPA }} + {{- $additionalNodeVPA | toYaml | nindent 4 }} + {{- end }} + - containerName: "node-driver-registrar" + minAllowed: + {{- include "node_driver_registrar_resources" $context | nindent 8 }} + maxAllowed: + cpu: 25m + memory: 50Mi + - containerName: "node" + minAllowed: + {{- include "node_resources" $context | nindent 8 }} + maxAllowed: + cpu: 25m + memory: 50Mi + {{- end }} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" "csi-node")) | nindent 2 }} +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: {{ $fullname }} + template: + metadata: + labels: + app: {{ $fullname }} + {{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} + security.deckhouse.io/security-policy-exception: {{ $fullname }} + {{- end }} + {{- if or (hasPrefix "cloud-provider-" $context.Chart.Name) ($additionalCsiNodePodAnnotations) }} + annotations: + {{- if hasPrefix "cloud-provider-" $context.Chart.Name }} + cloud-config-checksum: {{ include (print $context.Template.BasePath "/cloud-controller-manager/secret.yaml") $context | sha256sum }} + {{- end }} + {{- if $additionalCsiNodePodAnnotations }} + {{- $additionalCsiNodePodAnnotations | toYaml | nindent 8 }} + {{- end }} + {{- end }} + spec: + {{- if $customNodeSelector }} + nodeSelector: + {{- $customNodeSelector | toYaml | nindent 8 }} + {{- else }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - operator: In + key: node.deckhouse.io/type + values: + - CloudEphemeral + - CloudPermanent + - CloudStatic + {{- if $forceCsiNodeAndStaticNodesDepoloy }} + - Static + {{- end }} + {{- if $additionalNodeSelectorTerms }} + {{- $additionalNodeSelectorTerms | toYaml | nindent 14 }} + {{- end }} + {{- end }} + imagePullSecrets: + - name: deckhouse-registry + {{- if $additionalPullSecrets }} + {{- $additionalPullSecrets | toYaml | nindent 6 }} + {{- end }} + {{- include "helm_lib_priority_class" (tuple $context "system-node-critical") | nindent 6 }} + {{- include "helm_lib_tolerations" (tuple $context "any-node" "with-no-csi" "with-uninitialized") | nindent 6 }} + {{- include "helm_lib_module_pod_security_context_run_as_user_root" . | nindent 6 }} + hostNetwork: {{ $csiNodeHostNetwork }} + hostPID: {{ $csiNodeHostPID }} + {{- if eq $csiNodeHostNetwork "true" }} + dnsPolicy: {{ $dnsPolicy | quote }} + {{- end }} + containers: + - name: node-driver-registrar + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true "uid" "0" "runAsNonRoot" false) | nindent 8 }} + image: {{ $driverRegistrarImage | quote }} + args: + - "--v=5" + - "--csi-address=$(CSI_ENDPOINT)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + {{- if $livenessProbePort }} + - "--http-endpoint=:{{ $livenessProbePort }}" + {{- end }} + env: + - name: CSI_ENDPOINT + value: "/csi/csi.sock" + - name: DRIVER_REG_SOCK_PATH + value: "/var/lib/kubelet/csi-plugins/{{ $driverFQDN }}/csi.sock" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if $csiNodeDriverRegistrarLifecycle }} + lifecycle: + {{- $csiNodeDriverRegistrarLifecycle | toYaml | nindent 10 }} + {{- end }} + {{- if $additionalNodeLivenessProbesCmd }} + livenessProbe: + initialDelaySeconds: 3 + exec: + command: + {{- $additionalNodeLivenessProbesCmd | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" 10 | nindent 12 }} + {{- if not ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "node_driver_registrar_resources" $context | nindent 12 }} + {{- end }} + - name: node + securityContext: + allowPrivilegeEscalation: true + privileged: true + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL + {{- if $setSysAdminCapability }} + add: + - SYS_ADMIN + {{- end }} + image: {{ $nodeImage }} + args: + {{- if $additionalNodeArgs }} + {{- $additionalNodeArgs | toYaml | nindent 8 }} + {{- end }} + {{- if $additionalNodeEnvs }} + env: + {{- $additionalNodeEnvs | toYaml | nindent 8 }} + {{- end }} + {{- if $csiNodeLifecycle }} + lifecycle: + {{- $csiNodeLifecycle | toYaml | nindent 10 }} + {{- end }} + {{- if $livenessProbePort }} + livenessProbe: + httpGet: + path: /healthz + port: {{ $livenessProbePort }} + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + {{- if $additionalNodeVolumeMounts }} + {{- $additionalNodeVolumeMounts | toYaml | nindent 8 }} + {{- end }} + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- if not ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "node_resources" $context | nindent 12 }} + {{- end }} + + {{- if $additionalContainers }} + {{- $additionalContainers | toYaml | nindent 6 }} + {{- end }} + + {{- if $initContainers }} + initContainers: + {{- range $initContainer := $initContainers }} + - resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} + {{- $initContainer | toYaml | nindent 8 }} + {{- end }} + {{- end }} + + serviceAccount: {{ $serviceAccount | quote }} + serviceAccountName: {{ $serviceAccount | quote }} + automountServiceAccountToken: true + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/csi-plugins/{{ $driverFQDN }}/ + type: DirectoryOrCreate + - name: device-dir + hostPath: + path: /dev + type: Directory + + {{- if $additionalNodeVolumes }} + {{- $additionalNodeVolumes | toYaml | nindent 6 }} + {{- end }} + +{{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} +--- +apiVersion: deckhouse.io/v1alpha1 +kind: SecurityPolicyException +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} +spec: + securityContext: + privileged: + allowedValue: true + metadata: + description: | + Allow privileged mode for CSI Node Driver. + The CSI Node Driver requires privileged access to perform critical storage operations such as mounting/unmounting volumes, formatting block devices, and interacting directly with the host kernel for disk management. + runAsNonRoot: + allowedValue: false + metadata: + description: | + Allow running as root for CSI Node Driver. + The CSI Node Driver requires root access to perform privileged storage operations on the host, including device management and filesystem mounting. + allowPrivilegeEscalation: + allowedValue: true + metadata: + description: | + Allow privilege escalation for CSI Node Driver. + The node plugin may need to escalate privileges during mount, unmount, and device operations that rely on setuid helpers or additional capabilities beyond the container's initial security context. + runAsUser: + allowedValues: + - 0 + metadata: + description: | + Allow running as root user (UID 0) for CSI Node Driver. + The CSI Node Driver and node-driver-registrar require root privileges to perform storage operations, interact with host devices, and manage volume mounts. + {{- if $setSysAdminCapability }} + capabilities: + allowedValues: + add: + - SYS_ADMIN + metadata: + description: | + Allow SYS_ADMIN capability for CSI Node Driver. + The CSI Node Driver requires SYS_ADMIN capability to perform privileged filesystem operations such as mounting, unmounting, and managing block devices on the host. + {{- end }} + + {{- if or (eq $csiNodeHostNetwork "true") (ne $csiNodeHostPID "false") }} + network: + {{- if eq $csiNodeHostNetwork "true" }} + hostNetwork: + allowedValue: true + metadata: + description: | + Allow host network access for CSI Node Driver. + The CSI Node Driver requires host network access to communicate with the CSI Controller, coordinate volume attachment operations, and synchronize storage metadata across the cluster. + {{- end }} + {{- if ne $csiNodeHostPID "false" }} + hostPID: + allowedValue: true + metadata: + description: | + Allow host PID namespace access for CSI Node Driver. + The CSI Node Driver requires host PID namespace access to interact with host processes for storage operations such as detecting mount points and managing device attachments. + {{- end }} + {{- end }} + + volumes: + types: + allowedValues: + - hostPath + metadata: + description: | + Allow hostPath volume type for CSI Node. + The CSI Node Driver requires hostPath volumes to access host filesystem paths for proper operation, including communication with the container runtime and access to block devices. + hostPath: + allowedValues: + - path: /var/lib/kubelet/plugins_registry/ + readOnly: false + metadata: + description: | + Allow access to the CSI plugin registry directory. + CSI Node Driver registers itself in this directory to enable dynamic discovery and communication with the kubelet. + - path: /var/lib/kubelet + readOnly: false + metadata: + description: | + Allow access to the kubelet root directory. + Required for CSI Node Driver to manage volume mounts and access kubelet data structures. + - path: /var/lib/kubelet/csi-plugins/{{ $driverFQDN }}/ + readOnly: false + metadata: + description: | + Allow access to the CSI plugin directory. + This directory contains the CSI driver socket and persistent data required for volume attachment and mounting operations. + - path: /dev + readOnly: false + metadata: + description: | + Allow access to host device directory. + CSI Node Driver requires access to /dev to manage block devices and perform disk operations for persistent volumes. + {{- if $additionalNodeVolumes }} + {{- range $volume := $additionalNodeVolumes }} + {{- if $volume.hostPath }} + {{- $readOnly := false }} + {{- range $volumeMount := $additionalNodeVolumeMounts }} + {{- if eq $volumeMount.name $volume.name }} + {{- $readOnly = (default false $volumeMount.readOnly) }} + {{- end }} + {{- end }} + - path: {{ $volume.hostPath.path }} + readOnly: {{ $readOnly }} + metadata: + description: | + Allow access to additional hostPath volume at {{ $volume.hostPath.path }}. + This additional hostPath volume is required by the CSI Node Driver for extended storage operations specific to the cloud provider implementation. + {{- end }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/helm_lib/templates/_dns_policy.tpl b/charts/helm_lib/templates/_dns_policy.tpl new file mode 100644 index 000000000..31d3e73ec --- /dev/null +++ b/charts/helm_lib/templates/_dns_policy.tpl @@ -0,0 +1,12 @@ +{{- /* Usage: {{ include "helm_lib_dns_policy_bootstraping_state" (list . "Default" "ClusterFirstWithHostNet") }} */ -}} +{{- /* returns the proper dnsPolicy value depending on the cluster bootstrap phase */ -}} +{{- define "helm_lib_dns_policy_bootstraping_state" }} +{{- $context := index . 0 }} +{{- $valueDuringBootstrap := index . 1 }} +{{- $valueAfterBootstrap := index . 2 }} +{{- if $context.Values.global.clusterIsBootstrapped }} +{{- printf $valueAfterBootstrap }} +{{- else }} +{{- printf $valueDuringBootstrap }} +{{- end }} +{{- end }} diff --git a/charts/helm_lib/templates/_enable_ds_eviction.tpl b/charts/helm_lib/templates/_enable_ds_eviction.tpl new file mode 100644 index 000000000..b912c050d --- /dev/null +++ b/charts/helm_lib/templates/_enable_ds_eviction.tpl @@ -0,0 +1,6 @@ +{{- /* Usage: {{ include "helm_lib_prevent_ds_eviction_annotation" . }} */ -}} +{{- /* Adds `cluster-autoscaler.kubernetes.io/enable-ds-eviction` annotation to manage DaemonSet eviction by the Cluster Autoscaler. */ -}} +{{- /* This is important to prevent the eviction of DaemonSet pods during cluster scaling. */ -}} +{{- define "helm_lib_prevent_ds_eviction_annotation" -}} +cluster-autoscaler.kubernetes.io/enable-ds-eviction: "false" +{{- end }} diff --git a/charts/helm_lib/templates/_envs_for_proxy.tpl b/charts/helm_lib/templates/_envs_for_proxy.tpl new file mode 100644 index 000000000..398d7c43c --- /dev/null +++ b/charts/helm_lib/templates/_envs_for_proxy.tpl @@ -0,0 +1,43 @@ +{{- /* Usage: {{ include "helm_lib_envs_for_proxy" . }} or {{ include "helm_lib_envs_for_proxy" (list . (list "extra1" "extra2")) }} */ -}} +{{- /* Add HTTP_PROXY, HTTPS_PROXY and NO_PROXY environment variables for container */ -}} +{{- /* depends on [proxy settings](https://deckhouse.io/products/kubernetes-platform/documentation/v1/reference/api/global.html#parameters-modules-proxy) */ -}} +{{- define "helm_lib_envs_for_proxy" }} + {{- /* Template context with .Values, .Chart, etc */ -}} + {{- /* List of additional NO_PROXY entries (optional) */ -}} + {{- $context := . -}} + {{- $extraNoProxy := list -}} + + {{- /* If a list is passed, then the first element is the context, and the second is the extraNoProxy list. */ -}} + {{- if kindIs "slice" . }} + {{- $context = index . 0 -}} + {{- $extraNoProxy = index . 1 -}} + {{- end }} + + {{- if $context.Values.global.clusterConfiguration }} + {{- if $context.Values.global.clusterConfiguration.proxy }} + {{- if $context.Values.global.clusterConfiguration.proxy.httpProxy }} +- name: HTTP_PROXY + value: {{ $context.Values.global.clusterConfiguration.proxy.httpProxy | quote }} +- name: http_proxy + value: {{ $context.Values.global.clusterConfiguration.proxy.httpProxy | quote }} + {{- end }} + {{- if $context.Values.global.clusterConfiguration.proxy.httpsProxy }} +- name: HTTPS_PROXY + value: {{ $context.Values.global.clusterConfiguration.proxy.httpsProxy | quote }} +- name: https_proxy + value: {{ $context.Values.global.clusterConfiguration.proxy.httpsProxy | quote }} + {{- end }} + {{- $noProxy := list "127.0.0.1" "169.254.169.254" "registry.d8-system.svc" $context.Values.global.clusterConfiguration.clusterDomain $context.Values.global.clusterConfiguration.podSubnetCIDR $context.Values.global.clusterConfiguration.serviceSubnetCIDR }} + {{- if $context.Values.global.clusterConfiguration.proxy.noProxy }} + {{- $noProxy = concat $noProxy $context.Values.global.clusterConfiguration.proxy.noProxy }} + {{- end }} + {{- if $extraNoProxy }} + {{- $noProxy = concat $noProxy $extraNoProxy }} + {{- end }} +- name: NO_PROXY + value: {{ $noProxy | join "," | quote }} +- name: no_proxy + value: {{ $noProxy | join "," | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/helm_lib/templates/_high_availability.tpl b/charts/helm_lib/templates/_high_availability.tpl new file mode 100644 index 000000000..8c7da23ac --- /dev/null +++ b/charts/helm_lib/templates/_high_availability.tpl @@ -0,0 +1,39 @@ +{{- /* Usage: {{ include "helm_lib_is_ha_to_value" (list . yes no) }} */ -}} +{{- /* returns value "yes" if cluster is highly available, else — returns "no" */ -}} +{{- define "helm_lib_is_ha_to_value" }} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $yes := index . 1 -}} {{- /* Yes value */ -}} + {{- $no := index . 2 -}} {{- /* No value */ -}} + + {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) }} + + {{- if hasKey $module_values "highAvailability" -}} + {{- if $module_values.highAvailability -}} {{- $yes -}} {{- else -}} {{- $no -}} {{- end -}} + {{- else if hasKey $context.Values.global "highAvailability" -}} + {{- if $context.Values.global.highAvailability -}} {{- $yes -}} {{- else -}} {{- $no -}} {{- end -}} + {{- else -}} + {{- if $context.Values.global.discovery.clusterControlPlaneIsHighlyAvailable -}} {{- $yes -}} {{- else -}} {{- $no -}} {{- end -}} + {{- end -}} +{{- end }} + +{{- /* Usage: {{- if (include "helm_lib_ha_enabled" .) }} */ -}} +{{- /* returns empty value, which is treated by go template as false */ -}} +{{- define "helm_lib_ha_enabled" }} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + + {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) }} + + {{- if hasKey $module_values "highAvailability" -}} + {{- if $module_values.highAvailability -}} + "not empty string" + {{- end -}} + {{- else if hasKey $context.Values.global "highAvailability" -}} + {{- if $context.Values.global.highAvailability -}} + "not empty string" + {{- end -}} + {{- else -}} + {{- if $context.Values.global.discovery.clusterControlPlaneIsHighlyAvailable -}} + "not empty string" + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/helm_lib/templates/_kube_rbac_proxy.tpl b/charts/helm_lib/templates/_kube_rbac_proxy.tpl new file mode 100644 index 000000000..af9f7a447 --- /dev/null +++ b/charts/helm_lib/templates/_kube_rbac_proxy.tpl @@ -0,0 +1,21 @@ +{{- /* Usage: {{ include "helm_lib_kube_rbac_proxy_ca_certificate" (list . "namespace") }} */ -}} +{{- /* Renders configmap with kube-rbac-proxy CA certificate which uses to verify the kube-rbac-proxy clients. */ -}} +{{- define "helm_lib_kube_rbac_proxy_ca_certificate" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +{{- /* Namespace where CA configmap will be created */ -}} + {{- $context := index . 0 }} + {{- $namespace := index . 1 }} +--- +apiVersion: v1 +data: + ca.crt: | + {{ $context.Values.global.internal.modules.kubeRBACProxyCA.cert | nindent 4 }} +kind: ConfigMap +metadata: + annotations: + kubernetes.io/description: | + Contains a CA bundle that can be used to verify the kube-rbac-proxy clients. + {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} + name: kube-rbac-proxy-ca.crt + namespace: {{ $namespace }} +{{- end }} diff --git a/charts/helm_lib/templates/_module_controller.tpl b/charts/helm_lib/templates/_module_controller.tpl new file mode 100644 index 000000000..3e855392e --- /dev/null +++ b/charts/helm_lib/templates/_module_controller.tpl @@ -0,0 +1,532 @@ +{{- /* Usage: {{ include "helm_lib_module_controller_resources" . }} */ -}} +{{- /* Returns default controller resources */ -}} +{{- define "helm_lib_module_controller_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_webhooks_resources" . }} */ -}} +{{- /* Returns default webhooks resources */ -}} +{{- define "helm_lib_module_webhooks_resources" }} +cpu: 10m +memory: 50Mi +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_controller_manifests" (list . $config) }} */ -}} +{{- /* + Generates controller Deployment, VPA and PDB manifests. + + $config parameters: + - fullname: name for the deployment (default: "controller") + - valuesKey: key to access module values (e.g., "csiHpe", "csiS3") + - webhookEnabled: whether to include webhook container (default: false) + - webhookCertPath: path to webhook cert in values (e.g., "internal.customWebhookCert") + - onMasterNode: use master node selector and tolerations (default: true) + - priorityClass: priority class name (default: "system-cluster-critical") + - podSecurityContext: "deckhouse" or "nobody" (default: "nobody") + - additionalLabels: additional labels for VPA + - controllerMaxCpu: max CPU for controller VPA (default: "200m") + - controllerMaxMemory: max memory for controller VPA (default: "100Mi") + - webhooksMaxCpu: max CPU for webhooks VPA (default: "20m") + - webhooksMaxMemory: max memory for webhooks VPA (default: "100Mi") + - additionalContainers: additional containers to add to the pod + - additionalVolumes: additional volumes to add to the pod + - additionalControllerVolumeMounts: additional volume mounts for controller + - additionalControllerEnvs: additional environment variables for controller + - controllerImage: custom controller image (default: uses helm_lib_module_image) + - controllerImageName: image name for helm_lib_module_image (default: "controller") + - webhooksImageName: image name for webhooks (default: "webhooks") + - webhooksPort: port for webhooks container (default: 8443) + - webhooksCertMountPath: mount path for webhook certs (default: "/etc/webhook/certs") + - webhooksCommand: command for webhooks container + - controllerPort: port for controller probes (default: 8081) + - controllerMetricsPort: port for controller metrics (optional, no port exposed if not set) +*/ -}} +{{- define "helm_lib_module_controller_manifests" }} + {{- $context := index . 0 }} + {{- $config := index . 1 }} + + {{- $fullname := $config.fullname | default "controller" }} + {{- $valuesKey := $config.valuesKey | required "$config.valuesKey is required" }} + {{- $webhookEnabled := dig "webhookEnabled" false $config }} + {{- $webhookCertPath := $config.webhookCertPath | default "internal.customWebhookCert" }} + {{- $onMasterNode := dig "onMasterNode" true $config }} + {{- $priorityClass := $config.priorityClass | default "system-cluster-critical" }} + {{- $podSecurityContext := $config.podSecurityContext | default "nobody" }} + {{- $additionalLabels := $config.additionalLabels | default dict }} + {{- $controllerMaxCpu := $config.controllerMaxCpu | default "200m" }} + {{- $controllerMaxMemory := $config.controllerMaxMemory | default "100Mi" }} + {{- $webhooksMaxCpu := $config.webhooksMaxCpu | default "20m" }} + {{- $webhooksMaxMemory := $config.webhooksMaxMemory | default "100Mi" }} + {{- $additionalContainers := $config.additionalContainers }} + {{- $additionalVolumes := $config.additionalVolumes }} + {{- $additionalControllerVolumeMounts := $config.additionalControllerVolumeMounts }} + {{- $additionalControllerEnvs := $config.additionalControllerEnvs }} + {{- $controllerImage := $config.controllerImage }} + {{- $controllerImageName := $config.controllerImageName | default "controller" }} + {{- $webhooksImageName := $config.webhooksImageName | default "webhooks" }} + {{- $webhooksPort := $config.webhooksPort | default 8443 }} + {{- $webhooksCertMountPath := $config.webhooksCertMountPath | default "/etc/webhook/certs" }} + {{- $webhooksCommand := $config.webhooksCommand }} + {{- $controllerPort := $config.controllerPort | default 8081 }} + {{- $controllerMetricsPort := $config.controllerMetricsPort }} + + {{- /* Get module values */ -}} + {{- $moduleValues := index $context.Values $valuesKey }} + + {{- /* Build VPA labels */ -}} + {{- $vpaLabels := dict "app" $fullname }} + {{- if $onMasterNode }} + {{- $vpaLabels = merge $vpaLabels (dict "workload-resource-policy.deckhouse.io" "master") }} + {{- end }} + {{- $vpaLabels = merge $vpaLabels $additionalLabels }} + + {{- if ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context $vpaLabels) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: {{ $fullname }} + updatePolicy: + updateMode: "Initial" + resourcePolicy: + containerPolicies: + - containerName: "controller" + minAllowed: + {{- include "helm_lib_module_controller_resources" $context | nindent 8 }} + maxAllowed: + cpu: {{ $controllerMaxCpu }} + memory: {{ $controllerMaxMemory }} + {{- if $webhookEnabled }} + - containerName: "webhooks" + minAllowed: + {{- include "helm_lib_module_webhooks_resources" $context | nindent 8 }} + maxAllowed: + cpu: {{ $webhooksMaxCpu }} + memory: {{ $webhooksMaxMemory }} + {{- end }} + {{- end }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +spec: + minAvailable: {{ include "helm_lib_is_ha_to_value" (list $context 1 0) }} + selector: + matchLabels: + app: {{ $fullname }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +spec: + revisionHistoryLimit: 2 + {{- if $onMasterNode }} + {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" $context | nindent 2 }} + {{- else }} + {{- include "helm_lib_deployment_strategy_and_replicas_for_ha" $context | nindent 2 }} + {{- end }} + selector: + matchLabels: + app: {{ $fullname }} + template: + metadata: + {{- if $webhookEnabled }} + {{- $certPath := printf "%s.ca" $webhookCertPath }} + {{- $certValue := $moduleValues }} + {{- range $part := (split "." $certPath) }} + {{- $certValue = index $certValue $part }} + {{- end }} + annotations: + checksum/ca: {{ $certValue | sha256sum | quote }} + {{- end }} + labels: + app: {{ $fullname }} + spec: + {{- include "helm_lib_priority_class" (tuple $context $priorityClass) | nindent 6 }} + {{- if $onMasterNode }} + {{- include "helm_lib_tolerations" (tuple $context "any-node" "with-uninitialized" "with-cloud-provider-uninitialized") | nindent 6 }} + {{- include "helm_lib_node_selector" (tuple $context "master") | nindent 6 }} + {{- else }} + {{- include "helm_lib_node_selector" (tuple $context "system") | nindent 6 }} + {{- include "helm_lib_tolerations" (tuple $context "system") | nindent 6 }} + {{- end }} + {{- include "helm_lib_pod_anti_affinity_for_ha" (list $context (dict "app" $fullname)) | nindent 6 }} + {{- if eq $podSecurityContext "deckhouse" }} + {{- include "helm_lib_module_pod_security_context_run_as_user_deckhouse" $context | nindent 6 }} + {{- else }} + {{- include "helm_lib_module_pod_security_context_run_as_user_nobody" $context | nindent 6 }} + {{- end }} + imagePullSecrets: + - name: {{ $context.Chart.Name }}-module-registry + serviceAccountName: {{ $fullname }} + containers: + - name: controller + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 10 }} + {{- if $controllerImage }} + image: {{ $controllerImage | quote }} + {{- else }} + image: {{ include "helm_lib_module_image" (list $context $controllerImageName) }} + {{- end }} + imagePullPolicy: IfNotPresent + readinessProbe: + httpGet: + path: /readyz + port: {{ $controllerPort }} + scheme: HTTP + initialDelaySeconds: 5 + failureThreshold: 2 + periodSeconds: 1 + livenessProbe: + httpGet: + path: /healthz + port: {{ $controllerPort }} + scheme: HTTP + periodSeconds: 1 + failureThreshold: 3 + {{- if $controllerMetricsPort }} + ports: + - name: metrics + containerPort: {{ $controllerMetricsPort }} + protocol: TCP + {{- end }} + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 14 }} +{{- if not ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "helm_lib_module_controller_resources" $context | nindent 14 }} +{{- end }} + env: + - name: LOG_LEVEL + value: {{ include "helm_lib_module_controller_log_level" (list $context $valuesKey) | quote }} + - name: CONTROLLER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if $additionalControllerEnvs }} + {{- $additionalControllerEnvs | toYaml | nindent 12 }} + {{- end }} + {{- if $additionalControllerVolumeMounts }} + volumeMounts: + {{- $additionalControllerVolumeMounts | toYaml | nindent 12 }} + {{- end }} + {{- if $webhookEnabled }} + - name: webhooks + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 10 }} + {{- if $webhooksCommand }} + command: + {{- $webhooksCommand | toYaml | nindent 12 }} + {{- else }} + command: + - /webhooks + - -tls-cert-file={{ $webhooksCertMountPath }}/tls.crt + - -tls-key-file={{ $webhooksCertMountPath }}/tls.key + {{- end }} + image: {{ include "helm_lib_module_image" (list $context $webhooksImageName) }} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: webhook-certs + mountPath: {{ $webhooksCertMountPath }} + readOnly: true + readinessProbe: + httpGet: + path: /healthz + port: {{ $webhooksPort }} + scheme: HTTPS + initialDelaySeconds: 5 + failureThreshold: 2 + periodSeconds: 1 + livenessProbe: + httpGet: + path: /healthz + port: {{ $webhooksPort }} + scheme: HTTPS + periodSeconds: 1 + failureThreshold: 3 + ports: + - name: https + containerPort: {{ $webhooksPort }} + protocol: TCP + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 14 }} +{{- if not ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "helm_lib_module_webhooks_resources" $context | nindent 14 }} +{{- end }} + {{- end }} + {{- if $additionalContainers }} + {{- $additionalContainers | toYaml | nindent 8 }} + {{- end }} + {{- if or $webhookEnabled $additionalVolumes }} + volumes: + {{- if $webhookEnabled }} + - name: webhook-certs + secret: + secretName: webhooks-https-certs + {{- end }} + {{- if $additionalVolumes }} + {{- $additionalVolumes | toYaml | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_module_controller_log_level" (list . "csiHpe") }} */ -}} +{{- /* Returns numeric log level from module values */ -}} +{{- define "helm_lib_module_controller_log_level" }} + {{- $context := index . 0 }} + {{- $valuesKey := index . 1 }} + {{- $moduleValues := index $context.Values $valuesKey }} + {{- $logLevel := $moduleValues.logLevel | default "INFO" }} + {{- if eq $logLevel "ERROR" -}} +0 + {{- else if eq $logLevel "WARN" -}} +1 + {{- else if eq $logLevel "INFO" -}} +2 + {{- else if eq $logLevel "DEBUG" -}} +3 + {{- else if eq $logLevel "TRACE" -}} +4 + {{- else -}} +2 + {{- end -}} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_module_webhook_service" (list . $config) }} */ -}} +{{- /* + Generates webhook Service manifest. + + $config parameters: + - fullname: name of the service (default: "webhooks") + - selectorApp: app label for selector (default: "controller") + - targetPort: target port name or number (default: "https") + - additionalPorts: list of additional ports (optional), each port is a dict with: + - name: port name (required) + - port: service port (required) + - targetPort: target port name or number (required) + - protocol: protocol (default: "TCP") +*/ -}} +{{- define "helm_lib_module_webhook_service" }} + {{- $context := index . 0 }} + {{- $config := index . 1 | default dict }} + + {{- $fullname := $config.fullname | default "webhooks" }} + {{- $selectorApp := $config.selectorApp | default "controller" }} + {{- $targetPort := $config.targetPort | default "https" }} + {{- $additionalPorts := $config.additionalPorts }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $selectorApp)) | nindent 2 }} +spec: + type: ClusterIP + ports: + - port: 443 + targetPort: {{ $targetPort }} + protocol: TCP + name: https + {{- range $additionalPorts }} + - name: {{ .name }} + port: {{ .port }} + targetPort: {{ .targetPort }} + protocol: {{ .protocol | default "TCP" }} + {{- end }} + selector: + app: {{ $selectorApp }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_module_validating_webhook_configuration" (list . $config) }} */ -}} +{{- /* + Generates ValidatingWebhookConfiguration manifest. + + $config parameters: + - name: webhook configuration name (required, e.g., "sc-validation") + - webhookName: webhook name suffix (required, e.g., "sc-validation") + - valuesKey: key to access module values (required, e.g., "csiHpe") + - webhookCertPath: path to webhook cert in values (default: "internal.customWebhookCert") + - serviceName: service name (default: "webhooks") + - path: webhook path (required, e.g., "/sc-validate") + - rules: webhook rules (required) + - matchConditions: optional match conditions + - sideEffects: side effects (default: "None") + - timeoutSeconds: timeout (default: 5) +*/ -}} +{{- define "helm_lib_module_validating_webhook_configuration" }} + {{- $context := index . 0 }} + {{- $config := index . 1 }} + + {{- $name := $config.name | required "$config.name is required" }} + {{- $webhookName := $config.webhookName | required "$config.webhookName is required" }} + {{- $valuesKey := $config.valuesKey | required "$config.valuesKey is required" }} + {{- $webhookCertPath := $config.webhookCertPath | default "internal.customWebhookCert" }} + {{- $serviceName := $config.serviceName | default "webhooks" }} + {{- $path := $config.path | required "$config.path is required" }} + {{- $rules := $config.rules | required "$config.rules is required" }} + {{- $matchConditions := $config.matchConditions }} + {{- $sideEffects := $config.sideEffects | default "None" }} + {{- $timeoutSeconds := $config.timeoutSeconds | default 5 }} + + {{- /* Get module values and cert */ -}} + {{- $moduleValues := index $context.Values $valuesKey }} + {{- $certPath := printf "%s.ca" $webhookCertPath }} + {{- $certValue := $moduleValues }} + {{- range $part := (split "." $certPath) }} + {{- $certValue = index $certValue $part }} + {{- end }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: "d8-{{ $context.Chart.Name }}-{{ $name }}" + {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} +webhooks: + - name: "d8-{{ $context.Chart.Name }}-{{ $webhookName }}.deckhouse.io" + rules: + {{- $rules | toYaml | nindent 6 }} + clientConfig: + service: + namespace: "d8-{{ $context.Chart.Name }}" + name: {{ $serviceName | quote }} + path: {{ $path | quote }} + caBundle: {{ $certValue | b64enc | quote }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: {{ $sideEffects }} + timeoutSeconds: {{ $timeoutSeconds }} + {{- if $matchConditions }} + matchConditions: + {{- $matchConditions | toYaml | nindent 6 }} + {{- end }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_module_webhook_certs_secret" (list . $config) }} */ -}} +{{- /* + Generates webhook TLS certificates Secret manifest. + + $config parameters: + - fullname: name of the secret (default: "webhooks-https-certs") + - valuesKey: key to access module values (required, e.g., "csiHpe", "csiS3") + - webhookCertPath: path to webhook cert in values (default: "internal.customWebhookCert") + - appLabel: app label for the secret (default: "webhooks") +*/ -}} +{{- define "helm_lib_module_webhook_certs_secret" }} + {{- $context := index . 0 }} + {{- $config := index . 1 }} + + {{- $fullname := $config.fullname | default "webhooks-https-certs" }} + {{- $valuesKey := $config.valuesKey | required "$config.valuesKey is required" }} + {{- $webhookCertPath := $config.webhookCertPath | default "internal.customWebhookCert" }} + {{- $appLabel := $config.appLabel | default "webhooks" }} + + {{- /* Get module values and certs */ -}} + {{- $moduleValues := index $context.Values $valuesKey }} + {{- $certValue := $moduleValues }} + {{- range $part := (split "." $webhookCertPath) }} + {{- $certValue = index $certValue $part }} + {{- end }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $appLabel)) | nindent 2 }} +type: kubernetes.io/tls +data: + ca.crt: {{ $certValue.ca | b64enc | quote }} + tls.crt: {{ $certValue.crt | b64enc | quote }} + tls.key: {{ $certValue.key | b64enc | quote }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_module_controller_rbac" (list . $config) }} */ -}} +{{- /* + Generates controller RBAC manifests (ServiceAccount, Role, ClusterRole, RoleBinding, ClusterRoleBinding). + + $config parameters: + - fullname: name for the resources (default: "controller") + - roleRules: rules for namespaced Role (required) + - clusterRoleRules: rules for ClusterRole (required) +*/ -}} +{{- define "helm_lib_module_controller_rbac" -}} + {{- $context := index . 0 -}} + {{- $config := index . 1 -}} + {{- $fullname := $config.fullname | default "controller" -}} + {{- $roleRules := $config.roleRules | required "$config.roleRules is required" -}} + {{- $clusterRoleRules := $config.clusterRoleRules | required "$config.clusterRoleRules is required" }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +rules: + {{- $roleRules | toYaml | nindent 2 }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ $context.Chart.Name }}:{{ $fullname }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +rules: + {{- $clusterRoleRules | toYaml | nindent 2 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} +roleRef: + kind: Role + name: {{ $fullname }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: d8:{{ $context.Chart.Name }}:{{ $fullname }} + {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: {{ $fullname }} + namespace: d8-{{ $context.Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ $context.Chart.Name }}:{{ $fullname }} + apiGroup: rbac.authorization.k8s.io +{{- end }} + + + diff --git a/charts/helm_lib/templates/_module_documentation_uri.tpl b/charts/helm_lib/templates/_module_documentation_uri.tpl new file mode 100644 index 000000000..c1b94812d --- /dev/null +++ b/charts/helm_lib/templates/_module_documentation_uri.tpl @@ -0,0 +1,15 @@ +{{- /* Usage: {{ include "helm_lib_module_documentation_uri" (list . "") }} */ -}} +{{- /* returns rendered documentation uri using publicDomainTemplate or deckhouse.io domains*/ -}} +{{- define "helm_lib_module_documentation_uri" }} + {{- $default_doc_prefix := "https://deckhouse.io" -}} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $path_portion := index . 1 -}} {{- /* Path to the document */ -}} + {{- $uri := "" -}} + {{- if $context.Values.global.modules.publicDomainTemplate }} + {{- $uri = printf "%s://%s%s" (include "helm_lib_module_uri_scheme" $context) (include "helm_lib_module_public_domain" (list $context "documentation")) $path_portion -}} + {{- else }} + {{- $uri = printf "%s%s" $default_doc_prefix $path_portion -}} + {{- end -}} + + {{ $uri }} +{{- end }} diff --git a/charts/helm_lib/templates/_module_ephemeral_storage.tpl b/charts/helm_lib/templates/_module_ephemeral_storage.tpl new file mode 100644 index 000000000..4b2dd02ed --- /dev/null +++ b/charts/helm_lib/templates/_module_ephemeral_storage.tpl @@ -0,0 +1,15 @@ +{{- /* Usage: {{ include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 }} */ -}} +{{- /* 50Mi for container logs `log-opts.max-file * log-opts.max-size` would be added to passed value */ -}} +{{- /* returns ephemeral-storage size for logs with extra space */ -}} +{{- define "helm_lib_module_ephemeral_storage_logs_with_extra" -}} +{{- /* Extra space in mebibytes */ -}} +ephemeral-storage: {{ add . 50 }}Mi +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_ephemeral_storage_only_logs" . }} */ -}} +{{- /* 50Mi for container logs `log-opts.max-file * log-opts.max-size` would be requested */ -}} +{{- /* returns ephemeral-storage size for only logs */ -}} +{{- define "helm_lib_module_ephemeral_storage_only_logs" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +ephemeral-storage: 50Mi +{{- end }} diff --git a/charts/helm_lib/templates/_module_gateway.tpl b/charts/helm_lib/templates/_module_gateway.tpl new file mode 100644 index 000000000..2ad4e63f1 --- /dev/null +++ b/charts/helm_lib/templates/_module_gateway.tpl @@ -0,0 +1,22 @@ +{{- /* Usage: {{- include "helm_lib_module_gateway" (list . $gateway) */ -}} +{{- /* accepts a dict that is updated with current gateway name and namespace */ -}} +{{- define "helm_lib_module_gateway" -}} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $result := index . 1 -}} {{- /* An empty dict to update with current default gateway name and namespace */ -}} + {{- $g := dict -}} + + {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} + + {{- if hasKey $module_values "gatewayAPIGateway" -}} + {{- $g = $module_values.gatewayAPIGateway -}} + {{- else if hasKey $context.Values.global.modules "gatewayAPIGateway" -}} + {{- $g = $context.Values.global.modules.gatewayAPIGateway -}} + {{- else if and (hasKey $context.Values.global "discovery") (hasKey $context.Values.global.discovery "gatewayAPIDefaultGateway") -}} + {{- $g = $context.Values.global.discovery.gatewayAPIDefaultGateway -}} + {{- end -}} + + {{- if and $g.name $g.namespace -}} + {{- $_ := set $result "name" $g.name -}} + {{- $_ := set $result "namespace" $g.namespace -}} + {{- end -}} +{{- end -}} diff --git a/charts/helm_lib/templates/_module_generate_common_name.tpl b/charts/helm_lib/templates/_module_generate_common_name.tpl new file mode 100644 index 000000000..fb142f824 --- /dev/null +++ b/charts/helm_lib/templates/_module_generate_common_name.tpl @@ -0,0 +1,13 @@ +{{- /* Usage: {{ include "helm_lib_module_generate_common_name" (list . "") }} */ -}} +{{- /* returns the commonName parameter for use in the Certificate custom resource(cert-manager) */ -}} +{{- define "helm_lib_module_generate_common_name" }} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $name_portion := index . 1 -}} {{- /* Name portion */ -}} + + {{- $domain := include "helm_lib_module_public_domain" (list $context $name_portion) -}} + + {{- $domain_length := len $domain -}} + {{- if le $domain_length 64 -}} +commonName: {{ $domain }} + {{- end -}} +{{- end }} diff --git a/charts/helm_lib/templates/_module_https.tpl b/charts/helm_lib/templates/_module_https.tpl new file mode 100644 index 000000000..24c5d3735 --- /dev/null +++ b/charts/helm_lib/templates/_module_https.tpl @@ -0,0 +1,201 @@ +{{- /* Usage: {{ include "helm_lib_module_uri_scheme" . }} */ -}} +{{- /* return module uri scheme "http" or "https" */ -}} +{{- define "helm_lib_module_uri_scheme" -}} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $mode := "" -}} + + {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} + {{- if hasKey $module_values "https" -}} + {{- if hasKey $module_values.https "mode" -}} + {{- $mode = $module_values.https.mode -}} + {{- else }} + {{- $mode = $context.Values.global.modules.https.mode | default "" -}} + {{- end }} + {{- else }} + {{- $mode = $context.Values.global.modules.https.mode | default "" -}} + {{- end }} + + + {{- if eq "Disabled" $mode -}} + http + {{- else -}} + https + {{- end -}} +{{- end -}} + +{{- /* Usage: {{ $https_values := include "helm_lib_https_values" . | fromYaml }} */ -}} +{{- define "helm_lib_https_values" -}} + {{- $context := . -}} + {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} + {{- $mode := "" -}} + {{- $certManagerClusterIssuerName := "" -}} + + {{- if hasKey $module_values "https" -}} + {{- if hasKey $module_values.https "mode" -}} + {{- $mode = $module_values.https.mode -}} + {{- if eq $mode "CertManager" -}} + {{- if not (hasKey $module_values.https "certManager") -}} + {{- cat ".https.certManager.clusterIssuerName is mandatory when .https.mode is set to CertManager" | fail -}} + {{- end -}} + {{- if hasKey $module_values.https.certManager "clusterIssuerName" -}} + {{- $certManagerClusterIssuerName = $module_values.https.certManager.clusterIssuerName -}} + {{- else -}} + {{- cat ".https.certManager.clusterIssuerName is mandatory when .https.mode is set to CertManager" | fail -}} + {{- end -}} + {{- end -}} + {{- else -}} + {{- cat ".https.mode is mandatory when .https is defined" | fail -}} + {{- end -}} + {{- end -}} + + {{- if empty $mode -}} + {{- $mode = $context.Values.global.modules.https.mode -}} + {{- if eq $mode "CertManager" -}} + {{- $certManagerClusterIssuerName = $context.Values.global.modules.https.certManager.clusterIssuerName -}} + {{- end -}} + {{- end -}} + + {{- if not (has $mode (list "Disabled" "CertManager" "CustomCertificate" "OnlyInURI")) -}} + {{- cat "Unknown https.mode:" $mode | fail -}} + {{- end -}} + + {{- if and (eq $mode "CertManager") (not ($context.Values.global.enabledModules | has "cert-manager")) -}} + {{- cat "https.mode has value CertManager but cert-manager module not enabled" | fail -}} + {{- end -}} + +mode: {{ $mode }} + {{- if eq $mode "CertManager" }} +certManager: + clusterIssuerName: {{ $certManagerClusterIssuerName }} + {{- end -}} + +{{- end -}} + +{{- /* Usage: {{ if (include "helm_lib_module_https_mode" .) }} */ -}} +{{- /* returns https mode for module */ -}} +{{- define "helm_lib_module_https_mode" -}} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $https_values := include "helm_lib_https_values" $context | fromYaml -}} + {{- $https_values.mode -}} +{{- end -}} + +{{- /* Usage: {{ include "helm_lib_module_https_cert_manager_cluster_issuer_name" . }} */ -}} +{{- /* returns cluster issuer name */ -}} +{{- define "helm_lib_module_https_cert_manager_cluster_issuer_name" -}} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $https_values := include "helm_lib_https_values" $context | fromYaml -}} + {{- $https_values.certManager.clusterIssuerName -}} +{{- end -}} + +{{- /* Usage: {{ if (include "helm_lib_module_https_cert_manager_cluster_issuer_is_dns01_challenge_solver" .) }} */ -}} +{{- define "helm_lib_module_https_cert_manager_cluster_issuer_is_dns01_challenge_solver" -}} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- if has (include "helm_lib_module_https_cert_manager_cluster_issuer_name" $context) (list "route53" "cloudflare" "digitalocean" "clouddns") }} + "not empty string" + {{- end -}} +{{- end -}} + +{{- /* Usage: {{ include "helm_lib_module_https_cert_manager_acme_solver_challenge_settings" . | nindent 4 }} */ -}} +{{- define "helm_lib_module_https_cert_manager_acme_solver_challenge_settings" -}} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- if (include "helm_lib_module_https_cert_manager_cluster_issuer_is_dns01_challenge_solver" $context) }} +- dns01: + provider: {{ include "helm_lib_module_https_cert_manager_cluster_issuer_name" $context }} + {{- else }} +- http01: + ingressClass: {{ include "helm_lib_module_ingress_class" $context | quote }} + {{- end }} +{{- end -}} + +{{- /* Usage: {{ if (include "helm_lib_module_https_ingress_tls_enabled" .) }} */ -}} +{{- /* returns not empty string if tls should be enabled for the ingress */ -}} +{{- define "helm_lib_module_https_ingress_tls_enabled" -}} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + + {{- $mode := include "helm_lib_module_https_mode" $context -}} + + {{- if or (eq "CertManager" $mode) (eq "CustomCertificate" $mode) -}} + not empty string + {{- end -}} +{{- end -}} + +{{- /* Usage: {{ if (include "helm_lib_module_https_route_tls_enabled" .) }} */ -}} +{{- /* returns not empty string if tls should be enabled for the route */ -}} +{{- define "helm_lib_module_https_route_tls_enabled" -}} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + + {{- $mode := include "helm_lib_module_https_mode" $context -}} + + {{- if or (eq "CertManager" $mode) (eq "CustomCertificate" $mode) -}} + not empty string + {{- end -}} +{{- end -}} + +{{- /* Usage: {{ include "helm_lib_module_https_copy_custom_certificate" (list . "namespace" "secret_name_prefix") }} */ -}} +{{- /* Renders secret with [custom certificate](https://deckhouse.io/products/kubernetes-platform/documentation/v1/reference/api/global.html#parameters-modules-https-customcertificate) */ -}} +{{- /* in passed namespace with passed prefix */ -}} +{{- define "helm_lib_module_https_copy_custom_certificate" -}} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $namespace := index . 1 -}} {{- /* Namespace */ -}} + {{- $secret_name_prefix := index . 2 -}} {{- /* Secret name prefix */ -}} + {{- $mode := include "helm_lib_module_https_mode" $context -}} + {{- if eq $mode "CustomCertificate" -}} + {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} + {{- $customCertificateData := $module_values.internal.customCertificateData -}} + {{- if not $customCertificateData -}} + {{- fail (printf "internal.customCertificateData is required to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} + {{- end -}} + {{- if not (kindIs "map" $customCertificateData) -}} + {{- fail (printf "internal.customCertificateData must be a map to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} + {{- end -}} + {{- $tlsCrt := index $customCertificateData "tls.crt" -}} + {{- $tlsKey := index $customCertificateData "tls.key" -}} + {{- if not $tlsCrt -}} + {{- fail (printf "internal.customCertificateData.tls.crt is required to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} + {{- end -}} + {{- if not $tlsKey -}} + {{- fail (printf "internal.customCertificateData.tls.key is required to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} + {{- end -}} + {{- if not (kindIs "string" $tlsCrt) -}} + {{- fail (printf "internal.customCertificateData.tls.crt must be a string to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} + {{- end -}} + {{- if not (kindIs "string" $tlsKey) -}} + {{- fail (printf "internal.customCertificateData.tls.key must be a string to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} + {{- end -}} + {{- $secret_name := include "helm_lib_module_https_secret_name" (list $context $secret_name_prefix) -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secret_name }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} +type: kubernetes.io/tls +data: +{{- if (hasKey $customCertificateData "ca.crt") }} + {{- $caCrt := index $customCertificateData "ca.crt" -}} + {{- if and $caCrt (kindIs "string" $caCrt) }} + ca.crt: {{ $caCrt | b64enc }} + {{- end }} +{{- end }} + tls.crt: {{ $tlsCrt | b64enc }} + tls.key: {{ $tlsKey | b64enc }} + {{- end -}} +{{- end -}} + +{{- /* Usage: {{ include "helm_lib_module_https_secret_name (list . "secret_name_prefix") }} */ -}} +{{- /* returns custom certificate name */ -}} +{{- define "helm_lib_module_https_secret_name" -}} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $secret_name_prefix := index . 1 -}} {{- /* Secret name prefix */ -}} + {{- $mode := include "helm_lib_module_https_mode" $context -}} + {{- if eq $mode "CertManager" -}} + {{- $secret_name_prefix -}} + {{- else -}} + {{- if eq $mode "CustomCertificate" -}} + {{- printf "%s-customcertificate" $secret_name_prefix -}} + {{- else -}} + {{- fail "https.mode must be CustomCertificate or CertManager" -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/helm_lib/templates/_module_image.tpl b/charts/helm_lib/templates/_module_image.tpl new file mode 100644 index 000000000..74ab35499 --- /dev/null +++ b/charts/helm_lib/templates/_module_image.tpl @@ -0,0 +1,136 @@ +{{- /* Usage: {{ include "helm_lib_module_image" (list . "" "(optional)") }} */ -}} +{{- /* returns image name */ -}} +{{- define "helm_lib_module_image" }} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} + + {{- /* New approach: use module package values */}} + {{- if and $context.Module $context.Module.Package }} + {{- $registryBase := $context.Module.Package.Registry.repository }} + {{- if not $registryBase }} + {{- fail "Registry base is not set" }} + {{- end }} + + {{- $packageName := $context.Module.Package.Name }} + {{- if not $packageName }} + {{- fail "Package name is not set" }} + {{- end }} + + {{- $imageDigest := index $context.Module.Package.Digests $containerName }} + {{- if not $imageDigest }} + {{- fail (printf "Image %s has no digest" $containerName) }} + {{- end }} + + {{- printf "%s/%s@%s" $registryBase $packageName $imageDigest }} + + {{- /* Legacy fallback: use global modulesImages values */}} + {{- else }} + {{- $rawModuleName := $context.Chart.Name }} + {{- if ge (len .) 3 }} + {{- $rawModuleName = (index . 2) }} {{- /* Optional module name */ -}} + {{- end }} + {{- $moduleName := (include "helm_lib_module_camelcase_name" $rawModuleName) }} + + {{- $imageDigest := index $context.Values.global.modulesImages.digests $moduleName $containerName }} + {{- if not $imageDigest }} + {{- $error := (printf "Image %s.%s has no digest" $moduleName $containerName ) }} + {{- fail $error }} + {{- end }} + + {{- $registryBase := $context.Values.global.modulesImages.registry.base }} + {{- /* handle external modules registry */}} + {{- if index $context.Values $moduleName }} + {{- if index $context.Values $moduleName "registry" }} + {{- if index $context.Values $moduleName "registry" "base" }} + {{- $host := trimAll "/" (index $context.Values $moduleName "registry" "base") }} + {{- $path := trimAll "/" (include "helm_lib_module_kebabcase_name" $rawModuleName) }} + {{- $registryBase = join "/" (list $host $path) }} + {{- end }} + {{- end }} + {{- end }} + {{- /* end of external module handling block */}} + {{- printf "%s@%s" $registryBase $imageDigest }} + {{- end }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_image_no_fail" (list . "") }} */ -}} +{{- /* returns image name if found */ -}} +{{- define "helm_lib_module_image_no_fail" }} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} + {{- $moduleName := (include "helm_lib_module_camelcase_name" $context) }} + {{- if ge (len .) 3 }} + {{- $moduleName = (include "helm_lib_module_camelcase_name" (index . 2)) }} {{- /* Optional module name */ -}} + {{- end }} + {{- $imageDigest := index $context.Values.global.modulesImages.digests $moduleName $containerName }} + {{- if $imageDigest }} + {{- $registryBase := $context.Values.global.modulesImages.registry.base }} + {{- if index $context.Values $moduleName }} + {{- if index $context.Values $moduleName "registry" }} + {{- if index $context.Values $moduleName "registry" "base" }} + {{- $host := trimAll "/" (index $context.Values $moduleName "registry" "base") }} + {{- $path := trimAll "/" $context.Chart.Name }} + {{- $registryBase = join "/" (list $host $path) }} + {{- end }} + {{- end }} + {{- end }} + {{- printf "%s@%s" $registryBase $imageDigest }} + {{- end }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_common_image" (list . "") }} */ -}} +{{- /* returns image name from common module */ -}} +{{- define "helm_lib_module_common_image" }} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} + {{- $imageDigest := index $context.Values.global.modulesImages.digests "common" $containerName }} + {{- if not $imageDigest }} + {{- $error := (printf "Image %s.%s has no digest" "common" $containerName ) }} + {{- fail $error }} + {{- end }} + {{- printf "%s@%s" $context.Values.global.modulesImages.registry.base $imageDigest }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_common_image_no_fail" (list . "") }} */ -}} +{{- /* returns image name from common module if found */ -}} +{{- define "helm_lib_module_common_image_no_fail" }} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} + {{- $imageDigest := index $context.Values.global.modulesImages.digests "common" $containerName }} + {{- if $imageDigest }} + {{- printf "%s@%s" $context.Values.global.modulesImages.registry.base $imageDigest }} + {{- end }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_image_digest" (list . "" "(optional)") }} */ -}} +{{- /* returns image digest */ -}} +{{- define "helm_lib_module_image_digest" }} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} + {{- $rawModuleName := $context.Chart.Name }} + {{- if ge (len .) 3 }} + {{- $rawModuleName = (index . 2) }} {{- /* Optional module name */ -}} + {{- end }} + {{- $moduleName := (include "helm_lib_module_camelcase_name" $rawModuleName) }} + {{- $moduleMap := index $context.Values.global.modulesImages.digests $moduleName | default dict }} + {{- $imageDigest := index $moduleMap $containerName | default "" }} + {{- if not $imageDigest }} + {{- $error := (printf "Image %s.%s has no digest" $moduleName $containerName ) }} + {{- fail $error }} + {{- end }} + {{- printf "%s" $imageDigest }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_image_digest_no_fail" (list . "" "(optional)") }} */ -}} +{{- /* returns image digest if found */ -}} +{{- define "helm_lib_module_image_digest_no_fail" }} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} + {{- $moduleName := (include "helm_lib_module_camelcase_name" $context) }} + {{- if ge (len .) 3 }} + {{- $moduleName = (include "helm_lib_module_camelcase_name" (index . 2)) }} {{- /* Optional module name */ -}} + {{- end }} + {{- $moduleMap := index $context.Values.global.modulesImages.digests $moduleName | default dict }} + {{- $imageDigest := index $moduleMap $containerName | default "" }} + {{- printf "%s" $imageDigest }} +{{- end }} diff --git a/charts/helm_lib/templates/_module_ingress_class.tpl b/charts/helm_lib/templates/_module_ingress_class.tpl new file mode 100644 index 000000000..db7f50ba7 --- /dev/null +++ b/charts/helm_lib/templates/_module_ingress_class.tpl @@ -0,0 +1,13 @@ +{{- /* Usage: {{ include "helm_lib_module_ingress_class" . }} */ -}} +{{- /* returns ingress class from module settings or if not exists from global config */ -}} +{{- define "helm_lib_module_ingress_class" -}} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + + {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} + + {{- if hasKey $module_values "ingressClass" -}} + {{- $module_values.ingressClass -}} + {{- else if hasKey $context.Values.global.modules "ingressClass" -}} + {{- $context.Values.global.modules.ingressClass -}} + {{- end -}} +{{- end -}} diff --git a/charts/helm_lib/templates/_module_ingress_snippets.tpl b/charts/helm_lib/templates/_module_ingress_snippets.tpl new file mode 100644 index 000000000..b2ae8fa1a --- /dev/null +++ b/charts/helm_lib/templates/_module_ingress_snippets.tpl @@ -0,0 +1,11 @@ +{{- /* Usage: nginx.ingress.kubernetes.io/configuration-snippet: | {{ include "helm_lib_module_ingress_configuration_snippet" . | nindent 6 }} */ -}} +{{- /* returns nginx ingress additional headers (e.g. HSTS) if HTTPS is enabled */ -}} +{{- define "helm_lib_module_ingress_configuration_snippet" -}} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + + {{- $mode := include "helm_lib_module_https_mode" $context -}} + + {{- if or (eq "CertManager" $mode) (eq "CustomCertificate" $mode) -}} +add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + {{- end -}} +{{- end -}} diff --git a/charts/helm_lib/templates/_module_init_container.tpl b/charts/helm_lib/templates/_module_init_container.tpl new file mode 100644 index 000000000..199723c9e --- /dev/null +++ b/charts/helm_lib/templates/_module_init_container.tpl @@ -0,0 +1,101 @@ +{{- /* ### Migration 11.12.2020: Remove this helper with all its usages after this commit reached RockSolid */ -}} +{{- /* Usage: {{ include "helm_lib_module_init_container_chown_nobody_volume" (list . "volume-name") }} */ -}} +{{- /* returns initContainer which chowns recursively all files and directories in passed volume */ -}} +{{- define "helm_lib_module_init_container_chown_nobody_volume" }} + {{- $context := index . 0 -}} + {{- $volume_name := index . 1 -}} + {{- $image := "alpine" -}} + {{- if hasKey $context.Values.global.modulesImages.digests.common "init" -}} + {{- $image = "init" -}} + {{- end -}} +- name: chown-volume-{{ $volume_name }} + image: {{ include "helm_lib_module_common_image" (list $context $image) }} + command: ["sh", "-c", "chown -R 65534:65534 /tmp/data"] + securityContext: + runAsNonRoot: false + readOnlyRootFilesystem: true + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: {{ $volume_name }} + mountPath: /tmp/data + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 6 }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_init_container_chown_deckhouse_volume" (list . "volume-name") }} */ -}} +{{- /* returns initContainer which chowns recursively all files and directories in passed volume */ -}} +{{- define "helm_lib_module_init_container_chown_deckhouse_volume" }} + {{- $context := index . 0 -}} + {{- $volume_name := index . 1 -}} + {{- $image := "alpine" -}} + {{- if hasKey $context.Values.global.modulesImages.digests.common "init" -}} + {{- $image = "init" -}} + {{- end -}} +- name: chown-volume-{{ $volume_name }} + image: {{ include "helm_lib_module_common_image" (list $context $image) }} + command: ["sh", "-c", "chown -R 64535:64535 /tmp/data"] + securityContext: + runAsNonRoot: false + readOnlyRootFilesystem: true + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: {{ $volume_name }} + mountPath: /tmp/data + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 6 }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_init_container_check_linux_kernel" (list . ">= 4.9.17") }} */ -}} +{{- /* returns initContainer which checks the kernel version on the node for compliance to semver constraint */ -}} +{{- define "helm_lib_module_init_container_check_linux_kernel" }} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $semver_constraint := index . 1 -}} {{- /* Semver constraint */ -}} +- name: check-linux-kernel + image: {{ include "helm_lib_module_common_image" (list $context "checkKernelVersion") }} + {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true) | nindent 2 }} + env: + - name: KERNEL_CONSTRAINT + value: {{ $semver_constraint | quote }} + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 6 }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_init_container_iptables_wrapper" . }} */ -}} +{{- /* returns initContainer with iptables-wrapper */ -}} +{{- define "helm_lib_module_init_container_iptables_wrapper" -}} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} + - name: iptables-wrapper-init + {{- include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "NET_ADMIN" "NET_RAW")) | nindent 2 }} + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: + type: RuntimeDefault + image: {{ include "helm_lib_module_image" (list $context "iptablesWrapperInit") }} + command: + - /bin/bash + - -ec + - | + /usr/bin/cp /iptables-wrapper /sbin/ -rv + /usr/bin/cp /_sbin/* /sbin/ -rv + /usr/bin/cp /relocate/sbin/* /sbin/ -rv + /sbin/iptables --version + /usr/bin/rm /sbin/iptables-wrapper -v + volumeMounts: + - mountPath: /sbin + name: sbin + - name: xtables-lock + mountPath: /run/xtables.lock + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 6 }} + {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler") }} + cpu: 10m + memory: 10Mi + {{- end }} +{{- end }} diff --git a/charts/helm_lib/templates/_module_labels.tpl b/charts/helm_lib/templates/_module_labels.tpl new file mode 100644 index 000000000..228dcf3c4 --- /dev/null +++ b/charts/helm_lib/templates/_module_labels.tpl @@ -0,0 +1,15 @@ +{{- /* Usage: {{ include "helm_lib_module_labels" (list . (dict "app" "test" "component" "testing")) }} */ -}} +{{- /* returns deckhouse labels */ -}} +{{- define "helm_lib_module_labels" }} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- /* Additional labels dict */ -}} +labels: + heritage: deckhouse + module: {{ $context.Chart.Name }} + {{- if eq (len .) 2 }} + {{- $deckhouse_additional_labels := index . 1 }} + {{- range $key, $value := $deckhouse_additional_labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/helm_lib/templates/_module_name.tpl b/charts/helm_lib/templates/_module_name.tpl new file mode 100644 index 000000000..8323c177e --- /dev/null +++ b/charts/helm_lib/templates/_module_name.tpl @@ -0,0 +1,23 @@ +{{- define "helm_lib_module_camelcase_name" -}} + +{{- $moduleName := "" -}} +{{- if (kindIs "string" .) -}} +{{- $moduleName = . | trimAll "\"" -}} +{{- else -}} +{{- $moduleName = .Chart.Name -}} +{{- end -}} + +{{ $moduleName | replace "-" "_" | camelcase | untitle }} +{{- end -}} + + +{{- define "helm_lib_module_kebabcase_name" -}} +{{- $moduleName := "" -}} +{{- if (kindIs "string" .) -}} +{{- $moduleName = . | trimAll "\"" -}} +{{- else -}} +{{- $moduleName = .Chart.Name -}} +{{- end -}} + +{{ $moduleName | kebabcase }} +{{- end -}} diff --git a/charts/helm_lib/templates/_module_public_domain.tpl b/charts/helm_lib/templates/_module_public_domain.tpl new file mode 100644 index 000000000..bfbaae743 --- /dev/null +++ b/charts/helm_lib/templates/_module_public_domain.tpl @@ -0,0 +1,11 @@ +{{- /* Usage: {{ include "helm_lib_module_public_domain" (list . "") }} */ -}} +{{- /* returns rendered publicDomainTemplate to service fqdn */ -}} +{{- define "helm_lib_module_public_domain" }} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $name_portion := index . 1 -}} {{- /* Name portion */ -}} + + {{- if not (contains "%s" $context.Values.global.modules.publicDomainTemplate) }} + {{ fail "Error!!! global.modules.publicDomainTemplate must contain \"%s\" pattern to render service fqdn!" }} + {{- end }} + {{- printf $context.Values.global.modules.publicDomainTemplate $name_portion }} +{{- end }} diff --git a/charts/helm_lib/templates/_module_security_context.tpl b/charts/helm_lib/templates/_module_security_context.tpl new file mode 100644 index 000000000..1f510812e --- /dev/null +++ b/charts/helm_lib/templates/_module_security_context.tpl @@ -0,0 +1,263 @@ +{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_custom" (list . 1000 1000) }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with custom user and group */ -}} +{{- define "helm_lib_module_pod_security_context_run_as_user_custom" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +{{- /* User id */ -}} +{{- /* Group id */ -}} +securityContext: + runAsNonRoot: true + runAsUser: {{ index . 1 }} + runAsGroup: {{ index . 2 }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_nobody" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with user and group "nobody" */ -}} +{{- define "helm_lib_module_pod_security_context_run_as_user_nobody" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_nobody_with_writable_fs" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with user and group "nobody" with write access to mounted volumes */ -}} +{{- define "helm_lib_module_pod_security_context_run_as_user_nobody_with_writable_fs" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_deckhouse" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with user and group "deckhouse" */ -}} +{{- define "helm_lib_module_pod_security_context_run_as_user_deckhouse" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + runAsNonRoot: true + runAsUser: 64535 + runAsGroup: 64535 +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_deckhouse_with_writable_fs" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with user and group "deckhouse" with write access to mounted volumes */ -}} +{{- define "helm_lib_module_pod_security_context_run_as_user_deckhouse_with_writable_fs" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + runAsNonRoot: true + runAsUser: 64535 + runAsGroup: 64535 + fsGroup: 64535 +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_run_as_user_deckhouse_pss_restricted" . }} */ -}} +{{- /* returns SecurityContext parameters for Container with user and group "deckhouse" plus minimal required settings to comply with the Restricted mode of the Pod Security Standards */ -}} +{{- define "helm_lib_module_container_security_context_run_as_user_deckhouse_pss_restricted" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + runAsGroup: 64535 + runAsNonRoot: true + runAsUser: 64535 + seccompProfile: + type: RuntimeDefault +{{- end }} + + +{{- /* SecurityContext for Deckhouse UID/GID 64535 (or root), PSS Restricted */ -}} +{{- /* Optional keys: */ -}} +{{- /* .ro – bool, read-only root FS (default true) */ -}} +{{- /* .caps – []string, capabilities.add (default empty) */ -}} +{{- /* .uid – int, runAsUser/runAsGroup (default 64535) */ -}} +{{- /* .runAsNonRoot – bool, run as Deckhouse user when true, root when false (default true) */ -}} +{{- /* .seccompProfile – bool, disable seccompProfile when false (default true) */ -}} +{{- /* Usage: include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" false "caps" (list "NET_ADMIN" "SYS_TIME") "uid" 1001 "seccompProfile" false "runAsNonRoot" true) */ -}} +{{- define "helm_lib_module_container_security_context_pss_restricted_flexible" -}} +{{- $ro := true -}} +{{- if hasKey . "ro" -}} + {{- $ro = .ro -}} +{{- end -}} +{{- $seccompProfile := true -}} +{{- if hasKey . "seccompProfile" -}} + {{- $seccompProfile = .seccompProfile -}} +{{- end -}} +{{- $caps := default (list) .caps -}} +{{- $uid := default 64535 .uid -}} +{{- $runAsNonRoot := true -}} +{{- if hasKey . "runAsNonRoot" -}} + {{- $runAsNonRoot = .runAsNonRoot -}} +{{- end -}} + +securityContext: + readOnlyRootFilesystem: {{ $ro }} + allowPrivilegeEscalation: {{ not $runAsNonRoot }} +{{- if $runAsNonRoot }} + privileged: false +{{- end }} + capabilities: + drop: + - ALL +{{- if $caps }} + add: {{ $caps | toJson }} +{{- end }} + runAsUser: {{ ternary $uid 0 $runAsNonRoot }} + runAsGroup: {{ ternary $uid 0 $runAsNonRoot }} + runAsNonRoot: {{ $runAsNonRoot }} +{{- if $seccompProfile }} + seccompProfile: + type: RuntimeDefault +{{- end }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_root" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with user and group 0 */ -}} +{{- define "helm_lib_module_pod_security_context_run_as_user_root" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_pod_security_context_runtime_default" . }} */ -}} +{{- /* returns PodSecurityContext parameters for Pod with seccomp profile RuntimeDefault */ -}} +{{- define "helm_lib_module_pod_security_context_runtime_default" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + seccompProfile: + type: RuntimeDefault +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_not_allow_privilege_escalation" . }} */ -}} +{{- /* returns SecurityContext parameters for Container with allowPrivilegeEscalation false */ -}} +{{- define "helm_lib_module_container_security_context_not_allow_privilege_escalation" -}} +securityContext: + allowPrivilegeEscalation: false +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_read_only_root_filesystem_with_selinux" . }} */ -}} +{{- /* returns SecurityContext parameters for Container with read only root filesystem and options for SELinux compatibility*/ -}} +{{- define "helm_lib_module_container_security_context_read_only_root_filesystem_with_selinux" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + seLinuxOptions: + level: 's0' + type: 'spc_t' +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_read_only_root_filesystem" . }} */ -}} +{{- /* returns SecurityContext parameters for Container with read only root filesystem */ -}} +{{- define "helm_lib_module_container_security_context_read_only_root_filesystem" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_privileged" . }} */ -}} +{{- /* returns SecurityContext parameters for Container running privileged */ -}} +{{- define "helm_lib_module_container_security_context_privileged" -}} +securityContext: + privileged: true +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_escalated_sys_admin_privileged" . }} */ -}} +{{- /* returns SecurityContext parameters for Container running privileged with escalation and sys_admin */ -}} +{{- define "helm_lib_module_container_security_context_escalated_sys_admin_privileged" -}} +securityContext: + allowPrivilegeEscalation: true + readOnlyRootFilesystem: true + capabilities: + add: + - SYS_ADMIN + privileged: true +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" . }} */ -}} +{{- /* returns SecurityContext parameters for Container running privileged with read only root filesystem */ -}} +{{- define "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + privileged: true + readOnlyRootFilesystem: true +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all" . }} */ -}} +{{- /* returns SecurityContext for Container with read only root filesystem and all capabilities dropped */ -}} +{{- define "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} */ -}} +{{- /* returns SecurityContext parameters for Container with read only root filesystem, all dropped and some added capabilities */ -}} +{{- define "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +{{- /* List of capabilities */ -}} +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + add: {{ index . 1 | toJson }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} */ -}} +{{- /* returns SecurityContext parameters for Container with all dropped and some added capabilities */ -}} +{{- define "helm_lib_module_container_security_context_capabilities_drop_all_and_add" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +{{- /* List of capabilities */ -}} +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + add: {{ index . 1 | toJson }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_capabilities_drop_all_and_run_as_user_custom" (list . 1000 1000) }} */ -}} +{{- /* returns SecurityContext parameters for Container with read only root filesystem, all dropped, and custom user ID */ -}} +{{- define "helm_lib_module_container_security_context_capabilities_drop_all_and_run_as_user_custom" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +{{- /* User id */ -}} +{{- /* Group id */ -}} +securityContext: + runAsUser: {{ index . 1 }} + runAsGroup: {{ index . 2 }} + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault +{{- end }} + +{{- /* Usage: {{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . }} */ -}} +{{- /* returns SecurityContext parameters for Container with minimal required settings to comply with the Restricted mode of the Pod Security Standards */ -}} +{{- define "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" -}} +{{- /* Template context with .Values, .Chart, etc */ -}} +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault +{{- end }} diff --git a/charts/helm_lib/templates/_module_storage_class.tpl b/charts/helm_lib/templates/_module_storage_class.tpl new file mode 100644 index 000000000..e9c071766 --- /dev/null +++ b/charts/helm_lib/templates/_module_storage_class.tpl @@ -0,0 +1,36 @@ +{{- /* Usage: {{ include "helm_lib_module_storage_class_annotations" (list $ $index $storageClass.name) }} */ -}} +{{- /* return module StorageClass annotations */ -}} +{{- define "helm_lib_module_storage_class_annotations" -}} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $sc_index := index . 1 -}} {{- /* Storage class index */ -}} + {{- $sc_name := index . 2 -}} {{- /* Storage class name */ -}} + {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} + {{- $annotations := dict -}} + + {{- $volume_expansion_mode_offline := false -}} + {{- range $module_name := list "cloud-provider-azure" "cloud-provider-vsphere" "cloud-provider-vcd"}} + {{- if has $module_name $context.Values.global.enabledModules }} + {{- $volume_expansion_mode_offline = true }} + {{- end }} + {{- end }} + + {{- if $volume_expansion_mode_offline }} + {{- $_ := set $annotations "storageclass.deckhouse.io/volume-expansion-mode" "offline" }} + {{- end }} + + {{- if $context.Values.global.discovery.defaultStorageClass }} + {{- if eq $context.Values.global.discovery.defaultStorageClass $sc_name }} + {{- $_ := set $annotations "storageclass.kubernetes.io/is-default-class" "true" }} + {{- end }} + {{- else }} + {{- /* Annotate first StorageClass in list as default in case there is `global.discovery.defaultStorageClass` and */ -}} + {{- /* `global.defaultClusterStorageClass` NOT defined/empty */ -}} + {{- if (eq $sc_index 0) }} + {{- if or (not (hasKey $context.Values.global "defaultClusterStorageClass")) (and (hasKey $context.Values.global "defaultClusterStorageClass") (not $context.Values.global.defaultClusterStorageClass)) }} + {{- $_ := set $annotations "storageclass.kubernetes.io/is-default-class" "true" }} + {{- end }} + {{- end }} + {{- end }} + +{{- (dict "annotations" $annotations) | toYaml -}} +{{- end -}} diff --git a/charts/helm_lib/templates/_monitoring_grafana_dashboards.tpl b/charts/helm_lib/templates/_monitoring_grafana_dashboards.tpl new file mode 100644 index 000000000..def3d7ad7 --- /dev/null +++ b/charts/helm_lib/templates/_monitoring_grafana_dashboards.tpl @@ -0,0 +1,101 @@ +{{- /* Usage: {{ include "helm_lib_grafana_dashboard_definitions_recursion" (list . [current dir]) }} */ -}} +{{- /* returns all the dashboard-definintions from / */ -}} +{{- /* current dir is optional — used for recursion but you can use it for partially generating dashboards */ -}} +{{- define "helm_lib_grafana_dashboard_definitions_recursion" -}} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $rootDir := index . 1 }} {{- /* Dashboards root dir */ -}} + {{- /* Dashboards current dir */ -}} + + {{- $currentDir := "" }} + {{- if gt (len .) 2 }} {{- $currentDir = index . 2 }} {{- else }} {{- $currentDir = $rootDir }} {{- end }} + + {{- $currentDirIndex := (sub ($currentDir | splitList "/" | len) 1) }} + {{- $rootDirIndex := (sub ($rootDir | splitList "/" | len) 1) }} + {{- $folderNamesIndex := (add1 $rootDirIndex) }} + + {{- range $path, $_ := $context.Files.Glob (print $currentDir "/*.json") }} + {{- $fileName := ($path | splitList "/" | last ) }} + {{- $definition := ($context.Files.Get $path) }} + + {{- $folder := (index ($currentDir | splitList "/") $folderNamesIndex | replace "-" " " | title) }} + {{- $resourceName := (regexReplaceAllLiteral "\\.json$" $path "") }} + {{- $resourceName = ($resourceName | replace " " "-" | replace "." "-" | replace "_" "-") }} + {{- $resourceName = (slice ($resourceName | splitList "/") $folderNamesIndex | join "-") }} + {{- $resourceName = (printf "%s-%s" $context.Chart.Name $resourceName) }} + +{{ include "helm_lib_single_dashboard" (list $context $resourceName $folder $definition) }} + {{- end }} + + {{- range $path, $_ := $context.Files.Glob (print $currentDir "/*.tpl") }} + {{- $fileName := ($path | splitList "/" | last ) }} + {{- $definition := tpl ($context.Files.Get $path) $context }} + + {{- $folder := (index ($currentDir | splitList "/") $folderNamesIndex | replace "-" " " | title) }} + {{- $resourceName := (regexReplaceAllLiteral "\\.tpl$" $path "") }} + {{- $resourceName = ($resourceName | replace " " "-" | replace "." "-" | replace "_" "-") }} + {{- $resourceName = (slice ($resourceName | splitList "/") $folderNamesIndex | join "-") }} + {{- $resourceName = (printf "%s-%s" $context.Chart.Name $resourceName) }} + +{{ include "helm_lib_single_dashboard" (list $context $resourceName $folder $definition) }} + {{- end }} + + {{- $subDirs := list }} + {{- range $path, $_ := ($context.Files.Glob (print $currentDir "/**.json")) }} + {{- $pathSlice := ($path | splitList "/") }} + {{- $subDirs = append $subDirs (slice $pathSlice 0 (add $currentDirIndex 2) | join "/") }} + {{- end }} + {{- range $path, $_ := ($context.Files.Glob (print $currentDir "/**.tpl")) }} + {{- $pathSlice := ($path | splitList "/") }} + {{- $subDirs = append $subDirs (slice $pathSlice 0 (add $currentDirIndex 2) | join "/") }} + {{- end }} + + {{- range $subDir := ($subDirs | uniq) }} +{{ include "helm_lib_grafana_dashboard_definitions_recursion" (list $context $rootDir $subDir) }} + {{- end }} + +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_grafana_dashboard_definitions" . }} */ -}} +{{- /* returns dashboard-definintions from monitoring/grafana-dashboards/ */ -}} +{{- define "helm_lib_grafana_dashboard_definitions" -}} + {{- $context := . }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- if ( $context.Values.global.enabledModules | has "prometheus-crd" ) }} +{{- include "helm_lib_grafana_dashboard_definitions_recursion" (list $context "monitoring/grafana-dashboards") }} + {{- end }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_single_dashboard" (list . "dashboard-name" "folder" $dashboard) }} */ -}} +{{- /* renders a single dashboard */ -}} +{{- define "helm_lib_single_dashboard" -}} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $resourceName := index . 1 }} {{- /* Dashboard name */ -}} + {{- $folder := index . 2 }} {{- /* Folder */ -}} + {{- $definition := index . 3 }} {{/* Dashboard definition */}} + {{- $propagated := contains "-propagated-" $resourceName }} + {{- $resourceName = $resourceName | replace "-propagated-" "-" }} +--- +apiVersion: deckhouse.io/v1 +kind: GrafanaDashboardDefinition +metadata: + name: d8-{{ $resourceName }} + {{- include "helm_lib_module_labels" (list $context (dict "prometheus.deckhouse.io/grafana-dashboard" "" "observability.deckhouse.io/skip-dashboard-conversion" "")) | nindent 2 }} +spec: + folder: {{ $folder | quote }} + definition: | + {{- $definition | nindent 4 }} + {{- if $context.Values.global.enabledModules | has "observability" }} +--- +apiVersion: observability.deckhouse.io/v1alpha1 +kind: {{ $propagated | ternary "ClusterObservabilityPropagatedDashboard" "ClusterObservabilityDashboard" }} +metadata: + annotations: + metadata.deckhouse.io/category: {{ $folder | quote }} + name: d8-{{ $resourceName }} + {{- include "helm_lib_module_labels" (list $context (dict "observability.deckhouse.io/dashboard-origin" "module")) | nindent 2 }} +spec: + definition: | + {{- $definition | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/helm_lib/templates/_monitoring_prometheus_rules.tpl b/charts/helm_lib/templates/_monitoring_prometheus_rules.tpl new file mode 100644 index 000000000..86c75bb7b --- /dev/null +++ b/charts/helm_lib/templates/_monitoring_prometheus_rules.tpl @@ -0,0 +1,135 @@ +{{- /* Usage: {{ include "helm_lib_prometheus_rules_recursion" (list . [current dir] [file list]) }} */ -}} +{{- /* returns all the prometheus rules from / */ -}} +{{- /* current dir is optional — used for recursion but you can use it for partially generating rules */ -}} +{{- /* file list is optional - list of files to include (filters all files if provided) */ -}} +{{- define "helm_lib_prometheus_rules_recursion" -}} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $namespace := index . 1 }} {{- /* Namespace for creating rules */ -}} + {{- $rootDir := index . 2 }} {{- /* Rules root dir */ -}} + {{- $currentDir := "" }} {{- /* Current dir (optional) */ -}} + {{- $fileList := list }} {{- /* File list for filtering (optional) */ -}} + + {{- if gt (len .) 3 }} {{- $currentDir = index . 3 }} {{- else }} {{- $currentDir = $rootDir }} {{- end }} + {{- if gt (len .) 4 }} {{- $fileList = index . 4 }} {{- end }} + + {{- $currentDirIndex := (sub ($currentDir | splitList "/" | len) 1) }} + {{- $rootDirIndex := (sub ($rootDir | splitList "/" | len) 1) }} + {{- $folderNamesIndex := (add1 $rootDirIndex) }} + + {{- range $path, $_ := $context.Files.Glob (print $currentDir "/*.{yaml,tpl}") }} + {{- /* Filter files if fileList is provided */ -}} + {{- $shouldProcess := true }} + {{- if gt (len $fileList) 0 }} + {{- $shouldProcess = has $path $fileList }} + {{- end }} + + {{- if $shouldProcess }} + {{- $fileName := ($path | splitList "/" | last ) }} + {{- $definition := "" }} + {{- if eq ($path | splitList "." | last) "tpl" -}} + {{- $definition = tpl ($context.Files.Get $path) $context }} + {{- else }} + {{- $definition = $context.Files.Get $path }} + {{- end }} + + {{- $definition = $definition | replace "__SCRAPE_INTERVAL__" (printf "%ds" ($context.Values.global.discovery.prometheusScrapeInterval | default 30)) | replace "__SCRAPE_INTERVAL_X_2__" (printf "%ds" (mul ($context.Values.global.discovery.prometheusScrapeInterval | default 30) 2)) | replace "__SCRAPE_INTERVAL_X_3__" (printf "%ds" (mul ($context.Values.global.discovery.prometheusScrapeInterval | default 30) 3)) | replace "__SCRAPE_INTERVAL_X_4__" (printf "%ds" (mul ($context.Values.global.discovery.prometheusScrapeInterval | default 30) 4)) }} + +{{/* Patch expression based on `d8_ignore_on_update` annotation*/}} + + {{ $definition = printf "Rules:\n%s" ($definition | nindent 2) }} + {{- $definitionStruct := ( $definition | fromYaml )}} + {{- if $definitionStruct.Error }} + {{- fail ($definitionStruct.Error | toString) }} + {{- end }} + {{- range $rule := $definitionStruct.Rules }} + + {{- range $dedicatedRule := $rule.rules }} + {{- if $dedicatedRule.annotations }} + {{- if (eq (get $dedicatedRule.annotations "d8_ignore_on_update") "true") }} + {{- $_ := set $dedicatedRule "expr" (printf "(%s) and ON() ((max(d8_is_updating) != 1) or ON() absent(d8_is_updating))" $dedicatedRule.expr) }} + {{- end }} + {{- end }} + {{- end }} + + {{- end }} + + {{- $resourceName := (regexReplaceAllLiteral "\\.(yaml|tpl)$" $path "") }} + {{- $resourceName = ($resourceName | replace " " "-" | replace "." "-" | replace "_" "-") }} + {{- $resourceName = (slice ($resourceName | splitList "/") $folderNamesIndex | join "-") }} + {{- $resourceName = (printf "%s-%s" $context.Chart.Name $resourceName) }} + {{- $propagated := contains "propagated-" $resourceName }} + {{- $hasObservabilityModule := has "observability" $context.Values.global.enabledModules }} + {{- $useObservabilityRules := has "observability.deckhouse.io/v1alpha1/ClusterObservabilityMetricsRulesGroup" $context.Values.global.discovery.apiVersions }} + {{- if and $hasObservabilityModule $useObservabilityRules }} + {{- range $idx, $group := $definitionStruct.Rules }} + {{- if $group.rules }} + {{- $_ := unset $group "name" }} + {{- $resourceName = $resourceName | replace "propagated-" "" }} + {{- $groupResourceName := printf "%s-%d" $resourceName $idx }} +--- +apiVersion: observability.deckhouse.io/v1alpha1 +kind: {{ $propagated | ternary "ClusterObservabilityPropagatedMetricsRulesGroup" "ClusterObservabilityMetricsRulesGroup" }} +metadata: + name: {{ $groupResourceName }} + {{- include "helm_lib_module_labels" (list $context (dict "app" "prometheus" "prometheus" "main" "component" "rules")) | nindent 2 }} +spec: + {{- $group | toYaml | nindent 2 }} + {{- end }} + {{- end }} + {{- else }} + {{- if $definitionStruct.Rules }} + {{- $definition := $definitionStruct.Rules | toYaml }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ $resourceName }} + namespace: {{ $namespace }} + {{- include "helm_lib_module_labels" (list $context (dict "app" "prometheus" "prometheus" "main" "component" "rules")) | nindent 2 }} +spec: + groups: + {{- $definition | nindent 4 }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{- $subDirs := list }} + {{- range $path, $_ := ($context.Files.Glob (print $currentDir "/**.{yaml,tpl}")) }} + {{- $pathSlice := ($path | splitList "/") }} + {{- $subDirs = append $subDirs (slice $pathSlice 0 (add $currentDirIndex 2) | join "/") }} + {{- end }} + + {{- range $subDir := ($subDirs | uniq) }} +{{ include "helm_lib_prometheus_rules_recursion" (list $context $namespace $rootDir $subDir $fileList) }} + {{- end }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_prometheus_rules" (list . [fileList]) }} */ -}} +{{- /* returns all the prometheus rules from monitoring/prometheus-rules/ optionally filtered by fileList */ -}} +{{- define "helm_lib_prometheus_rules" -}} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $namespace := index . 1 }} {{- /* Namespace for creating rules */ -}} + {{- $rootDir := "monitoring/prometheus-rules" }} + {{- $fileList := list }} + {{- if gt (len .) 2 }} + {{- $fileList = index . 2 }} + {{- end }} + {{- if ( $context.Values.global.enabledModules | has "operator-prometheus-crd" ) }} +{{- include "helm_lib_prometheus_rules_recursion" (list $context $namespace $rootDir $rootDir $fileList) }} + {{- end }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_prometheus_target_scrape_timeout_seconds" (list . ) }} */ -}} +{{- /* returns adjust timeout value to scrape interval / */ -}} +{{- define "helm_lib_prometheus_target_scrape_timeout_seconds" -}} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $timeout := index . 1 }} {{- /* Target timeout in seconds */ -}} + {{- $scrape_interval := (int $context.Values.global.discovery.prometheusScrapeInterval | default 30) }} + {{- if gt $timeout $scrape_interval -}} +{{ $scrape_interval }}s + {{- else -}} +{{ $timeout }}s + {{- end }} +{{- end }} diff --git a/charts/helm_lib/templates/_node_affinity.tpl b/charts/helm_lib/templates/_node_affinity.tpl new file mode 100644 index 000000000..ab06d959c --- /dev/null +++ b/charts/helm_lib/templates/_node_affinity.tpl @@ -0,0 +1,262 @@ +{{- /* Verify node selector strategy. */ -}} +{{- define "helm_lib_internal_check_node_selector_strategy" -}} + {{ if not (has . (list "frontend" "monitoring" "system" "master" )) }} + {{- fail (printf "unknown strategy \"%v\"" .) }} + {{- end }} + {{- . -}} +{{- end }} + +{{- /* Returns node selector for workloads depend on strategy. */ -}} +{{- define "helm_lib_node_selector" }} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $strategy := index . 1 | include "helm_lib_internal_check_node_selector_strategy" }} {{- /* strategy, one of "frontend" "monitoring" "system" "master" "any-node" "wildcard" */ -}} + {{- $module_values := dict }} + {{- if lt (len .) 3 }} + {{- $module_values = (index $context.Values (include "helm_lib_module_camelcase_name" $context)) }} + {{- else }} + {{- $module_values = index . 2 }} + {{- end }} + {{- $camel_chart_name := (include "helm_lib_module_camelcase_name" $context) }} + + {{- if eq $strategy "monitoring" }} + {{- if $module_values.nodeSelector }} +nodeSelector: {{ $module_values.nodeSelector | toJson }} + {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole $camel_chart_name | int) 0 }} +nodeSelector: + node-role.deckhouse.io/{{$context.Chart.Name}}: "" + {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole $strategy | int) 0 }} +nodeSelector: + node-role.deckhouse.io/{{$strategy}}: "" + {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "system" | int) 0 }} +nodeSelector: + node-role.deckhouse.io/system: "" + {{- end }} + + {{- else if or (eq $strategy "frontend") (eq $strategy "system") }} + {{- if $module_values.nodeSelector }} +nodeSelector: {{ $module_values.nodeSelector | toJson }} + {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole $camel_chart_name | int) 0 }} +nodeSelector: + node-role.deckhouse.io/{{$context.Chart.Name}}: "" + {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole $strategy | int) 0 }} +nodeSelector: + node-role.deckhouse.io/{{$strategy}}: "" + {{- end }} + + {{- else if eq $strategy "master" }} + {{- if gt (index $context.Values.global.discovery "clusterMasterCount" | int) 0 }} +nodeSelector: + node-role.kubernetes.io/control-plane: "" + {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 0 }} +nodeSelector: + node-role.deckhouse.io/control-plane: "" + {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "system" | int) 0 }} +nodeSelector: + node-role.deckhouse.io/system: "" + {{- end }} + {{- end }} +{{- end }} + + +{{- /* Returns tolerations for workloads depend on strategy. */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "with-uninitialized" "without-storage-problems") }} */ -}} +{{- define "helm_lib_tolerations" }} + {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $strategy := index . 1 | include "helm_lib_internal_check_tolerations_strategy" }} {{- /* base strategy, one of "frontend" "monitoring" "system" any-node" "wildcard" */ -}} + {{- $additionalStrategies := tuple }} {{- /* list of additional strategies. To add strategy list it with prefix "with-", to remove strategy list it with prefix "without-". */ -}} + {{- if eq $strategy "custom" }} + {{ if lt (len .) 3 }} + {{- fail (print "additional strategies is required") }} + {{- end }} + {{- else }} + {{- $additionalStrategies = tuple "storage-problems" }} + {{- end }} + {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) }} + {{- if gt (len .) 2 }} + {{- range $as := slice . 2 (len .) }} + {{- if hasPrefix "with-" $as }} + {{- $additionalStrategies = mustAppend $additionalStrategies (trimPrefix "with-" $as) }} + {{- end }} + {{- if hasPrefix "without-" $as }} + {{- $additionalStrategies = mustWithout $additionalStrategies (trimPrefix "without-" $as) }} + {{- end }} + {{- end }} + {{- end }} +tolerations: + {{- /* Wildcard: gives permissions to schedule on any node with any taints (use with caution) */ -}} + {{- if eq $strategy "wildcard" }} + {{- include "_helm_lib_wildcard_tolerations" $context }} + + {{- else }} + {{- /* Any node: any node in the cluster with any known taints */ -}} + {{- if eq $strategy "any-node" }} + {{- include "_helm_lib_any_node_tolerations" $context }} + + {{- /* Tolerations from module config: overrides below strategies, if there is any toleration specified */ -}} + {{- else if $module_values.tolerations }} + {{- $module_values.tolerations | toYaml | nindent 0 }} + + {{- /* Monitoring: Nodes for monitoring components: prometheus, grafana, kube-state-metrics, etc. */ -}} + {{- else if eq $strategy "monitoring" }} + {{- include "_helm_lib_monitoring_tolerations" $context }} + + {{- /* Frontend: Nodes for ingress-controllers */ -}} + {{- else if eq $strategy "frontend" }} + {{- include "_helm_lib_frontend_tolerations" $context }} + + {{- /* System: Nodes for system components: prometheus, dns, cert-manager */ -}} + {{- else if eq $strategy "system" }} + {{- include "_helm_lib_system_tolerations" $context }} + {{- end }} + + {{- /* Additional strategies */ -}} + {{- range $additionalStrategies -}} + {{- include (printf "_helm_lib_additional_tolerations_%s" (. | replace "-" "_")) $context }} + {{- end }} + {{- end }} +{{- end }} + + +{{- /* Check cluster type. */ -}} +{{- /* Returns not empty string if this is cloud or hybrid cluster */ -}} +{{- define "_helm_lib_cloud_or_hybrid_cluster" }} + {{- if .Values.global.clusterConfiguration }} + {{- if eq .Values.global.clusterConfiguration.clusterType "Cloud" }} + "not empty string" + {{- /* We consider non-cloud clusters with enabled cloud-provider-.* module as Hybrid clusters */ -}} + {{- else }} + {{- range $v := .Values.global.enabledModules }} + {{- if hasPrefix "cloud-provider-" $v }} + "not empty string" + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- /* Verify base strategy. */ -}} +{{- /* Fails if strategy not in allowed list */ -}} +{{- define "helm_lib_internal_check_tolerations_strategy" -}} + {{ if not (has . (list "custom" "frontend" "monitoring" "system" "any-node" "wildcard" )) }} + {{- fail (printf "unknown strategy \"%v\"" .) }} + {{- end }} + {{- . -}} +{{- end }} + + +{{- /* Base strategy for any uncordoned node in cluster. */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node") }} */ -}} +{{- define "_helm_lib_any_node_tolerations" }} +- key: node-role.kubernetes.io/master +- key: node-role.kubernetes.io/control-plane +- key: node.deckhouse.io/etcd-arbiter +- key: dedicated.deckhouse.io + operator: "Exists" +- key: dedicated + operator: "Exists" +- key: DeletionCandidateOfClusterAutoscaler +- key: ToBeDeletedByClusterAutoscaler + {{- if .Values.global.modules.placement.customTolerationKeys }} + {{- range $key := .Values.global.modules.placement.customTolerationKeys }} +- key: {{ $key | quote }} + operator: "Exists" + {{- end }} + {{- end }} +{{- end }} + +{{- /* Base strategy that tolerates all. */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "wildcard") }} */ -}} +{{- define "_helm_lib_wildcard_tolerations" }} +- operator: "Exists" +{{- end }} + +{{- /* Base strategy that tolerates nodes with "dedicated.deckhouse.io: monitoring" and "dedicated.deckhouse.io: system" taints. */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "monitoring") }} */ -}} +{{- define "_helm_lib_monitoring_tolerations" }} +- key: dedicated.deckhouse.io + operator: Equal + value: {{ .Chart.Name | quote }} +- key: dedicated.deckhouse.io + operator: Equal + value: "monitoring" +- key: dedicated.deckhouse.io + operator: Equal + value: "system" +{{- end }} + +{{- /* Base strategy that tolerates nodes with "dedicated.deckhouse.io: frontend" taints. */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "frontend") }} */ -}} +{{- define "_helm_lib_frontend_tolerations" }} +- key: dedicated.deckhouse.io + operator: Equal + value: {{ .Chart.Name | quote }} +- key: dedicated.deckhouse.io + operator: Equal + value: "frontend" +{{- end }} + +{{- /* Base strategy that tolerates nodes with "dedicated.deckhouse.io: system" taints. */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "system") }} */ -}} +{{- define "_helm_lib_system_tolerations" }} +- key: dedicated.deckhouse.io + operator: Equal + value: {{ .Chart.Name | quote }} +- key: dedicated.deckhouse.io + operator: Equal + value: "system" +{{- end }} + + +{{- /* Additional strategy "uninitialized" - used for CNI's and kube-proxy to allow cni components scheduled on node after CCM initialization. */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "with-uninitialized") }} */ -}} +{{- define "_helm_lib_additional_tolerations_uninitialized" }} +- key: node.deckhouse.io/bashible-uninitialized + operator: "Exists" + effect: "NoSchedule" +- key: node.deckhouse.io/uninitialized + operator: "Exists" + effect: "NoSchedule" + {{- if include "_helm_lib_cloud_or_hybrid_cluster" . }} + {{- include "_helm_lib_additional_tolerations_no_csi" . }} + {{- end }} + {{- include "_helm_lib_additional_tolerations_node_problems" . }} +{{- end }} + +{{- /* Additional strategy "node-problems" - used for shedule critical components on non-ready nodes or nodes under pressure. */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "with-node-problems") }} */ -}} +{{- define "_helm_lib_additional_tolerations_node_problems" }} +- key: node.kubernetes.io/not-ready +- key: node.kubernetes.io/out-of-disk +- key: node.kubernetes.io/memory-pressure +- key: node.kubernetes.io/disk-pressure +- key: node.kubernetes.io/pid-pressure +- key: node.kubernetes.io/unreachable +- key: node.kubernetes.io/network-unavailable +{{- end }} + +{{- /* Additional strategy "storage-problems" - used for shedule critical components on nodes with drbd problems. This additional strategy enabled by default in any base strategy except "wildcard". */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "without-storage-problems") }} */ -}} +{{- define "_helm_lib_additional_tolerations_storage_problems" }} +- key: drbd.linbit.com/lost-quorum +- key: drbd.linbit.com/force-io-error +- key: drbd.linbit.com/ignore-fail-over +{{- end }} + +{{- /* Additional strategy "no-csi" - used for any node with no CSI: any node, which was initialized by deckhouse, but have no csi-node driver registered on it. */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "with-no-csi") }} */ -}} +{{- define "_helm_lib_additional_tolerations_no_csi" }} +- key: ToBeDeletedTaint + operator: "Exists" +- key: node.deckhouse.io/csi-not-bootstrapped + operator: "Exists" + effect: "NoSchedule" +{{- end }} + +{{- /* Additional strategy "cloud-provider-uninitialized" - used for any node which is not initialized by CCM. */ -}} +{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "with-cloud-provider-uninitialized") }} */ -}} +{{- define "_helm_lib_additional_tolerations_cloud_provider_uninitialized" }} + {{- if not .Values.global.clusterIsBootstrapped }} +- key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + {{- end }} +{{- end }} diff --git a/charts/helm_lib/templates/_pod_disruption_budget.tpl b/charts/helm_lib/templates/_pod_disruption_budget.tpl new file mode 100644 index 000000000..ccd4f211d --- /dev/null +++ b/charts/helm_lib/templates/_pod_disruption_budget.tpl @@ -0,0 +1,6 @@ +{{- /* Usage: {{ include "helm_lib_pdb_daemonset" . }} */ -}} +{{- /* Returns PDB max unavailable */ -}} +{{- define "helm_lib_pdb_daemonset" }} + {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} +maxUnavailable: 10% +{{- end -}} diff --git a/charts/helm_lib/templates/_priority_class.tpl b/charts/helm_lib/templates/_priority_class.tpl new file mode 100644 index 000000000..7da49b94e --- /dev/null +++ b/charts/helm_lib/templates/_priority_class.tpl @@ -0,0 +1,7 @@ +{{- /* Usage: {{ include "helm_lib_priority_class" (tuple . "priority-class-name") }} /* -}} +{{- /* returns priority class */ -}} +{{- define "helm_lib_priority_class" }} + {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} + {{- $priorityClassName := index . 1 }} {{- /* Priority class name */ -}} +priorityClassName: {{ $priorityClassName }} +{{- end -}} diff --git a/charts/helm_lib/templates/_resources_management.tpl b/charts/helm_lib/templates/_resources_management.tpl new file mode 100644 index 000000000..748f7dbb0 --- /dev/null +++ b/charts/helm_lib/templates/_resources_management.tpl @@ -0,0 +1,160 @@ +{{- /* Usage: {{ include "helm_lib_resources_management_pod_resources" (list [ephemeral storage requests]) }} */ -}} +{{- /* returns rendered resources section based on configuration if it is */ -}} +{{- define "helm_lib_resources_management_pod_resources" -}} + {{- $configuration := index . 0 -}} {{- /* VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) */ -}} + {{- /* Ephemeral storage requests */ -}} + + {{- $ephemeral_storage := "50Mi" -}} + {{- if eq (len .) 2 -}} + {{- $ephemeral_storage = index . 1 -}} + {{- end -}} + + {{- $pod_resources := (include "helm_lib_resources_management_original_pod_resources" $configuration | fromYaml) -}} + {{- if not (hasKey $pod_resources "requests") -}} + {{- $_ := set $pod_resources "requests" (dict) -}} + {{- end -}} + {{- $_ := set $pod_resources.requests "ephemeral-storage" $ephemeral_storage -}} + + {{- $pod_resources | toYaml -}} +{{- end -}} + + +{{- /* Usage: {{ include "helm_lib_resources_management_original_pod_resources" }} */ -}} +{{- /* returns rendered resources section based on configuration if it is present */ -}} +{{- define "helm_lib_resources_management_original_pod_resources" -}} + {{- $configuration := . -}} {{- /* VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) */ -}} + + {{- if $configuration -}} + {{- if eq $configuration.mode "Static" -}} +{{- $configuration.static | toYaml -}} + + {{- else if eq $configuration.mode "VPA" -}} + {{- $resources := dict "requests" (dict) "limits" (dict) -}} + + {{- if $configuration.vpa.cpu -}} + {{- if $configuration.vpa.cpu.min -}} + {{- $_ := set $resources.requests "cpu" ($configuration.vpa.cpu.min | toString) -}} + {{- end -}} + {{- if $configuration.vpa.cpu.limitRatio -}} + {{- $cpuLimitMillicores := round (mulf (include "helm_lib_resources_management_cpu_units_to_millicores" $configuration.vpa.cpu.min) $configuration.vpa.cpu.limitRatio) 0 | int64 -}} + {{- $_ := set $resources.limits "cpu" (printf "%dm" $cpuLimitMillicores) -}} + {{- end -}} + {{- end -}} + + {{- if $configuration.vpa.memory -}} + {{- if $configuration.vpa.memory.min -}} + {{- $_ := set $resources.requests "memory" ($configuration.vpa.memory.min | toString) -}} + {{- end -}} + {{- if $configuration.vpa.memory.limitRatio -}} + {{- $memoryLimitBytes := round (mulf (include "helm_lib_resources_management_memory_units_to_bytes" $configuration.vpa.memory.min) $configuration.vpa.memory.limitRatio) 0 | int64 -}} + {{- $_ := set $resources.limits "memory" (printf "%d" $memoryLimitBytes) -}} + {{- end -}} + {{- end -}} +{{- $resources | toYaml -}} + + {{- else -}} + {{- cat "ERROR: unknown resource management mode: " $configuration.mode | fail -}} + {{- end -}} + {{- end -}} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_resources_management_vpa_spec" (list ) }} */ -}} +{{- /* returns rendered vpa spec based on configuration and target reference */ -}} +{{- define "helm_lib_resources_management_vpa_spec" -}} + {{- $targetAPIVersion := index . 0 -}} {{- /* Target API version */ -}} + {{- $targetKind := index . 1 -}} {{- /* Target Kind */ -}} + {{- $targetName := index . 2 -}} {{- /* Target Name */ -}} + {{- $targetContainer := index . 3 -}} {{- /* Target container name */ -}} + {{- $configuration := index . 4 -}} {{- /* VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) */ -}} + +targetRef: + apiVersion: {{ $targetAPIVersion }} + kind: {{ $targetKind }} + name: {{ $targetName }} + {{- if eq ($configuration.mode) "VPA" }} +updatePolicy: + updateMode: {{ $configuration.vpa.mode | quote }} +resourcePolicy: + containerPolicies: + - containerName: {{ $targetContainer }} + maxAllowed: + cpu: {{ $configuration.vpa.cpu.max | quote }} + memory: {{ $configuration.vpa.memory.max | quote }} + minAllowed: + cpu: {{ $configuration.vpa.cpu.min | quote }} + memory: {{ $configuration.vpa.memory.min | quote }} + controlledValues: RequestsAndLimits + {{- else }} +updatePolicy: + updateMode: "Off" + {{- end }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_resources_management_cpu_units_to_millicores" }} */ -}} +{{- /* helper for converting cpu units to millicores */ -}} +{{- define "helm_lib_resources_management_cpu_units_to_millicores" -}} + {{- $units := . | toString -}} + {{- if hasSuffix "m" $units -}} + {{- trimSuffix "m" $units -}} + {{- else -}} + {{- atoi $units | mul 1000 -}} + {{- end }} +{{- end }} + + +{{- /* Usage: {{ include "helm_lib_resources_management_memory_units_to_bytes" }} */ -}} +{{- /* helper for converting memory units to bytes */ -}} +{{- define "helm_lib_resources_management_memory_units_to_bytes" }} + {{- $units := . | toString -}} + {{- if hasSuffix "k" $units -}} + {{- trimSuffix "k" $units | atoi | mul 1000 -}} + {{- else if hasSuffix "M" $units -}} + {{- trimSuffix "M" $units | atoi | mul 1000000 -}} + {{- else if hasSuffix "G" $units -}} + {{- trimSuffix "G" $units | atoi | mul 1000000000 -}} + {{- else if hasSuffix "T" $units -}} + {{- trimSuffix "T" $units | atoi | mul 1000000000000 -}} + {{- else if hasSuffix "P" $units -}} + {{- trimSuffix "P" $units | atoi | mul 1000000000000000 -}} + {{- else if hasSuffix "E" $units -}} + {{- trimSuffix "E" $units | atoi | mul 1000000000000000000 -}} + {{- else if hasSuffix "Ki" $units -}} + {{- trimSuffix "Ki" $units | atoi | mul 1024 -}} + {{- else if hasSuffix "Mi" $units -}} + {{- trimSuffix "Mi" $units | atoi | mul 1024 | mul 1024 -}} + {{- else if hasSuffix "Gi" $units -}} + {{- trimSuffix "Gi" $units | atoi | mul 1024 | mul 1024 | mul 1024 -}} + {{- else if hasSuffix "Ti" $units -}} + {{- trimSuffix "Ti" $units | atoi | mul 1024 | mul 1024 | mul 1024 | mul 1024 -}} + {{- else if hasSuffix "Pi" $units -}} + {{- trimSuffix "Pi" $units | atoi | mul 1024 | mul 1024 | mul 1024 | mul 1024 | mul 1024 -}} + {{- else if hasSuffix "Ei" $units -}} + {{- trimSuffix "Ei" $units | atoi | mul 1024 | mul 1024 | mul 1024 | mul 1024 | mul 1024 | mul 1024 -}} + {{- else if regexMatch "^[0-9]+$" $units -}} + {{- $units -}} + {{- else -}} + {{- cat "ERROR: unknown memory format:" $units | fail -}} + {{- end }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_vpa_kube_rbac_proxy_resources" . }} */ -}} +{{- /* helper for VPA resources for kube_rbac_proxy */ -}} +{{- define "helm_lib_vpa_kube_rbac_proxy_resources" }} +{{- /* Template context with .Values, .Chart, etc */ -}} +- containerName: kube-rbac-proxy + minAllowed: + {{- include "helm_lib_container_kube_rbac_proxy_resources" . | nindent 4 }} + maxAllowed: + cpu: 20m + memory: 25Mi +{{- end }} + +{{- /* Usage: {{ include "helm_lib_container_kube_rbac_proxy_resources" . }} */ -}} +{{- /* helper for container resources for kube_rbac_proxy */ -}} +{{- define "helm_lib_container_kube_rbac_proxy_resources" }} +{{- /* Template context with .Values, .Chart, etc */ -}} +cpu: 10m +memory: 25Mi +{{- end }} diff --git a/charts/helm_lib/templates/_spec_for_high_availability.tpl b/charts/helm_lib/templates/_spec_for_high_availability.tpl new file mode 100644 index 000000000..2cd393a83 --- /dev/null +++ b/charts/helm_lib/templates/_spec_for_high_availability.tpl @@ -0,0 +1,179 @@ +{{- /* Usage: {{ include "helm_lib_pod_anti_affinity_for_ha" (list . (dict "app" "test")) }} */ -}} +{{- /* returns pod affinity spec */ -}} +{{- define "helm_lib_pod_anti_affinity_for_ha" }} +{{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} +{{- $labels := index . 1 }} {{- /* Match labels for podAntiAffinity label selector */ -}} + {{- if (include "helm_lib_ha_enabled" $context) }} +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + {{- range $key, $value := $labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: kubernetes.io/hostname + {{- end }} +{{- end }} + +{{- /* Usage: {{- include "helm_lib_pod_affinity" (list . (dict "app" "test") (list "amd64")) }} */}} +{{- /* Returns affinity spec that combines: podAntiAffinity by provided labels when HA is enabled and optional nodeAffinity that schedules pods only on specified architectures. If the list of architectures is not provided or empty, node affinity is not rendered. */ -}} +{{- define "helm_lib_pod_affinity" }} +{{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} +{{- $labels := dict -}} {{- /* Match labels for podAntiAffinity label selector */ -}} +{{- if ge (len .) 2 }} + {{- $labels = index . 1 }} +{{- end }} +{{- $allowedArchs := list -}} {{- /* List of supported architectures */ -}} +{{- if ge (len .) 3 }} + {{- $allowedArchs = index . 2 }} +{{- end }} +{{- $haEnabled := (include "helm_lib_ha_enabled" $context) -}} +{{- $hasArch := gt (len $allowedArchs) 0 -}} +{{- if or $haEnabled $hasArch }} +affinity: + {{- if $haEnabled }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + {{- range $key, $value := $labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: kubernetes.io/hostname + {{- end }} + {{- if $hasArch }} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + {{- range $allowedArchs }} + - {{ . | quote }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" }} */ -}} +{{- /* returns deployment strategy and replicas for ha components running on master nodes */ -}} +{{- define "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" }} +{{- /* Template context with .Values, .Chart, etc */ -}} + {{- if (include "helm_lib_ha_enabled" .) }} + {{- if gt (index .Values.global.discovery "clusterMasterCount" | int) 0 }} +replicas: {{ index .Values.global.discovery "clusterMasterCount" }} +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + {{- if gt (index .Values.global.discovery "clusterMasterCount" | int) 2 }} + maxUnavailable: 2 + {{- else }} + maxUnavailable: 1 + {{- end }} + {{- else if gt (index .Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 0 }} +replicas: {{ index .Values.global.discovery.d8SpecificNodeCountByRole "master" }} +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + {{- if gt (index .Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 2 }} + maxUnavailable: 2 + {{- else }} + maxUnavailable: 1 + {{- end }} + {{- else }} +replicas: 2 +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + {{- end }} + {{- else }} +replicas: 1 +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + {{- end }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha" (list . (dict "strategy" "strategy_type")) }} */ -}} +{{- /* returns deployment with custom strategy and replicas for ha components running on master nodes */ -}} +{{- define "helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha" }} +{{- $context := index . 0 }} +{{- $optionalArgs := dict }} +{{- $strategy := "RollingUpdate" }} +{{- if ge (len .) 2 }} + {{- $optionalArgs = index . 1 }} +{{- end }} +{{- if hasKey $optionalArgs "strategy" }} + {{- $strategy = $optionalArgs.strategy }} +{{- end }} +{{- /* Template context with .Values, .Chart, etc */ -}} + {{- if (include "helm_lib_ha_enabled" $context) }} + {{- if gt (index $context.Values.global.discovery "clusterMasterCount" | int) 0 }} +replicas: {{ index $context.Values.global.discovery "clusterMasterCount" }} +strategy: + type: {{ $strategy }} + {{- if eq $strategy "RollingUpdate" }} + rollingUpdate: + maxSurge: 0 + {{- if gt (index $context.Values.global.discovery "clusterMasterCount" | int) 2 }} + maxUnavailable: 2 + {{- else }} + maxUnavailable: 1 + {{- end }} + {{- end }} + {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 0 }} +replicas: {{ index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" }} +strategy: + type: {{ $strategy }} + {{- if eq $strategy "RollingUpdate" }} + rollingUpdate: + maxSurge: 0 + {{- if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 2 }} + maxUnavailable: 2 + {{- else }} + maxUnavailable: 1 + {{- end }} + {{- end }} + {{- else }} +replicas: 2 +strategy: + type: {{ $strategy }} + {{- if eq $strategy "RollingUpdate" }} + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + {{- end }} + {{- end }} + {{- else }} +replicas: 1 +strategy: + type: {{ $strategy }} + {{- if eq $strategy "RollingUpdate" }} + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + {{- end }} + {{- end }} +{{- end }} + +{{- /* Usage: {{ include "helm_lib_deployment_strategy_and_replicas_for_ha" }} */ -}} +{{- /* returns deployment strategy and replicas for ha components running not on master nodes */ -}} +{{- define "helm_lib_deployment_strategy_and_replicas_for_ha" }} +{{- /* Template context with .Values, .Chart, etc */ -}} +replicas: {{ include "helm_lib_is_ha_to_value" (list . 2 1) }} +{{- if (include "helm_lib_ha_enabled" .) }} +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 +{{- end }} +{{- end }} From 79000377d70888136afadbb1cd98a7e9e80a2b2b Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Wed, 24 Jun 2026 13:16:23 +0300 Subject: [PATCH 11/32] =?UTF-8?q?fix(crd):=20=D0=B8=D1=81=D0=BF=D1=80?= =?UTF-8?q?=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D0=B0=20CEL-=D0=B2=D0=B0=D0=BB?= =?UTF-8?q?=D0=B8=D0=B4=D0=B0=D1=86=D0=B8=D1=8F=20=D0=BD=D0=B5=D0=B8=D0=B7?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D0=BD=D0=BE=D1=81=D1=82=D0=B8=20fileDevices?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transition rule перенесена с полей элементов map-list на spec/lvmVolumeGroupTemplate: oldSelf на item-level внутри fileDevices отклонялся apiserver как uncorrelatable и превышал CEL cost budget. Добавлены maxItems и maxLength для снижения стоимости правил. Agents: Composer 2.5 (ведущий, авто-режим) --- crds/lvmvolumegroup.yaml | 11 +++++------ crds/lvmvolumegroupset.yaml | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/crds/lvmvolumegroup.yaml b/crds/lvmvolumegroup.yaml index dedaceb23..d2f908e9e 100644 --- a/crds/lvmvolumegroup.yaml +++ b/crds/lvmvolumegroup.yaml @@ -41,6 +41,8 @@ spec: - rule: | has(self.blockDeviceSelector) || (has(self.fileDevices) && size(self.fileDevices) > 0) message: "At least one of 'blockDeviceSelector' or 'fileDevices' must be specified." + - rule: '!has(oldSelf.fileDevices) || oldSelf.fileDevices.all(o, has(self.fileDevices) && self.fileDevices.exists(n, n.directory == o.directory && n.size == o.size))' + message: 'Existing fileDevices entries are immutable. Add a new entry instead of editing directory or size.' required: - type - actualVGNameOnTheNode @@ -124,6 +126,7 @@ spec: message: "The actualVGNameOnTheNode field is immutable." fileDevices: type: array + maxItems: 32 x-kubernetes-list-type: map x-kubernetes-list-map-keys: - directory @@ -147,22 +150,18 @@ spec: properties: directory: type: string + maxLength: 4096 description: | Absolute directory on the node's filesystem where the backing file will be created. The directory must already exist and be writable. - x-kubernetes-validations: - - rule: self == oldSelf - message: "fileDevices[].directory is immutable. Add a new entry instead of editing an existing one." size: type: string + maxLength: 32 pattern: '^[0-9]+(\.[0-9]+)?(E|P|T|G|M|k|Ei|Pi|Ti|Gi|Mi|Ki)?$' description: | Size of the preallocated backing file. Must be at least 1Gi. - x-kubernetes-validations: - - rule: self == oldSelf - message: "fileDevices[].size is immutable. Add a new entry instead of resizing." thinPools: type: array description: | diff --git a/crds/lvmvolumegroupset.yaml b/crds/lvmvolumegroupset.yaml index 994f4550b..c388a57af 100644 --- a/crds/lvmvolumegroupset.yaml +++ b/crds/lvmvolumegroupset.yaml @@ -90,6 +90,8 @@ spec: - rule: | has(self.blockDeviceSelector) || (has(self.fileDevices) && size(self.fileDevices) > 0) message: "At least one of 'blockDeviceSelector' or 'fileDevices' must be specified." + - rule: '!has(oldSelf.fileDevices) || oldSelf.fileDevices.all(o, has(self.fileDevices) && self.fileDevices.exists(n, n.directory == o.directory && n.size == o.size))' + message: 'Existing fileDevices entries are immutable. Add a new entry instead of editing directory or size.' required: - type - actualVGNameOnTheNode @@ -167,6 +169,7 @@ spec: message: "The actualVGNameOnTheNode field is immutable." fileDevices: type: array + maxItems: 32 x-kubernetes-list-type: map x-kubernetes-list-map-keys: - directory @@ -189,20 +192,16 @@ spec: properties: directory: type: string + maxLength: 4096 description: | Absolute directory on the node's filesystem where the backing file will be created. - x-kubernetes-validations: - - rule: self == oldSelf - message: "fileDevices[].directory is immutable. Add a new entry instead of editing an existing one." size: type: string + maxLength: 32 pattern: '^[0-9]+(\.[0-9]+)?(E|P|T|G|M|k|Ei|Pi|Ti|Gi|Mi|Ki)?$' description: | Size of the preallocated backing file. Must be at least 1Gi. - x-kubernetes-validations: - - rule: self == oldSelf - message: "fileDevices[].size is immutable. Add a new entry instead of resizing." thinPools: type: array description: | From 654a121393a2973871cc508c24e0ba4691342b0c Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Wed, 24 Jun 2026 14:18:52 +0300 Subject: [PATCH 12/32] =?UTF-8?q?fix(agent):=20=D0=B4=D0=BE=D0=B1=D0=B0?= =?UTF-8?q?=D0=B2=D0=B8=D1=82=D1=8C=20=D0=B8=D0=BC=D0=BF=D0=BE=D1=80=D1=82?= =?UTF-8?q?=20time=20=D0=B2=20main.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reattachFileDevicesAtStartup использует time.Duration; без импорта падает сборка agent-golang-artifact. Agents: Composer 2.5 (ведущий, авто-режим) --- images/agent/cmd/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/images/agent/cmd/main.go b/images/agent/cmd/main.go index 0887faeb7..f0c292870 100644 --- a/images/agent/cmd/main.go +++ b/images/agent/cmd/main.go @@ -23,6 +23,7 @@ import ( "os/signal" goruntime "runtime" "syscall" + "time" v1 "k8s.io/api/core/v1" sv1 "k8s.io/api/storage/v1" From 8b1648ad2f85b5f2863ceb254846128f7125717f Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Wed, 24 Jun 2026 14:32:37 +0300 Subject: [PATCH 13/32] changes in helm lib Signed-off-by: v.oleynikov --- charts/deckhouse_lib_helm-1.72.0.tgz | Bin 0 -> 45558 bytes charts/helm_lib/Chart.yaml | 5 - charts/helm_lib/LICENSE | 201 -- charts/helm_lib/README.md | 1873 ----------------- .../templates/_admission_client_ca.tpl | 21 - .../templates/_api_version_and_kind.tpl | 36 - .../helm_lib/templates/_application_image.tpl | 23 - .../_application_security_context.tpl | 263 --- .../templates/_capi_controller_manager.tpl | 249 --- .../templates/_cloud_controller_manager.tpl | 280 --- .../templates/_cloud_data_discoverer.tpl | 311 --- .../_cloud_provider_user_authz_roles.tpl | 83 - charts/helm_lib/templates/_csi_controller.tpl | 986 --------- charts/helm_lib/templates/_csi_node.tpl | 397 ---- charts/helm_lib/templates/_dns_policy.tpl | 12 - .../templates/_enable_ds_eviction.tpl | 6 - charts/helm_lib/templates/_envs_for_proxy.tpl | 43 - .../helm_lib/templates/_high_availability.tpl | 39 - .../helm_lib/templates/_kube_rbac_proxy.tpl | 21 - .../helm_lib/templates/_module_controller.tpl | 532 ----- .../templates/_module_documentation_uri.tpl | 15 - .../templates/_module_ephemeral_storage.tpl | 15 - charts/helm_lib/templates/_module_gateway.tpl | 22 - .../_module_generate_common_name.tpl | 13 - charts/helm_lib/templates/_module_https.tpl | 201 -- charts/helm_lib/templates/_module_image.tpl | 136 -- .../templates/_module_ingress_class.tpl | 13 - .../templates/_module_ingress_snippets.tpl | 11 - .../templates/_module_init_container.tpl | 101 - charts/helm_lib/templates/_module_labels.tpl | 15 - charts/helm_lib/templates/_module_name.tpl | 23 - .../templates/_module_public_domain.tpl | 11 - .../templates/_module_security_context.tpl | 263 --- .../templates/_module_storage_class.tpl | 36 - .../_monitoring_grafana_dashboards.tpl | 101 - .../_monitoring_prometheus_rules.tpl | 135 -- charts/helm_lib/templates/_node_affinity.tpl | 262 --- .../templates/_pod_disruption_budget.tpl | 6 - charts/helm_lib/templates/_priority_class.tpl | 7 - .../templates/_resources_management.tpl | 160 -- .../templates/_spec_for_high_availability.tpl | 179 -- 41 files changed, 7106 deletions(-) create mode 100644 charts/deckhouse_lib_helm-1.72.0.tgz delete mode 100644 charts/helm_lib/Chart.yaml delete mode 100644 charts/helm_lib/LICENSE delete mode 100644 charts/helm_lib/README.md delete mode 100644 charts/helm_lib/templates/_admission_client_ca.tpl delete mode 100644 charts/helm_lib/templates/_api_version_and_kind.tpl delete mode 100644 charts/helm_lib/templates/_application_image.tpl delete mode 100644 charts/helm_lib/templates/_application_security_context.tpl delete mode 100644 charts/helm_lib/templates/_capi_controller_manager.tpl delete mode 100644 charts/helm_lib/templates/_cloud_controller_manager.tpl delete mode 100644 charts/helm_lib/templates/_cloud_data_discoverer.tpl delete mode 100644 charts/helm_lib/templates/_cloud_provider_user_authz_roles.tpl delete mode 100644 charts/helm_lib/templates/_csi_controller.tpl delete mode 100644 charts/helm_lib/templates/_csi_node.tpl delete mode 100644 charts/helm_lib/templates/_dns_policy.tpl delete mode 100644 charts/helm_lib/templates/_enable_ds_eviction.tpl delete mode 100644 charts/helm_lib/templates/_envs_for_proxy.tpl delete mode 100644 charts/helm_lib/templates/_high_availability.tpl delete mode 100644 charts/helm_lib/templates/_kube_rbac_proxy.tpl delete mode 100644 charts/helm_lib/templates/_module_controller.tpl delete mode 100644 charts/helm_lib/templates/_module_documentation_uri.tpl delete mode 100644 charts/helm_lib/templates/_module_ephemeral_storage.tpl delete mode 100644 charts/helm_lib/templates/_module_gateway.tpl delete mode 100644 charts/helm_lib/templates/_module_generate_common_name.tpl delete mode 100644 charts/helm_lib/templates/_module_https.tpl delete mode 100644 charts/helm_lib/templates/_module_image.tpl delete mode 100644 charts/helm_lib/templates/_module_ingress_class.tpl delete mode 100644 charts/helm_lib/templates/_module_ingress_snippets.tpl delete mode 100644 charts/helm_lib/templates/_module_init_container.tpl delete mode 100644 charts/helm_lib/templates/_module_labels.tpl delete mode 100644 charts/helm_lib/templates/_module_name.tpl delete mode 100644 charts/helm_lib/templates/_module_public_domain.tpl delete mode 100644 charts/helm_lib/templates/_module_security_context.tpl delete mode 100644 charts/helm_lib/templates/_module_storage_class.tpl delete mode 100644 charts/helm_lib/templates/_monitoring_grafana_dashboards.tpl delete mode 100644 charts/helm_lib/templates/_monitoring_prometheus_rules.tpl delete mode 100644 charts/helm_lib/templates/_node_affinity.tpl delete mode 100644 charts/helm_lib/templates/_pod_disruption_budget.tpl delete mode 100644 charts/helm_lib/templates/_priority_class.tpl delete mode 100644 charts/helm_lib/templates/_resources_management.tpl delete mode 100644 charts/helm_lib/templates/_spec_for_high_availability.tpl diff --git a/charts/deckhouse_lib_helm-1.72.0.tgz b/charts/deckhouse_lib_helm-1.72.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..bf178e500ce1f149bb181d920ba9993207c6a9ed GIT binary patch literal 45558 zcmV)oK%BoHiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMZ%avV33C_KOQ6xgyHCVkk=CPhkK?r>r$lD2s)QT#~Rb0%gT zJzy2O8^h{C4Ny&~M^?nX!#Pj5Px1wBRk+oqFTBXZiW!St1tb!QL?Qv?B|-7cgv=Nk zrucd|LFsh+cmiqBpTlYThxI&z!C>%mZ%_O?7!2Hh2fI7Ff7sdI9qbQw2Yb8we;DlS zy?VL#2QXMu`;E_x6_Ea6uyR{&=f02!=J*#x87A2QxZQo0AQn@c7vlGiD4l{?fl~$w zG|f|3AdsLD&M^PZz=%+AqWK3*Nis_j>p#oj6deGcv7Qz4936lZUsFiu&u(?IJN;L? z{lT+`ro_MI6PW*ctl9PuCR5D#C=KHjqpTRluwUeh%7vEszw>f;V9)>EomV@9r}=*$ z&!CFgnZYqS0G~bqoWK zLBu5h%VCV7P4M|M_;wrgK7W3unz}?;f+zzq$wqiQg*o_uiwP)O0jd=sTLH&Mz-R@0 znBaHEqX+M`FpNo7ppS*@ zK>rt*&Jf!IeX)RU0aV0J*UvRwz=sKEoO*KHjf0-Po2 zBj|$xs6Lk@Hw*0W0=-`EnI+|Sp55RqIRM9^>9;U{Hbn(YU;z)F0U)J!NtT`@M=yr` zm=@~0Xh?92(XPM+W*I4Z$3IL8r0dd9vS3yN+A>iXJ`}%jId9!fTW=&8wQKo8UK)ajInn*yTOZ)R5iQxbhCG z8fvurT{+tTH6DPJy4eLKks|~x5)n-57jz7 zgjq7=jI;Ox&?5g|?7w{B$p78JtCvsm|301$@}Cd-5Pig~V1|tUBY)lF)BjH)DNj)` zqZtDQogsinV$!l4#dw5K0w`i+Mq>mx5U>$LhI%+-C}X(5w`db!1}MrYVkqOX5N85F zoX%K*Xrn;4u=T~Kcq|#(`}`R!E!Y)w(I}^dEqx8pKk<%=5;>t<{@D=l2%vv}jTB{- zggJ&eyv8XmFk=0qi!&v`*``EVNu(YaL7aj}DC-n)=nP~8#76DNXo<@i{R>c-jS+Yr z!<-}D+A=U&#Un6*>}oa|;g4V=B_9w48#%>UF#^%{*DNxqvbm{C@TN937yw3OmL=+M z4)mHLcw@Grq@wl1?8UPV{5wX4CCIMlf=L$V<#UwGV*iX30H&$<#Q>xTuvyLr5+$3Byk6V0C%$jwd(keT zU<3Wb>_hYmOmPAWoQ+Fm@`9u|o=2PA%f7~0g0pe7X(5Y8?H%m@2pNb<&4HpZW(5`6 zRR88Xp+Md4Y}X?x8f>Xv&8(E6^3{e0blLyAOkl2@;zj$?DT>K;0B3w1oWcS`YDUkua~R*iG5XI0j}a@P z<{i6_PWvudf0uYzi`8$Spm=(erXc#S$b==zo(PPp8l|{As?g|PNc8)%Fw@PLs|{b{ zEr{huTO^%{nqM=nT=C^hQ$*I?^pqiutS04a$dIMWh48#oh5LNnWVj%7?jvhKw6Tpe z0HODAWu4bxsEBIuQzLw?bcIIrL?I6!6M}<|s;Ky@9UD#Tui5t3>_3IBYW3D?%N?}` z+^pZh(_Q{E6wfFw=0iPnotJ`U`M>*e=f#dA|6jb?eeop!@8ele{^um|7-2|f*$}ef zj3GLVXRIL8YK7Pt3qYRE@!4>CQk2eNjjS%w!Udv+sVf>#sG}&Z_vv{{H})C zXkcxFXu!PpVcXk(vF{F>cy}+ObZ@iMAo2Z9(59|Fp20Tf=&GY1%S%i|yAmd6Y)BC+ zD2@x1G;%=8@E_}@TbUqFXADenhNmzU_v4tNgzuCJoQ(xX#AKSMa|x)JAaGe?2&N=K zfXJI1&T$$ma8cVFLgP~3Oq$8Xar|zVezP@fy*GaVb;%QC)+=q zoyexZ2BClwQfzL4i>oVhfNkb`CvNFs3U1Jx9XKERlmGz!=fD3ixF#gs0u;fdN3wJ- zK!fkEz(ykR6EiIVrSv4G` z=p(*PQ3N&;92X#>BvNQZe8BXkE;>JbKRh~ldv+dy=;|+5!}n)zPoqr`&2SR&z1|Lp z>>ib1Op-@_pOzIGwH?ubpHl)34&`OEekU(5e?oK6mt8gPwI~6Hg2~;pd-2s7u13Hv zA&f>GFozze1REDq=im@%#t>t&siss}O3V(Ul$`r=(o{q-CqOfKtZ_35Wr8?#=9G(n zaP>wod6%VgUg3M3A~t6Qno8RuN|aGit#tnMNu5=i%~l=o-UEq)=_HhU% zL$2hC**+YVtCig3GL+(M_Hm8$=M92NtF9UpGK}Pv7sl1;8{W3qzQF>E6>Pqa-B`yi zuU+ufXa#Ej)V^I~NDxGBV$@2WqFa<6fN$8~8)M3T!}54od~@GCb1!UimmKiu2K{aZ zu+kUTGh%d>WjGrfdgxj&`|JOwQ>jnSt76w z3b*xZ;gw1s1vN9hzaQZ{4Rz|o_ugV3et*MTHFMXZt!|L-RN8gx-ns>!H+C&-ao_5V z$%I3`ALcsSLZiH}as$)U;(f6V&ke7ryU}WDdiuy#y0+73eMs@(JR*FCV3I_jv(ucMLTWL-bO9YuBE>TfpbKCthSw*UK*q*%ML?)wP zryhnM%|kNF_A+!(wnmP9M0|OV%=h?M?Dxq-t6IL@g<#tf%w-zNot@k*Pd!cxx@C;z z>h5drLq0z6sp6vR)t%WdJaYSl70L7ks*=mG=0h7cKRunEX8Zpb=2);kC25N2a0)Xx zMsz_huonCOSFgPIFS{=XPxk-!@|d!)X0(UeTVEOP#rf<2?CwvWP0^Il`2pA;yv6oK ze;24z_)*^dh{G1Yh5f;*Eu{DsWr(q1PRX^hy_gh5{sWT7F0hy!fb9u_X)*aHe?e$* z0KOZ1xARQIxPs{kO5ywp#UxAE0od7pmLrPAEvJ05J6IemF0pYtTD;i23sxRdI{~Cx zbcFgb%J2wTfghgBh&7FYeWp{AfwHbxsc~Jg;B(9 z=c;u6l|7tEPh&${ajoF~7Z}abR5+b&Xl-Q^{Lg>?zbd|k@Q)KEL=YQkm{)b<;^g%f z_~qhA>g?C^(q2bL3YbEifl?h2kX*d~R~3?{>fM%xbO8RAeAhDr*U0dG%8l-Cb@&Q+ zepAx1(dn&x8AjO$!FlaMal?6lcDG#rDWR#5AN`L#g3sYW)5!e|RBW%HtzpB{QvFkLu@Ooa|BS2_-Ks3k^Vs7Q*W+-IsuTqiC58o_MdGih0_k|hd4h#dG1~GG(Ynh8*AZ&Ol~x+SCtH=-GIWN|TcAdl*#e$2qa~eg90(1? zSr5C)`BKkWs0&i<>Cywg13QCiJyUJ3)B~6#SmJy{$8}twhZ=n>DC7gqmVken-F5(#&@`$Hfqu8#}D3Gd#ei2?wZ~0Qt zt(jZ1!r~qKMR`qiX^Klj}8F3B6AyVOjV&lXUr=BO(KweaO)y-){BI<8|&lSft1 z#!l@TK^~WCJWsAq#oCj!ssVb93aRo`>^E>W4z}+*;2E!%2pa^pjp4M z+Z;A_C$#2<_b&}Sy3OIwc>)XcmLzq`;#qc)!Wg}ymq-Q?)e9n5M`xn8I3*22e+xez z$=!I9a=bsNRgOR6>1PzBO$(BQ z32voG81z&-hUQ4I9rnn*mOwO1g6be!b7o`}}s?`1bj!t|U7KFjw)Ru6VT zelkbJ-PJDLst+oNfmh1($|?7`nwZ7`+eUf()8V|YMu8@v zhO%ES3FWJLdFpHMMs52`1a?9Y47FTWtzI<s#b3!5YV-2U>SO~0jql3WiHoT zm33X5TlDK{Ic+NDQcT}0;gQg)O>%G41j(0rdh%9=$OpfP7>CiV+6wh@lJsD?8S2F} z5fLT51O~bc-zQaZZDme_;Jzx5-{8|HLsJlxYR+r$Y4Mx_+KV3MxuKS9dZ`9jj$)2j z0qGbOmuRHy{LLQcIn%LwWk0LqX7RGFQ*zTM2ULQcl}s$l+uZ&e_{WSCsgpah0Vo!1jQXv#qDY;TlkVmd{Btgjs=y za3qq#%}dT+c!IsOdk|<#9U~0IEq4SqidmkbstNxDVj=L>wO{~xXv4Zt1I7+D0vj>O zVpw=dh{C(vh1yY)ZjtDF0WoAW({vmnXI~sNML?eCm!X;@|6J4UvQc`JJGVNebu$XOgD5hLn*6UL_4t1GVzwgJP%^4#E-!k=Li1x(PwEE=^fH=)Cnye+Utzvpq4nsf9 zZp|VnyRFekS8hq3iWr5&z3e(#vyQk90!Y{=V{4(MeXbR1)Hx8lvk|mTb&Rs+CGk3_ zT<&U2VA_PteAhBPQ9eOaL}5DQiXa@LVM@kK#^n&#wBi`pKy25iowCWR_Urw(8@yUu6;>)Y8`B}-A*n=KtisX;j%3CIDI-ad?B|5q| z12`K|$O<}(iy1|RrVq6Hf&uTnz87t+rSITauyx^^lodFeAr}Xn=Sfq#DT z|G0<8GqbEw_w{?WOC<=j7Z279k|o=N$20Csj?^#}bypTA)- zUvUq>kMG}KTs8Q8)KjsO}v#2u@x+~X^pskjm ze$PLJifTPF+B`^Vx+VKu<4a<=oP6nSCI7m<9go{i{;$yooF*}(N%X7LEU&PrReY;| zL%?*j!M&kT9B74%&Qb)%C{tK1>=F?v=jbTt81V(T%{+wf&_lWEti$a>EZAG_dc9>Y z*PkDOufp$F z-D=-5pwe&4+8Udtj&!l?{~db`JoyYf`3&5(&pKT_ZIcG z2Gwj-?;FhfCi91e6=qf98fQrlCJ9B19hw$Lb}`Wl)XNDi4z-n%Nr-k6{CYF$HY;G@ zTiIJS%{9x1I`TVXuL&vm+AFegI5_EH{^{q}rO3gA z`cSOadc85e?@3YbOR*{i0v;80J{9Zss@MxM)dTodbl80das3zat$1>xP)~~!Mco~y z>o!qSu7jh*bUMp$3=33cayhy<6RnHGQk;!_FABxcTWa?5GAV%xxP~drVnl5dq%C0= z2~s>l@jOnEa^sLzvz+-TnH3xc$xC}f`yiyeS%d6HWg%nS7szwe>O^7GFlTrH3bQc+ z&*kJ1cdHFkOhc5V`=x=7vylv;b|??T>$Y~*Pw$7|NFS#7ZcY1Qo>vmUWJya~)_AYN zp@w0SX!QM4i8r>=x`k< z@o1#!!hF<;lZ;EocUd}jnA1 z!vv>`1^o_#>WlNTr~EL)vARvPe(q6DH>Z!@6pmIIpY0o2zheJNVpbIP<^(L7Q0+T_ z-%l`Wf@TunA{?|aQ3HG|GyV6 zUhcX1pLX|OJ^BCL$MebaTysWy)$C6}m}|`a6okKY=BL4;ettf+mU8mz)^sP2KA+Zi z?ap@s3kXii7QfI8P7Q8)oTR;qq|5v0J-OoDuPdIuP;EAiuUv3 zS8ChdlzjM|IaE4{hd$t9(hFxFTEWqBP3eaQkKgO93CpK6LTzb;{uNxg9xRp7T2l+% zk2m7)%z}@~!&4`Bp8PA!R`itB`+H zn|+sIZxwQ_`qaU(>O0d+#2?Mu91d@0*JwzuVJz(o4#9r1a9tfaw5euiwYw9CcSNL_xP8K9l1j?el+Wh6*GW3UxxzLmyGxD`i%_ z&qFhml#E$Bl$x#wJ)G{yooRW8rY9$(hjB8p-F^b@mKht$b!2$0ojq_i9v(9_W?FaPwX+^2HqPOswym zH&EugKplKH_-@DQ;L>SEzH>?SU|B|<@_u~Tc|S;&&V`#wCtsycUuxbDEpE7?(c>GC zp4fmqw76+#aylj1P?YqS8{Uh{YZzY$C$r6Zam70xmA;maV0yms)6cI@hnKI9j)xbQ z@BYtUhHuWU-k+Wik4{c5Pp_^vxxgAkF0k9fUGH5=#sy(Tf+!VEpihAab*)WBx9y#ojqR3fqiugFOIuU5P zL)KHpf?n&ySntBV{{7hxe*3-!IQl5qinro#A^0$rKe6H1k=|?Nhx%RO=>c*oRh+8UQhW><{5- zV%v^Z{jGBkz0*dY*1CXsZn*|6c82yCGGdv>Ov{@h^UdFsWLR!xI~yZkkmRkbRT?C( zWRHB`cESe!rS)yV0#KZ8pI4T?wh6BMEK^05WjGb;Z3-)HxJ2S03TLhMXNGT(4C~FC zP?lR1eW18-jaWLGd{|@O!RVvKbYp=)x?Kz}T;eZcEn)QRz#GM&<%C-(c;{!1nSbSH zd=1|C>4Wyg#GG`|BD80$ck-607dWL?o#qS?6N(7y$7E_9+AFqiKX5iyRYNaGzUx(O ztaj>Lbx7DH;4JEX68(2_Ely#D<{?}*lD87w(=5peE_aAMwM>R|RkgC(FZJ>%DyVnY z2vS7FL6JP8tVFgXoiR4Tc)B`@Me_q-zA92CO`9qZIFmbxs~epEf+!x%Ehq8`$p}^F z-sXE2Q<$Uoc#6oZsFC>z1Wss#7 z1QdstK^OC7uT@GZaODFKg0o@{kp|k4sMP|Y9jHDnpCj@2{B-gE)OwaYpywOFe-0^0 z5$os)+U);1*x%dTcl{(RzTw?y^m}yy^I7QBpE|v;bWnVrVUJ_fD5?a zWvqZ%jKpo>r8d5HS~<~^wk)pvnM!MI(fpV~;Zda=yo5gQw;015Up9N}mAivbrd+E{ z*0IrQiQ!{*DRj^-P6={P0`r{nQR&Db#@!%pQ+p*wzy`}U z?QL4IIZGRvtAgf>VI^7x8qg@HEU7p!0-YEKQl3A5gIv)`Ol?j4&w{!Fw#(nE}*E1e8c--34dG;lhe77U*%8({9qRxSsx8;s7`R<^q zbs!9-Oiug!S$tk@7wB#6#C$$@)+QX?B-k_OAX93ZVLp=)k`TRCCC8``|8gwE zA0N2lT3Wx^jK#;oMZ^}eI&NkEV^0o=i`%CvI*$iiVcrkKeCmd=aR%sa5GuwN1*(04 zeP;OZtY%J$1z1jdML<>WOW?iIq>Y12k@aUxQofzSBzvYp9mSJp@_s1cl^!vJ261g* zOlZ6BSsHbuNx3jaex97(H8NAwm%Kv{aPnWwKhNTsQ z41=od8b&ZpuVH*s?fRlWQuv`V_Uv&P`=VHDQ4{X20qCc)9AIb)vjWFl*z)Ds zfhg#7g2#vnDnxdoKWQQR8nQB9VBoSrjtD#BkXT<=>Wj;;5*Vi@o%5{~SJOLl0r+;i zvPd7*VGzFVQ1E*d{K!1Sw7Yi*{!VZPqU~~lw?0RzlTjG#rUh7)_`WP4>C39o>O{#z z)rvfL5%^7SW~CjI*?K&q6lKL*oDszzh0i#B`Pr;j6?qH)-dI(9HmYua3(g8e`9zU3 z)x;nnA2L4dg9EP10Wumf%>V-tF4ic##rNGAh+!JfWY`A&R_T@C&<5PouoR0}T8R2| za0q5ufm5)-yRre3#G%iQdA)BK6j<}ZG5vdPVWr5l!DiPEY=y+8S3h<|gJivm^IYi> zeNX65-(CH18u`OyVdJ-zlv7)AUM+JvXjL2*MV{U7s08fUE?AU143!I7IMfBPEjjFa zCB8+%YCa{M09#_`5}HhOARsuA5$<(42CNeTHFCukhe5h6BmgB9qIMSb$jLc1FrxgEj<@hN z`%e*X!IJlWr6zM3t%1Oz;LK%(JJ3N7cZ{HiyZc7aLpSoE*rW<3UJVXK4F7XQ7;K)! ztAfa2&L*TNR)nL?>x6s!N|21)jV)lT;FMj%G+LXg1>!RX)z~nkRt5H1)5&pF+qAB6 z!j`uB8?dPA*PTrtw zTuj1Mjqm=b1dw+>)ncwF9{}NesfA$BJOCx|wH8v$a2199N2p_BlzbofT)}eERhZ|b zW<5W!N=dR#6*@IoR@S-118vx4LSU|xgslT%d~e1=hs9~aqH^o21!IJIX@X=KX14&V z0=~3EG(Fhr!CoA-Lc2I>0dhfTJ5sVx9iS@4P8f#i#~R>$BWF(S1Te}NGfLRNHNm|& zJ86W;zZ>ubmzw}B9}i9h6Po|A0`>3T4Zf>`qlER9w4bbn=oG79@7DoVp>s}93g>Om zLuJEFU@wwV88R;5r>u^oSV5-ehBLc03jWs+XBlq&Vb(GW7&FpsY1OdtTI#@K&zJqd)2w3o9jz!TKICUX)YPSx9Lz>9_rU^(@O_yBQAtK4>JH8 zN<^vaEqeh``BugZpMES5<2D!WaloN=n33TK;?(P|i@e>jO53^{0$)`Pv=P}ltynKM zZ%oiFc!*d`?kuzqJ-_r|Wlh-;))30bJ^NTHep#eqHA(yYHAXOt&a#UX#^@c@v5uk@ z1DBb{kJ_D}E?ikj*P0{CfVX2y6f>DSVlXXVSSN8|gtcVvjBV(Y_AXG>6VN$;Qc(Sl zsB)eA5jfzzt8O^gF}zpy!B}mlWUM`QEtiZrQ&7rVZGfAd59zTVsHE2LTrEf0^wXba7{C zKQvA@Z!r#qSbX%fa!BRnAr74$HsDeP&lr+yDC2w&b%ezs%#xuF*eFt|PhfCH?SjA! zbQSk;r#)(;#qr^&kh_l|xMNA^2S|IZG0?5rX43j?SzBY%+!fC}1CIcqU}FNAI>%Su z_oX*iO~QhWZb#8(@I616r9xoA#?3y-^75pl7f(=p!)8YKmW+op_^# z{?P!}JckBeEX(KSS)dab+UrbRK+|sFOhcytZ+N(m{Va1?>=lwBgSY*ULSl|%7aV}|uY)EEWUh@Md&JJmQB3GYtU5y((U zOG=a(NW@SalWBfI$q4fT{Ox9k$g18!p(mdFV5?EDr{aGcIv}(O!spMCUH(MUQl%7C3gCTXz3@|xyFdvf#C z)4Q2+6zazHgD_0#6%&eJ!rWP`%~s4=Y(?eNqRtP#sX||F#|-oT_Zf+AylNNkPKJEo z-Fkk=&bok|jZq;=>t^wj*NzW&#^*fMS9j_y6MV0}w_Oh`W4)y&DkZYE$%;I9rv8sNO zYRgD9W|HzrnmtLgR%zDFD4~NeYmjk4(*t9S#OE(S!kMOV-2|L7haZ;=+0n(DF2dam zI?Z9-AZVm$rp!TmSg*urLEhdE8jf4ydRF(M> z#E@$@ySTCya6NlcxK`dq2HeL?ss=bR;63Y}_vzB!S$P{6yJ2U-Jn4b!(F1=^GPtv~ zvv!Mu}I?T(aB0uR=h=U)A zs1C5XA+`Oe6b&N#k!*cK4)*eC+WtDV7TjkpmH#f3=}>0bsT2H0QSAFH2kM19n=LSf zwK4`EcuVKARQq0?DS>D1xdd~q5Mh3AR`pO49v2djC{xdMUasU++GQJ_l@T)e_U~`o#WepLNH7&9aq}wdq%rl?QtO-Ux(2?t(j7Ip>=x6!reEfl zHJsTzozARj5$x=fQ?mkw2hYup9vCiFq{*_?Xvz#Bw?GE=PgN93>$7myuLIP3a7HkQ zW~1Gip>5IuKPzQ}Ktj{Jn4e&J06v-HclVNLRXG%rLN%SSY11x&77;!Rm}_fY@d12Q zTt79-RpoXEruhVFpS7znW^47ul8z0ZO8=IQ>a$}lz0qO6xCu5gRO4Hq!pv-10(og{ z1Q0!}$FkeDZo1GOq0;F~XDaC$`HS2_8+OpH0~G)XVllI_tf^^H33C&>A#Bfl{dl1vN%3IcLOM}$^!rlq%zNZBMaVG7rnVgd`0A(SWyVF2K} z?@BIjDVdA2F-XuYjxC6MHc4>@QanQOJWi3`pvhjuFo)MT#RaY;qDf2B)htO&iQfZP zf4LeSoxD9eUzXG*fC`QI3iJ+wGe~ZXM>F-=ES>nxnGZ*oL5~;Mubb$FM+hn0zh<`p8)}9~48hsy%{5l=9KWS$exkLG~`5%384wj%0W~fgTiq_yKsd_?frN@ zflN>t784_!qZ7ZaqfTz@@@`2g_I8~#dBfXCobC(PVpq=(AbKbdH0zw%TBS<}ruc5{ z(TaKAs+{zI#(Ndk6a7}xjkmfY*BRg>GfBSV!kB+n+*0atLbRi#!$MGIDOE`!9T9@3 zc`EA2G-uV6g~gE_q>u|{oI*(0O+Q){jtK&D&sSI zBg0orUkSpB#@X zcCfuY#>HfI-H*w1yNWv4iy7WVDnw$>fVO>|lIv~h`Mxbf${5wQMV81poTf$`9y2Xu z=Zup6P$b_Sc$7bg89u0}cM!t(Zbv7O84~<8ihi|{DT2R7Iqx+qP*$h~O%##262V`i zF)E@h5OEavzao(*|3k(9+X9hH1{wXjcvEo-@w8)mG7e#vR%CdTE?_-iN=~WK4uF>5 zMRbWygzA7Ux<#3bZ3DFx9kxJ}*YrW{8TxIVQNyi@H}qy*O^8!budmH5wKNbjoRMT* zL(v`7SKz2zU}Yz(Z5!i@BH>H9Qu{A6g84Ph5}b{HfAX2luK$kWLWt<#n!ngePf3cDwUxiPt$`AH z{2Qom4B)CuYB!u7RQplu>sLT%?Ic5hjKqW<5w3yry`W6=&lTY6+=oygY`>YUIzxsM zK(c3Ffe(CY^SOxR`QzVw({8?l{aYWLmOvpzAVE1ru{b-#85qG^LODinL}T6z4y;2O zT5ztGE0LbGYSp(^B(+L;lD0fksfOKeUs7cajei9$xMUMogXQtkm6TY=Dk6rUm?@*h z6(b)rK7#q|I>l_l#m-lphI;#=<7oh#jR>8p{m9}=mR2JqIJ~&Z|#(YxnvQ;EU zaEl?==nGh&(JZ|}1=uL*&2(bc3zBS#Q@1K2CfG1b@WCXBC?axoUdnim99%)b2+Q|I z_00~Po|7a%agS?Zo-@C)jwl(_Fn@JK;2IXMFF+FB^-5+fVe4D1jP2*Muc{Cn*4xE* zq#Di_vK=)dk9@g%(e^#j9d0A;LKj{~pS*UrrMbP`eEyhp@B3lKN3VDrWOy7}w}H@O zP`MveyS%pr{;?4KQLOF@PKjy}fA&=wgT{Qf_`0s3d?Cwv6?f;$UDa#Wy)_;ycYz(; zD=Y;mh-LGleD`O^k73!r7iN9*tA0g=$Fb;F_&lxo_qFCf5}H4XC4aH`VC~w?fG8_& z=M7LTOFyToS(@Y8b6!3M)xeGx^9a;~bzgukaU;C88_wk~hSz$hT2ZV0c%hPf>%CCc zm<2VQN2-b3+uV5c+K5ew$DxtfI6Y}4_obCQBDy~c&1CVpVHg?y;dlI$4whoT`N(hU zq-Shf*g$3^LH#06mv3V*7z|$S?TLQ}gMst!i&umF7k}8<-yQ4^b_aXA`+pehyxQN{ z{{tATv!iv-j1`dnVX$&rZs)#`$25HvDT<`A(@-Ugf|djR=g-eXmdl;p=`)?(a(DkN ze&#w4TM(B4(7~ctz@EYkj}R*i>7>%#Z+5ZobzxMVAUf25$jP0ckDw0*hReTd#QeNt z)GnQkjr9BFJ0rh87oIXBQ&pG$F7tuw%GJx?Dhrgz|7sYhm@ws_e*gcSoSQ(6w;`y; z=NhnfdL$FH^Tp`0S--M#KPL(L<>JT$;eV{FC}Ppn)~w>&1geTaI<-}+cswC=D+i$( zlugHsS^&yxq6LcSTM+>D#tcToj;BdIJillK)Q_fPfT;r~ z0HFf)0E1yjf~0eUgZKeCtcsy{BA@DW`URWx5(CO{lB9c6x;UDn2LUsIe6RqD(tfYPStEVx)0m)5{eu8)*1gR`-*ot68yUlcGt z%}P*@F`>3l)B^ikmHW1Ubc~8JdZyX^NTmsv@mN(tafqfQyF!I{SJy>(hM;o8ijwlp zTausy5S?WgDU8uOs#9V|#?ggYs|=MWUSd=mjL_1xYSpv_lF<7(`Z*nYasu?qZARyG z{jHmnWi$8h`jYTCl!jqIsm!Cy}r;P9N z`64pD`zqE`uJ?9@wIT3Zjn=yM{O%8yU|VO0uORdGj=Z;^JhV8OQJjsh;t5J-Tuhve zGg7`j{fOe3;D5_|K6AII_UE1yDyJWFiloDa8#TM99Oel<0B4!=5r7*sKM;-Bvy+Q6 zuk5W8VWVo|m~-K&j_6od-~!Pp%urU;6_A>x_tgyuSFeXU@f#%Q2B?83{&ng|;GDp% z)L0|HUORhXI8AflUnkYRP|!N%*cevL1e6eCQ7kS?&Z`XHr8?7cwrCKy z4C;2^gd1uF5?YD|3u%K5NBA1KnjrhZSoklD0_bC<N_mhso#aEE;af+l8;}x?PB6}3o5yC7h@D!cM!)c>HC@!yj z5=wGk?LT_+#uNMD`ygjvoT$E`xW2*pV=Ro-RdCHJ7#1llnMcYZOKc12Odp{+HgdeQ6X30^5mayVAfAbO2uU}wP-zOJy; z5+Or$H&jZ{tv8R`F6S*aYAD>y&Ow4dbpD5jTAUVMi)w{WD6dA-{d@O}2~}%Pi9nW0 z1Y#QuskF1M5s|aKv%jC0HyCa8nu8!3dP7Zil3Lx;jq3_3vF+}P)Q(82T=Q*H2n%)O zM5iNhS`SIq?hSWtCLmH5=w9=tuO5ka^jO?udM9h1sBS>Ul-qtc8 `yeej?r;IAg zW>l%!6+A>P5k8axVvx(@F(@?l_oy|gsADoT1XPN#}OPehq167H4*C~l_ z6#J`u3uS7HY^<_fr1L7rfQXuYgJ_0Q8QuxA8zYAAgQjFyhSFwIhUWAk29-%x(xN>l z%c$HlmZKuEtKFR59cuzqw$g=aTs=~P8cd-K)WxZ!Q*W9XDH3tlCA+u<6x*qsDUvR3 zatn$?kUYRpF~bR%pfu--2%%zv}CB5@ew3dS%5n^ z(Sir-nr}n-R@`JmW*(J5<^yZa+qqc8(20ZwolX}XZ(5;c1i}J|g^ZcqAcD)uQ05R4 z)b^yWDOLKO4*SVV(GtGysgkppH%7H-Sr)lLda!gX%M4@Jge-n^`YkixM;R4ghs^TQ z6R~*~PbkUoKdS^#y7vIalrW|u*b5`ceoc0a`av_v|3GwWdvE?A)BwdKqR{D{GXj%)>8gh#7U?hce%4u5>*i>YySz_fk zE7x4U;1!t9i_BC|EpgU46A3V~WGu9et&(M!+GKbv8pi_!QefI-;RFhIMsU^~- zb^;4X77i(-hnP&JTM@#wOmD{8Nzwhl;6AykmQL?&Mo|-wIT7Pk8e2i2DwLEY4mBz5 zuVO-4>W!hBFU%sOeb{lQ)G_qZMdq23f)yN&v%yWamZgX#bUCqkHI^757b`+4aav{e z>TsU(sU@#c!y!;(P9@MOuTr4>P-dm(s|8CdTqvuOi{eswm1^ww0x$msaw|1Jt4XtD z&J7V}{$qi%Bz%ssRFRwwFg_Mm|%EKNWlsUbDWKbtbhfo;x0tWa3-qvCUs*nY<~+VDrQt^ zPHM|j8NLcjKaijti9k8}1V(MuL5U5>Cy=4WXb&x&P}OR~K8TM39qBXhx@_8w08y1Z z8Wo5J3-0QI!a^l@+oSq;#!3J=Dh0CeP8o%p(K1_U9$s-5|0i+(Fa0#jf4Th~CTz&H zbD?D{u@1Dz|G~>wuN?Wmv%f!hlK=Pdbddi!72zIDE*MsRd449%?Qg+Rk}&W$CBqDp zAHz$(kIA-VuU^7>8t~r?k3VCJSo*O7U9eQC)ml)wj`tm0gr3g-H_ZtxU?$Sgs6Gi0 zB}g(d8p=shUYh9zN4v0t#-*l>UcBNGq&d(B&TAFYZS((@4M&6yb4orgtpm2q|GgJ` z`|kW7?Cd_x|ND44%zv8+)O;tjTj4Thp!hcuW@S5(`eQfR45)-hNdkU+|Ndflary55 z{AEl0eIIcehF(sMIT?HxTGe#Z~RVU##XsJqytGvGo1OY?B(1QKCp(6}i z1~NG%$t*6I8ElwOjuD~L?S#a$a{Ik~yR%Kv2vL;9XdC8uTWNBWVw!%HLkg#;K$P{Q z@g3`l_BR_PiA&7#VuXt|HB0qnx;%eYTI80Bnz#i}5$h((s&DvEkdb+KVbHXo7%{Lx za`|b~u5N0`ee=B-&~tJjsCsY+#CZ3r%hBy`!Py8v`5rTnL&i|DC3A#{9DIDY0FXS< z;3f2x2KuP%6!;4!ER*$Tt5FWQZ)1x`BHG3o0})Gcj3o1xhahEVOl`ks7O)yHHT_=T zlB$zNe=OY2XYz;#*dbpQsM(k7>)X382%*o}{zAjj_IB0udscdOnI>IJKsT%~bu0X% zWbM0XrQs}WX+p6erLNXSD%MCVwl-qAa=B1VWj1WoUY%()>0Eo# ztQ09R$zoVkZ`RO8EzeuhYyp(Q^^9gTm+EE2W>F32H}H3YGZ1Y>Ar8+-h`pQZdcA!* z%2@De*8c#KKyAM#csv=xTZmJg%X*nDK#TtWVsC%X)&KVfPv^h)@^sMuF&j?cupmP* zEi09Nju^;D!zU-tRnf@Whxcril>O+>8 z3NRi^o&>yEA`9uKrtt}4EyLa z&;+Ve-y{P4hRKbld7*A-w!nu8jwfPMCcJmVh4+}0!PEd`%D|v$LYpnu?>Cp9#J{WA zrF5_;BZV3YR#2RcBTFZG&`Grvigz`a7DmRILcQOq8riJ>-OR4hkY2-hr32u$`~Us@ z9asO`89bf;-pkWL|8tmN7{g(VXn{w7TPYZ{=`U_{AV!jtMh4bA9eZq z=veH(kB|DCQf|ComAOjv?=8$NM^P_yr+vW{G}=4XhP05=?Ru6aDU!};G0ZsOW=wjx zb>M{MLehGnXMy2%Z8X!9bhRB{#eqdRNMFPmM6>)iBEd=Ln>*P5?+;$Ra_zr%_Fg{a z|GAf^gZ%fHz>LTdxct9VI|OZ7Tve1vDh zDAZV}KO(62a{V!g);%}U_P5Uf_yJ{z@@a1l*onxQIz?Hr1%A0W64!7SC$B*XyU({f zIvY}T%gH5v59kO12SP}Sn^`WglM-lYe|`WWGl`t2Eq6aZq4@#0L36`tN_JI!2OFqA z?r(|k;Xmdw-5$eNFBA|(aur`NYa3qKz7X)%Q#!STh<<*mNrOU{c%qq~E_R5fXVc&jE9hax9e z3!t6B;4QXUhlNln=f8)@)UREGMI8(ihfr7|i-=X^SLNKGeFbuoI1rRui&fq9NNlQG zx!;<8DFIO0Da<2v=Z=LN4nFkTRf< z1X9YcVD&~ZriZy6X2}Q!82!!xav>tU<%AY?&5-Q+Zg1~}U7HJ?w^e{M<)6#Nm;p@4 zF$$dpnV0EnGgzA8yHMluMcKTurTrOgXnq3)ux8qH3R8Snd6fvMoWr50+ zWea3PmP8+O!gmrXoEbyK=Cj2#l7*56R-eN;0~bykSn-$vzFaTO>eb6oiofb>O@=>v z`Q(5TJTAHOR3FzW?=jUuuex3{O~l+!M)w+4Oar#LY?g?bFIBKG;y*&7#P*9 zlqOJbxwIhrUk&hj8LgnX3uib%nfg{i64AG9el(X;D~*`@&A>-2r*&}TYE}zMeKw%U z^~~oA<|E~Nu~3@UfbjA0KQ>{rV&*+>eGS)bN(LETTVhDJIFuUtX{(A?yf*82(uUqS=5t#vd6#x7vs6j(|_@YOOJf~-D; zi0*dE`C@cy@ZC;$dRJ>|%>*|JShk%%pu*Xg*+-l5+b`vvTlmV&Si>ynuV*+-gr2N6 z4L0!G99Ck|NGaunsMB{!*2-a8dsc7`!4y&9q+PyJsSQ35dXo~E^zUo=xI`e5 zd$?%RvrFpImM{OUd&Q@0eO@N9ig;^F&{v*Ph?h%e=o7Ke7R<}%=bJNxqYh}(mUXV% zdD@b>==AtmDuiKDsE@L|a%x3(u6Z3jQ+@TxP|EaWi4Z!1TY&_Q!cs#3Umv2Ku8r49LVKhO=wK`rN8Rt6lOa zR`K3zS5M_V=_XjL%LL%(vh_3NRjpb#45q}q{}g$fHHl_`V;bq5!|2CI3FMXwq{$1h zh7F~b1fqz)9SfZGYzA6%qllHPp)(<#jm}ANK@qc)mr?{19L$|llRc+Q4!~>9|LJ}! z6X-Z+CKhsn)8Yxzet*`d*3>!xj37>D6uqBN#3m$7q(Ww3fhZONLss4~KGvoRYU^$& zUC>(k+kkASr!N{A*g00R5%m)9%Gs)}cN~0;4Yl9J+Ga#kloTW;=>d3ue9^|I!4xcY zNmyEhNMo9~P{x}*P=yb=<7_iObP7llzNNNFL#p9 zyn*o28M%k|cKQ0kElCmPB@|b)~7N76hB_zB{)TYLOh*D*E9kmbBmYG}nG<Ft3K}yCz)CMC;rs2E!j<#+FBAspBEMqjs+-+%t097?5<2Ryb?%8<1{9>3~oSlFF zF0u>&&_4_q(dp&oyG!Z&HE^6|?Sg+kx;&TVb{fmc26?&N#&Rd8uYdkQ7JSiI@cre{ z@u@7h=laU-Hn@B*?glYb2Wg9X3~19)l~A*E4_L%IfQ(dtFA1XF7FM4#ALY5YvXhqv zGA)fZmTxl1m39Df^^<=~qfm(jSP`3@z6?! zu_2?zMeSrm7GMVVnAOeKtu|>Dp-m9RTLXgRouGl9V4$#Fp8OQ{?9{nB1(hG$)wDs! z`rB!ll2wInJ>xp1p~Grk-`mqluF2ONd4tg9${SKrT^-oCFbz-|*!K@k0*z9`@8?## z4sQKs+(dmn#|$Rnb1Y~RVqadDx1lDtE-&uJk*j#a|9)p<+_ zTiH>)Jh+Zt&H^g8vbu4BjNgOpiXOTjly9M`isGlen)IZ7#WBg`^kN4zk8qoEkCVj4_<57UmpQAN}!_f1|T&>%YFvZa4 z&$g+&AhTRrj8l~7dOFo-*q4DmB!v!aLy+qD$WVP7(TuDbTFWv;ig!U+B{x$7#3>QO z*CK$4ZxMI>at~#(Tcc!=OLU9T2gN$<0Q@z&-HEn9bh~qn3b+&fDvDc6r$E=3{R*VJ zTng>kp5u-oc2MR3Z&^1-%B|8gW~(2NmCH4v1slqf!DWvt-oLprqYW`24U$$nz7P-s z%9nXyxhC~89P8jt^h}uN!fMt%acS=JVh0EIkgiVwTlBcV8mTW>Q_^E*9Wm&4bvja) zMSdO5UF%t|9U85D**1<_I9EQl(-u9N{!xoOVzEPR6)E$=f>}#bKZZ4{le(7o%8RqE zx(YXF?pC-F^Te;P`fUllhVcr4ZZ2OR9am9pz{ZtjQMv^#Ns6}gk(KX$wH{()G;?+yGO9W=_#yqYN^r{-jo7EPK(If_-Et*}gS#BGj zmkEYoR4-{{?a|7Y!F^i!)e_bw$_^1NCt3Gfs0pu-yDYSw_bP^bY=WrXBZ+rN_JDWV z>Ko^nKAaU3Lh(P{4bd0V!E)>3uw6h6b_iljyH)6te0NaW^#h-1_eP4qgs7)T=l)l` zq~z?E!zpZb{dcil69PFntwDpQara&bCY1K3#?-5a40HSgC9}MN^vevAza~Cv?tQr7 zF$KYrbm=mLmXf6cl6H#!X(c)v&M00i3}|co=Uq4d-;2TSQ~c+9dAjS`9upX5Nq@{? zF&P%5EcB?V8(DZ2Knqd#j#BOR0d)u6qx6RAQ7Kg2PHLtd8N`R zJ(3q1<`j+aN3PUGp=^Lb?N&T%&omp>95Z8p=x~hH{uanI2h#8f|VmG;`7l zG$POO4)H-PL}hCweoAm(M6Bn~(@Fk&*17AI|9dY7j{M(!wg2)-{@=&bUHdl(_a3u7Cp*Xo$vz7#2ta8LPd+D$=3<1}3KlaGZfDy2jV@qB6bp zFcidxNdo#{pObOv2+)nD^nuXUYa1YC_>Yps2ZduOTU}}(j1I1p!XYpyZYb`Dp0DeE zI?4YrEYJsdSNXqpcRl@o@an}={=a*9T)EP)yH;Fa9I~r7o6dxXag6e!v~w2uJ1|pm z(Gocg;~Aw~T~#3DDLBlMY7^Z>6qG1+ICfrHC#Q(b(!$rqHQ=L6WtNh?CaS}BfAz6- zd%sny<1(hOr=arlP?Zw{H`hP!9_kBf( z3Ctrv{=26ROB#>=iU|S|hB(eHZ^p$=V8+CLiI+Jx^1!L&4*Gh0Lqrpp!VHcP-Rz_( z1hK!b<2>G4>v5i>hAQObES0BVvg%UjWflsX0ocP7Wn*c8l%lL`(K^?)l$f@5R&o&%Hd|<-d7;A2>#m(p?)2 z&L~#FF632;Oc$kmAv=g`v$62rZpCEK)jkRt7bN$#y=vQCDM~rnkb{N$qAZA}BB*H3 zibWv%=WWE~d=}r#7W~kVW&P%@lmX8$%(Bt9!(1CJt*ssFdfRmNk4`XCiE5r(o#yad zhkE^M8{&gdTizvkKSeJ{4D$V@+{i4&-la$|D4@qvr@;Y0arx_8Bk3)l?A~lzeux@cxx;ajy46bLaTIgCCQ&jg(Ef{v z>ttk>j8aJX>XYCxE?`Px1T+0D$ygQbFcOkktof{ z_D0$=dvE!HZfoXlhxd*+zZ&n)pP%)BC?>tJb+(Z8h|7{B-k!ZD)+^mA&c+lmrV@@f zhSQZLbvhP95E*WRpM3xDrDcn5ut?dkR5_Z~VRr`bMv9aHHqjqp+pavVdwl#xgig0D6ZP%eoo$Lnh@vb;+c3x5Y6mtcrs-Fed$7uIQwAXrm>7t- z##T7f2CNuf=u zKV#g}l)FBw)uZNOasmrzgk8{TzY|=-t>B#I@cCW?+CI?{Tm22~bjir#6qcu3Um^EUe4)v(Vh?WYI4}>w*=lpA z2W1s*E=Q%}YNbp)bjL(apEV5DgLKTS&|w)eO8Ztnb!{&f=f*_Lb*tC0=H%kMtgrTh zacuNnyEMBE3nv-u#>~b3JcgZwqN-^x?!t0b0`R#{yjLw*L3Tv>GWJc~ZdbgwqU|W* zY1g#E>nPzAhP|@VZ4P>=XLr-B%L7(_wg_;Yl~Rkx??Vx7Rws8JP@N22nbo1(vk!0T z@8RloR9U+6mGw6yK?&D(t0|CIt$!>Ldv z=lT2@xY~v~43PKq?(hDEf?%!(Iz{qp$jTUMfja$5RjyA_P8b&EB!L}}1uh`VH5hQs z3t)M|%E~~Vws~HOr0TjE%sYhRxe;Z@M9Qolz)6!u2UhQdlHjouJVvYt#Ike{ljWGp zFacCeunij%nWFh?CawoF0uoUpavIod>Or+wZ-34HQv|oL2G63Ji&w?2Z_pUVb1;Hw zdJW?n$+61mO`TB+WSPmgb@rVUevp*&M%sJjgu1w~tB5Xm{^!m}OK9TPW#4rE|2lOZ^;Yxw>KUBl~UzIWhTV}1z! zPH-mMko$hoHG#yYOTN0ydfrdP4*WM6x!fR`1ZQJ#os#&*W{FBvhUBC-kHY))1$NW| z;xHpaK4Y46scsq*9)S^=Wl3k%scQO5QlVCr&8Epj!}8)bvPc`o3eFE;5&a}`7ZbS+`q2Ff(}xj^$}6{ZkJ(Kwij1cc&D{u!0qwB zUb*p~cV6s0-T&Ulv%oTC`|mXPQ);gmQBrPVlmOGtSAbBxwxf@jiyl!*-7cxbP+b-$ z8>hSWuXx~)o2IW-lNEX2Ea(~mb&s*x&|I_geO>cU<~fh=SxEk~4Cgs27F_~b<^M}J z{`2n3S5NUD@8z-Oe>TS1M?HW0w$-hbAHSzq;Q;*BZP73*Yd*3)eP1>>>v6QoWd$ZO z(=bfI1VQ1vwSoHMKKSwK{naKHa6i6(e{lscD}DGDSxUl~{@z{pV3G_aHNjOuaa{D? zQ<$-w(4u#R;u*!o97I$2u?NTK@WsylixGE`~(_rXS#({l}Bk{}P^# z@*fw&a$C3X2B5|MbFlMj*OmXfulAqh|9w14c765LSKuukOQ{ETcKSQJ{oTRt-~e2r zDY-?Um|zAbD9sU-at)>_zy$*{CTzgqs6bSfipg|}3qTQ!Cn6bbd~-!o?4||jwB<1Q zZWvF=NP_4>0ESggj1FD zI>8jh1)&%*!{Ju8*llrX??~I3ny~`ODu#oQJ{j{aO+>ssnC3V`juT6#O@Mtq)K^zg z5$}$(Bap2f+TTo>-&1}>qVGL5Sf4)Goe{MJsKu{~E2k&%TD#zo=@e$k0r+dgCeap% zda+2G&#?f#OYm}k|Ha;c_%GNlrulXP3mEt;1YQGC-Aedx(P ztYKD00_sl!oN^~N9c>$cYSQWnIaDBrIb%bLnAnD*WH?IEM|_>~rez{$N+K}b&rjbBkKdhNy}vvgBI4|IJirDZ2 zg?TRG<#nEDoGo?zo~pmZbn#KRlLM4$q+OhQ9HA9>kPCh}V3d$zI4WiTForq2#wji^ zV#9=zdHFc)$=kE@NdA3!^ykP=J>IwkwXHB6W^j1wf=E;C z&yr|oe2sS&SV80MYn*LgLpBk=dMFnE{wC1bo-r!E$2r*cSZAB@PoPI{{lY`>?c2sO z6s07F1=_ADD}L%$L9eG)1RD^YIt5GgdbhzP+FXTsMH$&jFq-a@dfFyLE63mn6=my@IiX9Zy~Ohshj(x5obX9r+s zFjYUNXiDguzj=#m&D|dQml>h5qo<{!>$6l!n zALZE6MpfE%Jn}+1 zeFDFMSytd8MZg8qxcs;t&>=J*HBHJ6xcz=;R(yIM;OQj)t<<9HxBvCyKkW`)y?m1Y z_wjVs|AYBu7fSp~MW}6*00~MO43)~cIH;$$I7VRfk0k4|6>w1UcLd^jblH6MF^oEX zWXcVq*{C)|Gb0R;ig~?cbbPJ$zq|V{U%2}J?yH>_PxAjho(1GTNdng-vup_2aK;c- zNdu~tVP`NHi2pP!2^S=}vhU-oeDF+MH9o%NXaK*rbIxBp__9W3$guE*xyW}-pedz*|+y?i`F zY|h6O#=OZoBuEq6ZmDtHTL#@R$Y~h*Je@Hx#TlN$v`Q>e`q_$n71KPOOF+d0fy)v@ znR1ql#B0uHrK8VP0kZ_sq!ZVyOs92o4op+>;ez5@oT4#0Wid>JUqfY8W7_aYql<)+ zT>jPL^=tU5D^D^MbFyDhGQug6aAhWo%UM?7DLRoc5iHJpR@(8|yulgbq`WBo`Rru- zhqDve6xbjXP(q5$EpTylWsa@QcV#4r8#HGJ&c{9_0D%Ab@Ba&~2}!qv(QA)n>0E#Y z-(P_Z9m7GI-8t1`m@`rRufNLk?JbMIeV8N_1d$ug12Myi!~kc-Rz=b6Y7i<2qURn2 zb7<_|%rXF(K|wK_M8b65n=RQ%@msJ*OK0t&H5*D~HgIR~J403y>8=bWB?b{6t0-ig zcJ-I5;rp|mCQ?~S%n75Eobht1 zR75e^zL`AMkeP%sK^!_W$-7Sqt<*I}@}EA5Q4lTF%j8d=RDgN2*{TEHdjN6`Ceaez zI*)7;?p(Ydy?Ijt7Zg{As4z)%s-B#I-#|hBH}g&;EpaanUPZKFI+x=$2;pn4Bn#9i z*BL>_8tORgCTcIU`h*sub?C)ZaNC_|z=O4@w6vTKcc;8+wZk#xwGDaKyD{OzJXDfn zUG!Ic1#-n~HaD#@8aB4;9*m{5v9;_IGExkM&qIc&Km@kqFgm7q?C-yCo5i^zTs_)%@{1OSf1u9?yWlZcI~3A z*(I)h^Tm*$h1`;;6*@(?C_Mn*u)#OR^!$e9@v!*j9y#S+nB$Jd{m~8e+J~)D{MIv0 zbe3f}8ygDcs+$-C!5W&H%B!H`+Nq>OGfIYR&W13V;_N;sCm5_+7K8{a5m=F~ZCzS; z1=2@1%?$4EB={adof7d4v)DY}%b->*+_mtk8``^)XWe?XZqeqASPL`Uqp}ioFkEPC z7gkkZnp(^*cFZ|Sz+J6Xb)9@BpU&RAQ73yB?=O#zPaA!4 z)}-BR31-s#BNhQNMO<*A#OASkWvz-z>#*{8!|6*#!Tvl97n%oHR;29EA=Y~M^bv96 z{jl8QW31mD=d5Z^b{9fyOR$t_es*?pXPopn?b4PpL94qhy$@OVa3_9?E=zZ2Gw^tA z0#>BZ7pNjG$B_48|J%*}TLqf9oB01PUhaGGpLbq7#ecq+r@PRzm|zIAj1*!bl;#A_ z!E+T6UlkPr;QN`%J`|K{Ix)bNQ3DvTyU3ab#Cjcq1c23Jbj3pnvOYAIL{Vi#kYzj= zZrHzF2_g)mHo?suCMZztn8R~$C<7uG-c35H9r~EVjA4?AxYa{48l`;r4i43wR0UCm zD=?hRc`3PTh*L64dO0PxI6<@r|2d;5@?Os&@2UU(mgN%^s&E@8k)5Y>0-23Mw-`BA zPR*9CE9;DwI0{BWx@K@@l!zgx7(<0UI+2-sL{r~K}|8gTY zk}!CG>nZS+y|I!%l9JkOdsgFdoGi(VR zyU8mLuN6BJi=wEV5as9Zs!==ygeIa*Xj(h&)e7N0g0OTK)UCQ0Hg8V2PlcrHf$#kj zUZl0lSC6p&ud@F#DjBEQY&_$5iZeV;u$WJn@&kGsf-CaBAAV7*|Mck>AM(FH&0i1u zuQu~zP%4l>NV#T^Uzr6@!oP%QN<#Tp=;yfLoMiCnmyWm%PE+tW9gd;Kk(nzLoGk*d z`xPhw5~WyeK}7UF|NVbZAxHxLu_F{s3W=7iKrznL&xq<1lN@tNahfjCjAVpklV$&(*)$C9f?x`PN=ded4#Uc*22B-;#oh3=z9VE#{SK*oDaSbd+K-|RKx&+&F9K@R{|%$?n^EnXF?`dFK%_J#bv!*7 zqbvNcg72$o4ojINg1kVbK6}O(X;AipX0w!dJyy@cUP5jsKMv2YH@@jKfyXo9BGvpp^x1!jjmn>wHh(fpuL=QPJ5k> z9xjxR$~?%Cs_eccZ>r`hHHGB8oltUi%JhAs88~PuHQDvAdqyGzPH5tm#l&$dY-Og}0+`&0D1wn$W$Pjxz+_|M$a)iFs zg6jry>c+9NAQC57;-ej#ct80Qloe{GwrAFACom=m=ND=@I$m2*k#cV%HW7s1>BHV- zxrex=xoA-)joXhWXdUeHLc=(nQ6(@MWQW*nC>9Nwx_ife{ZbioNaol78o-M9?`JPxysXB5 zKmU;b`Dy-oSOV3VZw&dWr6Y12ldoFCzvyRpHL7~_2?cw!2++RAAh5Z2%Og=ioY%KA|1#)eOC?@=!utv`&H^#N`+go)x-3U#hevsa>*HF z;5w?`_6R1-P0}8$(C{)t8O&ew#T1Bd_?fMrGlxhjAhsVfjiUN}jwWa$3vCZYGEaSJ z(K^(%0kS12+Ynbep)lEO3^7 z*Z`0Ip8Xq$Qlnx2qHT#@OJ2mVt)bnD1S=Z~u9obT7jUij2r`M67Z;=$-)(F97D;vu z>A&_G@4=K*wz3LLO`;00D|@XjKx{fCB%#WQORj8h1!ly{7xBU|1?@-SZJ(fV=_b_f zOsR!?sW6dC9Tn_(GZ>h-35M8~7N-Gn`MDN)w-68LVxa#2#0SAN%G(=r#W<9LcYae)>~*FM!np`Ox8i=*euA zqUI><&lch4vDV;-IXKaV2Kz4LnJ!IWj3j8X^pTQv!kNGKAZFCTsjw_nL&UN5!fZKa zQ;FxjXt`*!{#6B*6OJY0!!_n#b1`HaZ108f4_>4LCAjs=NXmV`^i z{PI=T>PY^p5E7Y|NLRy^3~AVi+Au-)y_Np7^;rZDT>^G6Z42NANOWVS88YP6-Ydcw z91rwW$o3aH>jqmYE5p%UL#+8;|Nh-O5kB3r%`BYY=YQ$)#ogz>8B53*Pr*=GK2rVn z7oWd)QGNe&@Y!b{-hV&IpIP>QBb-i`NX<7ABqgzA90|#>B(vqHQ@_EA%Wyi5=On(> zB4a^}EeNmNT=he>{kY^nQ?ArTmPFxR8llLhe~T7axqJ~S+nhQgXfHFgrO>ocnZOU*WP(r|HWm!)~l{mAqe*G360y#dE zl>OU={j=vBdYbHa1H>{&7t((QBb+UVnSO12PtzoZD?KZovV&U|?g?r6nAKZL<*wU( z1n-5Y@*22?b(Nv^F<=?TYUgrBu% z-O)Zw6jcf~7?ZY<{SB|Z|8$*`m`-VY!4h)9icEgJykaS7V3m?!dvIE!bqB$M9deeE zay06<-x@-%@7^7uD0(ES-8^hYt?mApG`h990f{zCPDoqH<+hYEZr+$XxpbOhulkFj zXYYf!{!zrWQ6wsA%IpjEixT}tg`!x;gI@t47*$`9F2D0vh&W4!d5Sagh|A87cmq~H z6^W0c8CFKUtkor13RmxJ_Wa&9*NQMf05?LrvdB|{MiqlZ&WA+?fmxi=f0D#}$qG3% zDI$hBXOomHM6K3&&EzW~d;;%&V~Z~!w9($1U_mx8`g)jpI7z6EBwlOqR9k#{c02|4 zUZrnu>7hi7&<#_+c++3s%ru*D}rPTRKCqpUd= zm!(;TuGNFA7UnDpA&<=wL}Qa($?2jFf3Nklrj0I;rgcgCQFrW1o5RsgUYk-5)WDI? z2Jxd=x<_b6?}$J-;R`DCX*+~N=LF8vA?d#fBIvZd*%C=iGbzw+A?j_~* zMQ9jUVH-PXv+c^dw(@MdX{A60Cr#~eSixbD?u+__1{fc+Y)WTGNNww!CPbhKN!fj%O$Lfcmw5{2#lb>CTLJYet=6Wf z7}VkRT)utT-mc*c`=C0!S8fYALdqi1o4?;!kUMV`N5PTR0Ga1n1ElP^p^#XT;X+>s zfL(IbHn5crbL+$fFJpR5-8W{Z3`a(@87D#vjZv2pUK6cmLjAV8icH(GACcF_bn_&d z8UbyLPckt;G2wE!z!{!}uNb@Nx8}hr>P=qQR@&=shF{u^duoL`yT>%C)nOxXF4h<< zVb{B4MZv|*Q8m%SINEEu%G}dLK2heHQ%`c4ld1viLzkx5D$+$+wFaPISU@&J<(kiz z6HXHcufdk}1BFgw#>cwT*mNqV{i!$`rp*Z`sKV%*n(FM)$$ojW%n6E46e)rHMAgAY zp)C9e8dA`NaFnrZs7G%A3RAd7pN^tRp2*QBR>xw2zAG`Sd95~G9=n5==>5`2HW>~T znj!_UbgQdn_^ozH4@;mSCT`9MO0Gav_`bn3g%=ZUu`AMOrqd{sMBCYG949n8Cf$b# z)*4?OlWz1+9{Y&F^6Kl5k)UnWzFcH6;|a@1;tUcaaNG2B+dS7*mdU$!J419!mPf5Y z6dl}H)v6T72pgAU2}H)p5c3HQK)TD#rKURw*{cjkC^~yXg%nY(OJ}Q7k`gt{6PzV9 z!IHe5nwy5lg=8YesVCxweNCV}NxojzTeYb}txLoRc7z4VWx7i~ACNvCn-Qi?l8^N9t zHH%_++cAO*$gK^nBGP`v7T6SEUd^aB8`jH9kF|F8j{>JVNZ*bDV#CYrfuL;+G91(c zpcr01%7pukW@|GUoRX}X$7Un{xy_;*(4)-2)>l$+7S+xDCMV^&@xDp+cUnEArLuEM zA@HGpL4R^_{_g^`8aUfIXK$8BGHu|YIHQ5H=t4sfu?KSoPnDl>a`Fl}1lqoT>@a>0 zi>KL6bn_U0g+G1A~+xB7IW6=Am) zW@T&~3+lSpY>0IrC1mWw)T0*DqfvUn>hygB=@lka8FMOWjMGq9K}545CpcLeox`~K zN0BM#5ng&0yykLmL0HQqZ!D}8u2qD!Y=dV^>cMujVTByB=`f+~ZZp7GmZk>NW`Q)t~+lqwK^w-P={kcBZqH*+c) zPO_FR3^7@Tbpw=NSaQ1&c@vXd24?a0MRKhZ&y|8{i?lS!ree-0rP+i^covj0A&37c z7%vuWO{R>;WJuYNaL#yJEB#b=s9g9VyCZxJ1u_IR7R+_|WFuqfoW^r> zj|B?!w4O(!#RrgcW{&TO>K+RU+(rqfcZ4HOW>n~B6f8qjK8lWkQ9VGpXyxu_E^vcy zl;PFsU)T3E%>y}{FeVlA^L)*%t?ztRB53W_jh|(hPAW|Nsx+e~C$Ao1b~E|}1MNjO9EBNpGulxRg&XRFL1ux*S*LYGS zGa}c12e6|4+ri6&gKGV^PY(}1-2Z=yzaIK4PbT98lLgBJk&T&u?UB>v>DOq1-=M(I z>V6tkhTM2TY82K{TmNFz0>AmeQ*(q45B}}3OX~ct{#LAiPMI#=xmo2${kLlUe|}JV z|MSHcpMF^XPx04h{g;FwG-2#7x-6E?1`U?V{wGcEDQ$C8!m#_%WEtm*yOR%65%&4I zK?!xkI#WJhmd6q1eXaqqnJ=oaAHT9!zrXJFZ&NOavA*8{XG~kJ|9bfOr=QmL|HDr| ztp6wZ>#_cuga9l<_k(>_{iXZHTuC#RH@-qYlYCAVkP6B~0^o8h5mI#JC(t?LNrF6J zL1I}-wpPLfVNOWX*ri?ijHvt=JHVj+>P2Mj9f`|hM{9+$pUE4%$W!vm?p(@T9PRIy zd|`8zyHADG=i|ZMm$_V|ANg|M_QDt(o(~<(ek^;nf?j8>?5`H}aua1kHjv=pvx8SO zs$JLg7k5jEFzX;fMLH}+hC|e)8EAPB*En@LaW+n?b6A-%Q6*_{0iK8@=lY8vh;_Exr+Zhq_ItDcji8*e5*x^J@x1#q_`MFwEEAn8WH z+^?5%b1~9zyd+H&utp*B<7||p8r2UFCC6?rO;LTw>xfcCgLP=da{m}h6rEjNy}mj^ z_6n)nLp&SN0v@B(_y*eat-gvntX$l!TNh22Jl5Xk?{YjAIf$Q0@b1qmx5S5$T0VZ=V4X)u}ai%(SjM`>z(V|Gv;laT{ zWh>p*)U=pJUmB}(y}1cHsp$b=>(5LJO`A^3?aFEKEzn!d{${=#hcbZuY9;Jf9k5q| z|8^z#Z@a;-#Bj3`!_7(zE0J8TL~_{&$x1wDEAgE5#Iq9BU+GF*L6Zuu7cW;x)oS3c zx`6k^^X+Op-}c1Q7v;@rlsD_6?2G+!HTKH~Ved=G*=j=0wna!MO;8cz71WA}{@R!ImeylOQ~uCxF(G3$x^&;aTGYAYjJXq|sG3$MwtNvd8y3hYbPU7*D@$sC_ z=3{e>N*DWB;}*b*^Z%DGzWA(K|MkV^FF&0BKgC}UoszR;j5A4HB4{lU*e6pjRn3q@ zNL?DDC8U?L1X)lzq}O>9s1$`)2&r8HGC=i)ZrVWMSe6`Tk{;WE!B0MQ za5dRcLtuT5$L9G*CBds5x!Nl#X!Q1B^Msui%1txZ<7qZKhxZTW>$5i`hRjR4&rnks zCiF9b5;d~#VPHoM&(H_xW1R^9=r4{(m9V^iWB%@)4z}plm8i;Z)wC3ns(2 zmnlIEeYn`F*E)*Ko z+10KM(eLT0v;;FAVsyPLO3WTw`j#-Zt?9N7?e=i}wyB2OK*xoga*RYJ&F~N=g$Z46 z2ww=#rlDrjnJaE8LKd3mKjk4PIWDouKlVA#N>n!d9PU0pGrucvZcN`l#O+68<&S@jI(6SNuJUei@+>tv3oo$|1wQ!d%??65{Cs4O#I6gj#O9!Uz2t$TdJ&wtmENtq2yY_dzh3pM*QegRAT6h?XaSmhPuA2;Xiyl{9D$_I`!2GO;@q|eMUL=&#@X~ zr~ex0ah=c>5I?<6?=|2~8{GbAeU4*3D^*+ht(sFjDowqE-flFPAa|S2I(!dTuuHw7 zFdum>(fMD>tw-*MI)X>t-bxO8qw0Bh9fJlnyBW|%1q(&g0`8SGXgPmtT?%^Y<88BL zr7rHTjT7PU zr}xw{CO6&LAJ0dt=#*M55TUj*$#mX!ZzR1yhkL>9+{?jxBx|u*8s7Ke$sgda;r#FW z^OLiS>j42V{?4=Q;pjg9d-3^U_5JVR%Y)B8y#IZYziN1`_II4)cuvswG$xrKwQyYU zXKzWr7#$4I|AVsv^Cfz5aPYFV(_G5@Xn+6y{(gi3&4}^YeyUN4eRxuFb9VLW8XaGp zqLbGbr{_24uP?6AH?ObI57%b{bai%l_4@RO6ZQ1~TAiL>-&~!4{e$`j0343csVlWF zM)o;SWJVMrF~@0&76j|kACmBePH-8sOy^Y63LjE!RVyVaygwxc3 zVRB9kfs_b)J_dKG2MBW#T&zZuTM3-XoMs54dc3eeZg}k zi11u>6K>}yGEgEIia@(`Pj_Yagbz?cIf*5Lq8$3)P$eqHg&?Y(fuer`4)E~Z4jkyj zb3@nw-OmXOf3nonV}KoU=AJ5PVjS&Kx~Hc{%xSJ5P3crFQBHWQK<|Eb@Nav7j&Wib zZvhojNSq~V4#XUDwfk#`bPr8PMy52TI4wgD==4+m6DuOL%Q*atM|(rllro*Kijog4=6VNz36G57&v=7P=G+zJ*KBEK$CJ z&TwWYSQ{tm=Sd;K41;H2B!Ln|Y%1@Sc)6D+2^qMx2Y?J1-C$tXfABtKN>dW3kJpp` zBC)KKjx&834{;)j6xN3>)kESr&S;G70+AeNLSNf60tMfs=HnD$q}c=D2IT>|6P(g8 zl|4!!+b$4ihB+e{BvMYw0}i#9zG3bvI_b5$APL1_#0A6tk@4G_Qn+XQ7AOWiq$JHR z0h-xixCNkDXNI-F2|+$xlwSXVUK$`ssc2FjD^bEuFIerCF-c?M)SIzW6NOu;6)Dx; z0vxi`8W6iUL+06$epPM`!%Y<^T@#GH(dCr(JQq$0r*~M=JAxELL{yakg`=4VX7mP< z=7A;LJ#AA4KVwkf6{j1Cl_Fbetki?kG&9lH!a>vrZ66@q}^v ziE%s4;c7C#)Gh}P?1B3j-OpJHivrpA^9^%e*T+^*O-svvfU4{>+|#KRwU2QR-veVf zSzzkcC&@7fNl|=(VJrwI=@O+hy9LHgXa;hU;RV^VlMZD?rZ|R8Vi4?+j;S@Gpj;N4 z`dL3ws@&}HjWb?d18&I%_#6Z613|NwIyw}v(wu+{o9vFlU6;|kfnJQa5ng<6^5c#Z-FA*DhlOKru0UGNht5LOmmPGo^oD}MKkr^!84 zwnoOXAckpo-&oQ4V$B8X*Dwo44CuD#q=j4vXkczvyn;ZX6S83JW78xjWFjsP1 z({x^)=fxzYak^B3nx}X<@ZaQw>#bJ6H^#6C{kG7|ce)siC@?HqKAo}f>I3A)5e z>HYy`XV)Yg#{xllMPnjSm$7FL<$A2a>@Dyu8R&%9%F43@fgjDLQ)TBdBuJVL%zx=3 zXI$!uaeBZQ%El@LHDgBx^(Ct-%w&!FUtuC5jX0|+QRq5S9Em1XlR#vdE23vxq6F$dXD!@-nv?_QxLRTb=42=Nw z0x!L0t7@T`<<#0XC4JOk<=Gkm%3~`Q11(2PSFdGmM=ra~dJAvSWCQOC0YkiKEeO%G zJY{Lh?)Cm|b=T33^Q-psNDCpMXfTkc{S8iHnp0gPAuxTNcc}hq1I4gER6U75!!~Mh zP6C`daN|vB-DkiIqT!S=#{p(eB&ONG z+RcFr58gqxtPL(e>=1f^7^oHIcgKN|?txN^2~l>_ATTCCM5SMDW^6hH(LhgCrIsx> ztvxLaATDA0+Bo5Am{9l!Rvedp<3h%QuMV`#v}Dg-P>1<9j-Q(9=;HOw`N>&?;B-sz zTP-_-1L=ed_AgvrfeL6?DK*Z*dxmI3H~}Bi3ms`>t9|Ddvfn5*&7PO0ZDwR2Zc-$&-kFuWo&J|Aj)NYg9a{5D=P&Ih&#gdWXU<@$xsb)>E zWmpi*N9bm5&Rw&T7#gkw^9{y>cBNgA52u0isw|W;Ic9Bv##)wQ^KRR~NuvJEmA?~8 zWB|pIZaCCuB@8t81x-q#1O61w6jG8U#lo7cC5f?$Mf-ntI;%Pj7#tM5XA zOmJ4jrVZ2#{IaoCNt#`eFgoX!62McagJ_;7pmthS+J&9f9kYctrIj3&!ZOZNsdxgV z3Hc119di<(eN}G-iV^xDONkILY2;0w(wHju7a#@)Brd+OtXh#lFe?aSwT7;|rLQnn z13B8%n}pHlbzC!JW&xRjz-SQKbWQB(qwasfr0V2OZ(x6!FzuA7Wjh1+PHiJVnkaI@ z1xbiL2vW-+m=c3XTNgS&Dv5WAW}IljUz!yH9vOkxAa03);&lw+WQMsuSE~9R<}Bdz z5xTJkh%IPk?dXIN{b&qK?%+g7G2R@gY4gP%W8ei5fkh*fL&)zaB*%k~j3Xn%y0H~n zOYy)*os64mpTvMnaDoMciz<3b;pQynOQ42kC?O)|bYiB$IURIH>+xB;n(Vxp-JoF$ z(46{Wgifgd*NJe|>PO5KUzTnWI8rA|?ZATPtz0ax24EJ!dGS%OfuAt5_Jt>PSJ6ta zdy^A7x2`pnqCB~K2vX^x==d6)Uq|Tc zkL6!~gN`qLLVrEKI2|BD^$Dx<#}#Wz>y7vG+}I=i?TpjT&CC*K`k+#G*>{{8vQPas0yoZnoWUF+)u#|Fmb z@zu@w$q(NjU!lt%t}b6+pXuFJpBAPhRlb_YStcl)I>5<{_6p15g7ciSoKt0h9QHD%u+fdSxoi@?zDSQ3gx=U z&q{gU)rNvV2}$V$3_2ig#+eZA+`^)i2**;u>3H+{(Au-KeK3wDcFIx;h)qNQrWP;o ztQ^Bvz3r6%e>ngi-TELK&0?A;!&V<9C<8{vbSX|PP^+QiIaVwp9AU1{%+)UL_G(e2 zvg+spHw&j@3jGbu%wz_7HH;YScAkk_s?@MkrUiD!SaMI(Fm`y0gk*V+XJi02L7@nr zVwx6Q??yPC7MVA0U=wV(2@|Jy#dIfGSI7mI0M54({XY~;gED{u5SoI z4HN9;1_NHNfd3ew<5=wpiVIfrD$GHhxL{HII9H~7dAU{(3pu0s#^?C>1OdW3$G9_7}d(2syG=$DEUw|gE#Op z%d?Bq^Zz_jQwY(5JWrSA`a*bhPyGg@+`F?RgwRdT4g+(Iq8v@IW;LTJ;rddt_T~oO zlb=$OCIXQxPMOe3f5LHmOQb;2&%Z?8Nl9^Rce|w(CeW(J>kB;T5!yXv*}uE{0>K)! z!2j1C0&gAMR552onkWt9+}Hv^;C6wTncL0S9<5G@zp?@+q?Qt)I!2Janj|QP`VUxCPCm`$7eyw)g1Yl~ z!#R8I>cTTV$N`xL?yRQthlz@7XLk^xJ$nOL3b?)204xw8Hpm#14Iky%(&5Q>MTG0uUvu*;1pG_c8hD*^pE5le?Nndr_)LeTWz8D=q zmp;aMo7rKmo9%=F$`ZdUc~q1dv>D$Iw7jj&4Q#t6q(p+1Z<9tOMk&r_1)dQ!V|RpS z)k}RQP~uH>Q6Jan!`1xX$zQ|$|Esg((^qGsMY1g%o%jC_zj%50#TV84|DV46kpJsR z{yswAk#vDlI^kHSkwm9fB#_CuzO(buN9YFB9-I2;_s-k;m+0-z+u?BdcIPen>9{K+S(1^PKCTWs2l@5yA&*zGumw&NI&W5Ue_Mt<4-Xx%P*$WW0}w;dg4 z31rl-k#S42WK5tYafmATdau%BM&uahbZoBijVDWJ9rkW%;t7M@1a{|h<+9Y7<6DfW zYEfcC^~WZ-*CZ~WRN;w!E7pl3XUSNY4rBAUY|M*njK#PR?!|c<5k0$C8F{z5W5y;d zS%!FfcdBAri(n06tgMf*+T4}2kEdc|2DHF?5RFcOZ%mbgupK(K$F6peU6lIVqdkG= zLX0_4Mp!IK(njqA!EQ)#8OV5=k~egcw$AnXkRCMWoH4oa?5fV2V3%}3#tE6?B3&DK z*muJj$(S5Nj;e9a=^ahUjEsqhv3|F)!Nha}+*HyDPR1-tm%!#R+*yGe!FnkPlG3br zvx%(t0=#Xiw@K)xE74BQCK|$b-VnPH2$GD&Qh)`NJ&dAmfDh);R;RKL*sZUV80R_; zM~N6GoaJhfZgs&lVtATyK#BpJY_!>JhH1lbZJYY1rLn*5ye05YBs~wem>z`CIyxsf zr#kq~(v)!Y3ZA&|nvWAJC(J!J=B5x1{4br7ip#gMvBRFSBI$)*HSCXnW$Q8lr&waR zV8HGOZ^fQqiN}d+)rGpXX;rwIn=8|6!7?ftUky8GTprb>zM;U;4}x%XT*&!9(G^RH zXc?cm(ZKa$z^eV&kf#f-nDK!F@+(b95Y3C8$9xr?v?pmVc?r zwohym_VM3Z3jSJ52)df!xMf7u@)zwn*@X(kmD~u?bu_E;!ZbC( zHqi;M-lzpyfH-BdaZ2t;+D)HpR^i2Ic&8Zsz-b#xN|%qv1*Z-8YHcdWd>#F3U3|R_ zS*=RDqF?BRPu@t58<4N;)56PoKiFDT=U!M`>&jri#gg3Px=O)g+|6^xw zwZ`2RZ?_+K%gfsa-LGROR+`N?5n?P;(WK{Awe5t>Ss_U;$hDT;oMQRX<~(iDp?%tS z;t(X}M0PViXj2ij<{-Z5T#Mg_-?tE@jfISjrIv zu7u8rkgdbgKlgyJl0oHW)k)LhjZRW5TJ?lo18|?C2~YTY3xey6<~fn8afDfC#zOb% z!r9cmMgx_|)%9(sfXcvYi|6b zPyLvFoIs-n$fe&a2v(j$S{UmeL9W-cWwJInf?TiX!|-;VsXY+Xrj#`vJg(jO!PvLL zn*Jw^fnTzvL&ohuT%XP!M~fQ+*>qN`2a4-1Y6Xa7UNhr)UYKejblo16kZL_${{0B#-lqf_5>aQ0|O)*sCl zX?`-wdfBFrbh@|wse6!>vXwJE>o4fgWVcstM*e4-W!-@;I@>#kK+OJFPo@rON-tenX1SV&vJa)hsXiU5<1Dobp_P89_L3|myuNIL zuT77!`%RypGCrO!C!8iW`uxl8M=RS@u~!BfVC}<~rsg$jakkVPD37$eK@G}1O_NwH zsx~N14QfypZV9!*X>L-3H05yPzMYFMfkb>JmS&?yzX z&<`BHE|M9k2m114GEOjAuuPQ0+NB>HOir1O8#WpOx%s&sm@ofg!Cl!?X>(a;L9#l9 z?&x~#GyFq5pz+{)b?d6mYwTguLa&Laxmz!ccR3!5oWv_}*oM8J#(ANnLyECv;{{Dq z8Z+KwAT9lR;ardf<4fOtvXnhAHukDLPRViYW=7-ebT+KroN`ulFW5brTLHh$NemZk zR$q;QTJ4#nu3)<ogH*VvV?G9s$(p0u3u+60LTD9u|ai=0stnZZ6S0n z&O5?wt~*EL6}hA29v)s3A;ky*sUPW@Z#z4`Hr6ofNxv^{7C|1V9LFSrTCcwvEXRBX zH-N;>j%s~_?6^{Ok{3{r zGI+sF5@huPQk~Flj0G9eOpr`aN$Ae^pkP|TVOZD|ovG%L2vsLj# z=TgWS2NPqb!lIT3mjSyVtw$yi&Z%X$kQ^TEyW$(Z7>NRm!sx}or^UC)_B4kxCLvF& z`XqGjD4oScnMZxt*+TkzWGzT#<%MchX;hz}2Aqb^8Yaz5-V79RGNVHB<$jLiTRbEG zx4xSZ^;_drbRl=(iwb+HW2l}&cMPZJadI%=OO{yp20Y3!#|t8rwp4PX4CJt4w1Q)^ zTNU}*Ea*w1XVT%p!2$ed+u{S4v}pqW{uaifK2*5d`^oNRJgqP@(y-PSte!+hXw<^Q zHQDzN8EiSx-!22;M+Kcg5+eco_5u>ZCrID65s$^?+R&Z_+ua%bb{OrzJ%X?Xu%8p- z9!X?_(Y_z4Z6DxM@c)5VNP;<7F#+x-;mT>q(?XyH&FBIOlbKXXf=EfTnXonM(xuPv zbmizl3zpc5hcM@E`@WVqOE6E?pTP&R=snM3%Wa#To!W$z^SJzQe!Bnd`6>GR2#C})SmQ6cIJ+4ipT0W3h){I>)Ajh~{MA{shoXWek+KO7 zQB)p_0~0bhGNVzp^#k?+#rdsh{DbYQa1O(Kf171^!E7@gnR!$$DhTD326}4baa$Y< z34PCcvHL3A#`_w8ytEL{96+-&wz&6n55RMG!_B|fExe7{q`)J;C3x^UXKOB91?}sz z?-eH4^RYCEV7Ix2_ZsN?HpSYAe1r*j3f{hNo5)=Y+=PF;$aI2B2W6W)^H3kUguAb2 zZfWE&H(u@N3WyfnHu=&$nuqeL&qdK~a_mW#U!OreSH;$R0kC5V*4|{GL|T9ZDI~c) zU3v7~rEz?}ui3D7W4FHv=>m3kLO`dm1yoidNqN%V-(1LpA3HA{WVGH9Y;U)#F}sFWNakgFrK zgZ?*~7HJA6^t)D;_R#@MGU$_KXh5&AXx)9|<7(8p$e^@NTt6dmv0{n|z&z|-LM<%7zo^q0+~ z>268{Z050x2EIV5CoF)1vq0zb9d(BMv zEzXN0^y0I{0O`AmN9ePISJX0~!du4=dv8il8APQ2c6n?nO+cCzjosdmg`2f{v`th* zdesiiXooSUAfk~WLunEVDyVm&ahNK8$1xNTZ%6X6meY|HNCW%2)IAVHQ{wfKnX%wP zGC15T_qbuVBx}Yfd10}bH7{I|Gq_7->JMUrTLLI5+!-RxOchB-1SfY?P?mj1g=BmQ zh1Z&qzc84yVlmM*0a==$D}8tGnn+=)NQcF2?W)tbY*}*3vTr$#$tB?wBE&39TEyw_ zAegbBKbk23C#=Pyb30L<1C+F3m^380^??ZB6I{TpD>w@1VNiC1;bCwe;Iu z%KTd*kI-*9mh&UDKPNbq^M4NDa@P_1)4`t(-wlu<=9Hv(X$0p89ey@2lHk62abPu$ zMXq)_U8OhJTe|eBvIXecWUF4wUR2kKK0NUIgu%Tw)&42OF`u=~u$@N8&!#3*^n`hq z-Sq-VvOCII1{JC9FsFL4^}}(=xa>!`-$!!j(+~c6Mx`@O`yry`E;S|Lf&UFIX}nVE zwjb!butUEA{LNeh=>uMAC3A74&gVpD<7pA?cO^jD1Tg-y*+g>C$;PZ*5Q!5kaRZuT z|8TVzuHd0-Z-jX=(Wya`m0PFP{Y@q*2V0qDGY9n*e)B`-Z`U^OQqH<5we^-)11gQ+ zF2|i@H|<%?_I8ExyBzkK%7JR$QO>i=6vyN>zasi#x7}Qn=~X~)@V^|ySNP3w znzH*gYPIqA*+HxE_l7Qtg}3^ELB{5z+TzpOr!~a1d!{$wGpMmfS++LAuAAX_mg#V8 zlISXRoIsf0TfY4+^RINpe4xNsj#~^9j372}QrafqZiQ7TEneNoIGZhLp{w5J$~4a# zxSK7gH}^LOE1XU3__YbeL!Hvbl?IF(dd3Y6yU*F7{o{L> z&(G@rIkHc|qhCs>REIt`{z~twVFT`PzB-qv>4JEX5@fRJLV4954qIey6h)10+cR>^ zpDGIRlmUQkID&@VpILJRQK5-qHK@09zcbk_%WV%+u)qwbt6%)cLC*eTFBE z9ZsF%#?W(PbC2$WSw`qeR_-dU+S!}uWN06>XhC8PTlFsA(~R! z!{(E;HQnqj9i<&SPk*V4vVZL(u|JY?@T5m^4?ZS)JiqjJ>Lk5ihXpj~b8=I9I|i+0 z^E2=YDi-*tcDZzACCtF~zhsHYy6f!!wax#hFwp}#oe^v{ddQyZGh$UNVf{0xIB$rn z5thvbuS2c~=w!I=+O0x-9}_cDV2wiOP3sw7C9AT!v^tyTyyYOyfB zR3;?OVgg35P32jI?BDy_=Elf%DhN={I0n<&Soa_y{u1a_xBKN}432_nHsx4IUc|EC zq>M*&Ao2pL_zrord+mx^nsJ`1`FBTbQU5G z(-`WMdO6|2siU@_5~75UW;4XfSqdFMl+SU`_xPRk1dO*wneb?hv=(XC_sh_Zbxg2O}%Etp>ezX4JlK_n8LLL%jB zB9U!q6_6PZ^46o$6gP7!koy0HQtcRm5Xevs-5eM;4S?q?5hy7j?rWz-0fn8l*MxL) zCU^=AY!0hOS6E!G5Up7Xe((75>h*vAG|)b<{s4R2#cT7S-JSFyv>hXcLJ8r|fWkey z`DGUhgC6bgyR<>(H9%6tQtbOF9ICZAWqh$;sx!QQceu~VlyH*8WFO~rf0nWdPDgXO zNI&x1>QJi~F*HT6`|!8S7Q;`*QLo08M|Q9!97g6JFCOoSU0}Kevm|*bMx@7IPjp<7 zi*lXsK;>xd5-rt6L#IxBsX%i&o2N^k2Xuf)Du@Z4=+05b`ekb?qxURb5`i+dZ3g~C z?6@EgEwdjgJn?nR1AagvCf#9Rvh-4cq)AgdS)v*9VF)bRlAsI~Gk5EZ@HithY8&?k zkGm=r*sDo%JFBXwqgX{$+)(*GAS){tgwq(MY=+=BZHpQs)I?b} zGE+aRVyL>gu#kpczNuev5S7 zS+T~LYcz$HcaG&;>22HStER!9*;@LxU{&+rR~Gi)*@388#pqCt(61?*4OuS5Xo24h zA)(hNsPvtn|0KVndsd{0TAeO-Yz{%p7U1=~C1z;79R)x~M-)Px1qQr7tqIoP=*odc z=-F&TVqk9GgOd#2#FPC-;EqL7a0#+3H(pBe>sU zN$zoVw=}fO?2nK-HJ1Wm9qE*FER|891H~?xwN9ySdTFl_XJJ6ES5%;kJaGQaH+$RM z=Z-UD@91$c8~ZMe$g2JoBXc~}*JssM2mt}Lm$oq2w6vao1}#g@u>J)?XU+it>C%xc z?aBofiAmsKx0hXIIM}Q4{XB&Z+CsgkJ`YWn`0WQEE^0aG4RgO!E><%)3r@%Spwsct z?t^Lz(LbR`IcbsFHQ-;OpZQ)x@mN`X3?d*+eF)_NzSQX(lmxT~ni>u78ssh7-yLI6 z0_d)7*|;7Rq9D8~@80!Nzv^YS&g}F<_#`t&vl%BsjAbesT&#?le7C0P1QhN_1rl?+ z)tEl+Ov%7&WXI~+ZlLCACZDrHlC3cTpl!&48YNF8{VZSFQb5fPrywyW2L8`Rdj~C_ zKBxrvhMJOqkUg}c_8VhN`kfC5H{a%*OzE2qmTWVZ8a-XePt1vgwLI1sAJP3)pRV;3 z>AEO(h#iw25LhZXQ=MBASYMZ00eH`#Ctyk=8~o1PR2at4-BA-}jnUUiy)+z%=%)I! zj}SiM`G^@r`>@9u8&5G!TT0?g^%>LJe6V1<`0;*5ZS2L4Ip6TKtO5S>9)0)JH*jGS zX=pe5F!Jeg;W2LW2I(Sox|OzpOiX(%4u_OHMnuMP<2s{xPUI?vgg0--LiY)c z>>178I5q|6X>)?X{M;p@*$91ieRI91_ymWehze(WS6Bn^Mh0HxP`X6R(9m*(-Zq~x z_8Exc?^~47EFqahpFaYxsq~WB#%U^p{D$$I-DkEcmA;p{vx8K}S9E4BKph^A4quF3 z9K1LiyT;gbkKnF|u0w?LJVKEKJ1tTo$1;*h^A*iz+rK<=>^C_x z3ZNup$|am5B7Kn9%o}~wI8IEH`Wa>8StKkZ{fKM&kECC%geHW~Ake;|MyjRyd6q|A zkGkiQZn`hk@2foOdh|VVoc#}06!n3ldf&``zok&urt;aa>vvQD_2FZCWUhB5q5l^y zd%7L&yuZ2oQ0(-NTkLe>*?;eKPWx)ZCoXZi4q5MM6MPnhP1m6D{b|1ETh8>MwD-C6 z{D&=Kx~(z3M|0|#l`ee%#qZIsc^?H!*CFcxi}ZI=ru3l$c>srgU&TqcGoELtu%3HK z(yb5ecQXgSmqMhEHt^?L+;a7hpMUw$2ae&7IXSga6)47;ez#|` zw_z?RPSd5UxRK2~5&b5&=V-l#^8-OY2gCSf+f(f42U(2O??WzDRbl72!YeBsbGC3& zvv0r+J$LsX&fdqer3KAqDe*XAxcapaM*2rT?q0*FLJaPgcn0dnanY2e3E>g?*!}Wo zqTw@3eFI1#x_CVlP-9Ta|cW$1hWz% z80xRs)ZAi}%uVm^l%}v@`#X+wYBTC3{T}3@X&x$grG!G=;?!G+0uAIF457iko8xq8sL<(k6eGdX z-(>!pXYFST^NHuCosBx*=UBuX=VUDDg0Mm=vyNp+iM`2x6+$A@z~r!y4C#)DW|Htb zoT7f(ye|O(7tJqUZJTj7I^n4u7}N;YMVmp@T`0r(cnbQjmdQ#DI>V{CXIsjuKDPPS zcXrU$12!-+82&=AW?fVagY3v+;227W{| z<*H+rL@4rWD?*VCFxqVN$R~?Oz7AVWB7f1PQR@$q~);WV*n0d{uKiCRN0^MG9DkTCVC>y5fek*E{s zGLAG%2cx#GD(f|ng@)Bynh8b9H<+fdvAfYLqG+a%aY>??u~Qo|&X#&bmu39x5I#sZ z&X%aiV#X7ekwmX{n%UuON50E%9z2m&YC4I8Y%`_-fgSfjL^(-qBg%{VI%KJFnxsil zLK1l3os7aq`$s5TL^w;@8`=dXG3}i-u4T9Z&b9&DWxnm_lap6lINs1aRRBc1-#X^#^&Ye*Jy#? zpdu(9)61(3;cfLO)~l6iI(?Z2U4T*HsZi_kt(w$e zstH{G40FBK9{KqkuY#ED=~(EMR<3zLr^+9p{bKg3X|Vx_FFgSzd|#oTU7|==tiV?F zTOH*2+vTw@Pf;Go&*TkWp9)!h@n1_Q73);8d5SYKbOZ52*5SCb)^XIl z^+vuiPG?kUphuWYwY`&`PodW!F6x-q*0uX==B&@8+~s&IauPR9!(EP%`mK%oP_$D= z$6VgB&1b+;;#QOajdN-q#((9$yQNv;KS4$m_sN|-f7LO0tHUxH*zx7Lubbh&{wvLr z@biT|9l5E{;LlBdS5gt@g)#=D7)v%@&@`nn z@}+~Jm5#jz(24M|PEq}Bk)C6XYlU(*A8`-lTeZ}#l|uc7@s(vT)JB9eNype6E1i9A}c+lN9}vKg_6|P^m$-QahO-hzb3&c_%p6QfVYttp-bz)MLL!tQ@u^~ z=kc6M0;R^p2%YOYRxa)y{%Udd#x1K1Xr^dVO)w)U>WI8Q<5tv}%T30c}Ok9EWifdr==sy2HVN4y*K8YipAl zJ&J~HB*Ua}U7vAb%8&QP#2(bwWZX86{6EKXnG-MK?R") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Kind name portion - - -### helm_lib_get_api_version_by_kind - - returns current apiVersion string, based on available helm capabilities, for the provided kind (not all kinds are supported) - -#### Usage - -`{{ include "helm_lib_get_api_version_by_kind" (list . "") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Kind name portion - -## Application Image - -### helm_lib_application_image - - returns image name in format "registry/package@digest" - -#### Usage - -`{{ include "helm_lib_application_image" (list . "") }} ` - - -## Application Security Context - -### helm_lib_application_pod_security_context_run_as_user_custom - - returns PodSecurityContext parameters for Pod with custom user and group - -#### Usage - -`{{ include "helm_lib_application_pod_security_context_run_as_user_custom" (list . 1000 1000) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- User id -- Group id - - -### helm_lib_v_pod_security_context_run_as_user_nobody - - returns PodSecurityContext parameters for Pod with user and group "nobody" - -#### Usage - -`{{ include "helm_lib_application_pod_security_context_run_as_user_nobody" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_pod_security_context_run_as_user_nobody_with_writable_fs - - returns PodSecurityContext parameters for Pod with user and group "nobody" with write access to mounted volumes - -#### Usage - -`{{ include "helm_lib_application_pod_security_context_run_as_user_nobody_with_writable_fs" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_pod_security_context_run_as_user_deckhouse - - returns PodSecurityContext parameters for Pod with user and group "deckhouse" - -#### Usage - -`{{ include "helm_lib_application_pod_security_context_run_as_user_deckhouse" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_pod_security_context_run_as_user_deckhouse_with_writable_fs - - returns PodSecurityContext parameters for Pod with user and group "deckhouse" with write access to mounted volumes - -#### Usage - -`{{ include "helm_lib_application_pod_security_context_run_as_user_deckhouse_with_writable_fs" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_container_security_context_run_as_user_deckhouse_pss_restricted - - returns SecurityContext parameters for Container with user and group "deckhouse" plus minimal required settings to comply with the Restricted mode of the Pod Security Standards - -#### Usage - -`{{ include "helm_lib_application_container_security_context_run_as_user_deckhouse_pss_restricted" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_container_security_context_pss_restricted_flexible - - SecurityContext for Deckhouse UID/GID 64535 (or root), PSS Restricted - Optional keys: - .ro – bool, read-only root FS (default true) - .caps – []string, capabilities.add (default empty) - .uid – int, runAsUser/runAsGroup (default 64535) - .runAsNonRoot – bool, run as Deckhouse user when true, root when false (default true) - .seccompProfile – bool, disable seccompProfile when false (default true) - -#### Usage - -`include "helm_lib_application_container_security_context_pss_restricted_flexible" (dict "ro" false "caps" (list "NET_ADMIN" "SYS_TIME") "uid" 1001 "seccompProfile" false "runAsNonRoot" true) ` - - - -### helm_lib_application_pod_security_context_run_as_user_root - - returns PodSecurityContext parameters for Pod with user and group 0 - -#### Usage - -`{{ include "helm_lib_application_pod_security_context_run_as_user_root" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_pod_security_context_runtime_default - - returns PodSecurityContext parameters for Pod with seccomp profile RuntimeDefault - -#### Usage - -`{{ include "helm_lib_application_pod_security_context_runtime_default" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_container_security_context_not_allow_privilege_escalation - - returns SecurityContext parameters for Container with allowPrivilegeEscalation false - -#### Usage - -`{{ include "helm_lib_application_container_security_context_not_allow_privilege_escalation" . }} ` - - - -### helm_lib_application_container_security_context_read_only_root_filesystem_with_selinux - - returns SecurityContext parameters for Container with read only root filesystem and options for SELinux compatibility - -#### Usage - -`{{ include "helm_lib_application_container_security_context_read_only_root_filesystem_with_selinux" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_container_security_context_read_only_root_filesystem - - returns SecurityContext parameters for Container with read only root filesystem - -#### Usage - -`{{ include "helm_lib_application_container_security_context_read_only_root_filesystem" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_container_security_context_privileged - - returns SecurityContext parameters for Container running privileged - -#### Usage - -`{{ include "helm_lib_application_container_security_context_privileged" . }} ` - - - -### helm_lib_application_container_security_context_escalated_sys_admin_privileged - - returns SecurityContext parameters for Container running privileged with escalation and sys_admin - -#### Usage - -`{{ include "helm_lib_application_container_security_context_escalated_sys_admin_privileged" . }} ` - - - -### helm_lib_application_container_security_context_privileged_read_only_root_filesystem - - returns SecurityContext parameters for Container running privileged with read only root filesystem - -#### Usage - -`{{ include "helm_lib_application_container_security_context_privileged_read_only_root_filesystem" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all - - returns SecurityContext for Container with read only root filesystem and all capabilities dropped - -#### Usage - -`{{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add - - returns SecurityContext parameters for Container with read only root filesystem, all dropped and some added capabilities - -#### Usage - -`{{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- List of capabilities - - -### helm_lib_application_container_security_context_capabilities_drop_all_and_add - - returns SecurityContext parameters for Container with all dropped and some added capabilities - -#### Usage - -`{{ include "helm_lib_application_container_security_context_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- List of capabilities - - -### helm_lib_application_container_security_context_capabilities_drop_all_and_run_as_user_custom - - returns SecurityContext parameters for Container with read only root filesystem, all dropped, and custom user ID - -#### Usage - -`{{ include "helm_lib_application_container_security_context_capabilities_drop_all_and_run_as_user_custom" (list . 1000 1000) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- User id -- Group id - - -### helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted - - returns SecurityContext parameters for Container with minimal required settings to comply with the Restricted mode of the Pod Security Standards - -#### Usage - -`{{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - -## Capi Controller Manager - -### helm_lib_capi_controller_manager_manifests - - Renders common manifests for provider-specific CAPI Controller Managers. - Includes Deployment, VerticalPodAutoscaler (optional) and PodDisruptionBudget (optional). - Supported configuration parameters: - + fullname (required) — resource base name used for Deployment, PDB, VPA, and by default for the main container name. - + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. - + image (required) — image for the main container. - + capiProviderName (required) — value for the cluster.x-k8s.io/provider label in selectors and pod labels. - + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. - + priorityClassName (optional, default: `"system-cluster-critical"`) — Pod priority class name. - + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. - + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. - + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. - + terminationGracePeriodSeconds (optional, default: `10`) — Pod termination grace period. - + hostNetwork (optional, default: `false`) — enables host networking for the Pod. - + dnsPolicy (optional, default: `nil`) — Pod DNS policy; if not set, the field is omitted. - + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. - + tolerationsStrategies (optional, default: `["any-node", "uninitialized"]`) — arguments passed to helm_lib_tolerations. - + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 8081}, initialDelaySeconds: 15, periodSeconds: 20}`) — liveness probe configuration for the main container. - + readinessProbe (optional, default: `{httpGet: {path: /readyz, port: 8081}, initialDelaySeconds: 5, periodSeconds: 10}`) — readiness probe configuration for the main container. - + additionalArgs (optional, default: `[]`) — extra args for the main container. - + additionalEnv (optional, default: `[]`) — extra environment variables for the main container. - + additionalPorts (optional, default: `[]`) — extra container ports for the main container. - + additionalInitContainers (optional, default: `[]`) — extra initContainers for the Pod. - + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. - + additionalVolumes (optional, default: `[]`) — extra Pod volumes. - + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. - + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. - + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. - + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. - + vpaEnabled (optional, default: `false`) — enables VerticalPodAutoscaler rendering. - + vpaUpdateMode (optional, default: `"InPlaceOrRecreate"`) — VPA update mode. - + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. - + securityPolicyExceptionEnabled (optional, default: `false`) — enables SecurityPolicyException rendering and adds the related pod label. - -#### Usage - -`{{ include "helm_lib_capi_controller_manager_manifests" (list . $config) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc. -- Configuration dict for the CAPI Controller Manager. - -## Cloud Controller Manager - -### helm_lib_cloud_controller_manager_manifests - - Renders common manifests for provider-specific Cloud Controller Managers. - Includes Deployment, VerticalPodAutoscaler (optional), PodDisruptionBudget (optional), and SecurityPolicyException (optional). - Supported configuration parameters: - + fullname (optional, default: `"cloud-controller-manager"`) — resource base name used for Deployment, PDB, VPA, SecurityPolicyException, and the main container name by default. - + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. - + image (required) — image for the main container. - + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. - + priorityClassName (optional, default: `"system-cluster-critical"`) — Pod priority class name. - + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. - + tolerationsStrategies (optional, default: ["wildcard"]) — strategies passed to helm_lib_tolerations. - + hostNetwork (optional, default: `true`) — enables host networking for the Pod and SecurityPolicyException network rule generation. - + dnsPolicy (optional, default: `"Default"`) — Pod DNS policy. - + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. - + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. - + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. - + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 10471, host: 127.0.0.1, scheme: HTTPS}}`) — liveness probe configuration for the main container. - + readinessProbe (optional, default: `{httpGet: {path: /healthz, port: 10471, host: 127.0.0.1, scheme: HTTPS}}`) — readiness probe configuration for the main container. - + additionalEnvs (optional, default: `[]`) — extra environment variables for the main container. - + additionalArgs (optional, default: `nil`) — extra args for the main container. - + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. - + additionalVolumes (optional, default: `[]`) — extra Pod volumes; hostPath volumes are also used to build SecurityPolicyException rules when enabled. - + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. - + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. - + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. - + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. - + additionalPDBAnnotations (optional, default: `{}`) — extra annotations added to PodDisruptionBudget metadata. - + vpaEnabled (optional, default: `true`) — enables VerticalPodAutoscaler rendering. - + vpaUpdateMode (optional, default: `"InPlaceOrRecreate"`) — VPA update mode. - + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. - + securityPolicyExceptionEnabled (optional, default: `false`) — enables SecurityPolicyException rendering and adds the related pod label. - -#### Usage - -`{{ include "helm_lib_cloud_controller_manager_manifests" (list . $config) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc. -- Configuration dict for the Cloud Controller Manager. - -## Cloud Data Discoverer - -### helm_lib_cloud_data_discoverer_manifests - - Renders common manifests for provider-specific Cloud Data Discoverers. - Includes Deployment, VerticalPodAutoscaler (optional) and PodDisruptionBudget (optional). - Supported configuration parameters: - + fullname (optional, default: `"cloud-data-discoverer"`) — resource base name used for Deployment, PDB, VPA, and the main container name by default. - + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. - + image (required) — image for the main container. - + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. - + replicas (optional, default: `1`) — number of Deployment replicas. - + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. - + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. - + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. - + priorityClassName (optional, default: `"cluster-low"`) — Pod priority class name. - + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. - + tolerationsStrategies (optional, default: `["any-node", "with-uninitialized"]`) — strategies passed to helm_lib_tolerations. - + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 8080, scheme: HTTPS}}`) — liveness probe configuration for the main container. - + readinessProbe (optional, default: `{httpGet: {path: /healthz, port: 8080, scheme: HTTPS}}`) — readiness probe configuration for the main container. - + additionalArgs (optional, default: `[]`) — extra args for the main container. - + additionalEnv (optional, default: `[]`) — extra environment variables for the main container. - + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. - + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. - + additionalInitContainers (optional, default: `[]`) — extra initContainers for the Pod. - + additionalVolumes (optional, default: `[]`) — extra Pod volumes. - + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. - + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. - + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. - + vpaEnabled (optional, default: `true`) — enables VerticalPodAutoscaler rendering. - + vpaUpdateMode (optional, default: `"Initial"`) — VPA update mode. - + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. - -#### Usage - -`{{ include "helm_lib_cloud_data_discoverer_manifests" (list . $config) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc. -- Configuration dict for the Cloud Data Discoverer. - - -### helm_lib_cloud_data_discoverer_pod_monitor - - Renders PodMonitor manifest for provider-specific Cloud Data Discoverers. - Supported configuration parameters: - + fullname (optional, default: `"cloud-data-discoverer"`) — PodMonitor base name. - + targetNamespace (required) — target pod namespace for selector. - + additionalRelabelings (optional, default: `[]`) — additional rules for labels rewriting. - -#### Usage - -`{{ include "helm_lib_cloud_data_discoverer_pod_monitor" (list . $config) }} ` - - -## Cloud Provider User Authz Roles - -### helm_lib_cloud_provider_user_authz_cluster_roles - - Renders user-authz ClusterRoles for provider-specific cloud resources. - Includes User and ClusterAdmin ClusterRoles. - Supported configuration parameters: - + providerName (required) — provider name segment used in ClusterRole names. - + instanceClassResource (required) — Deckhouse instance class resource name granted by the rules. - + capiResources (optional, default: `[]`) — CAPI infrastructure resource names granted by the rules. - + additionalUserRules (optional, default: `[]`) — extra rules appended to the User ClusterRole. - + additionalClusterAdminRules (optional, default: `[]`) — extra rules appended to the ClusterAdmin ClusterRole. - -#### Usage - -`{{- include "helm_lib_cloud_provider_user_authz_cluster_roles" (list . $config) }} ` - - -## Csi Controller - -### helm_lib_csi_image_with_common_fallback - - returns image name from storage foundation module if enabled, otherwise from common module - -#### Usage - -`{{ include "helm_lib_csi_image_with_common_fallback" (list . "" "") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Container raw name -- Kubernetes semantic version - -## Dns Policy - -### helm_lib_dns_policy_bootstraping_state - - returns the proper dnsPolicy value depending on the cluster bootstrap phase - -#### Usage - -`{{ include "helm_lib_dns_policy_bootstraping_state" (list . "Default" "ClusterFirstWithHostNet") }} ` - - -## Enable Ds Eviction - -### helm_lib_prevent_ds_eviction_annotation - - Adds `cluster-autoscaler.kubernetes.io/enable-ds-eviction` annotation to manage DaemonSet eviction by the Cluster Autoscaler. - This is important to prevent the eviction of DaemonSet pods during cluster scaling. - -#### Usage - -`{{ include "helm_lib_prevent_ds_eviction_annotation" . }} ` - - -## Envs For Proxy - -### helm_lib_envs_for_proxy - - Add HTTP_PROXY, HTTPS_PROXY and NO_PROXY environment variables for container - depends on [proxy settings](https://deckhouse.io/products/kubernetes-platform/documentation/v1/reference/api/global.html#parameters-modules-proxy) - -#### Usage - -`{{ include "helm_lib_envs_for_proxy" . }} or {{ include "helm_lib_envs_for_proxy" (list . (list "extra1" "extra2")) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- List of additional NO_PROXY entries (optional) - -## High Availability - -### helm_lib_is_ha_to_value - - returns value "yes" if cluster is highly available, else — returns "no" - -#### Usage - -`{{ include "helm_lib_is_ha_to_value" (list . yes no) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Yes value -- No value - - -### helm_lib_ha_enabled - - returns empty value, which is treated by go template as false - -#### Usage - -`{{- if (include "helm_lib_ha_enabled" .) }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - -## Kube Rbac Proxy - -### helm_lib_kube_rbac_proxy_ca_certificate - - Renders configmap with kube-rbac-proxy CA certificate which uses to verify the kube-rbac-proxy clients. - -#### Usage - -`{{ include "helm_lib_kube_rbac_proxy_ca_certificate" (list . "namespace") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Namespace where CA configmap will be created - -## Module Controller - -### helm_lib_module_controller_resources - - Returns default controller resources - -#### Usage - -`{{ include "helm_lib_module_controller_resources" . }} ` - - - -### helm_lib_module_webhooks_resources - - Returns default webhooks resources - -#### Usage - -`{{ include "helm_lib_module_webhooks_resources" . }} ` - - - -### helm_lib_module_controller_log_level - - Returns numeric log level from module values - -#### Usage - -`{{ include "helm_lib_module_controller_log_level" (list . "csiHpe") }} ` - - -## Module Documentation Uri - -### helm_lib_module_documentation_uri - - returns rendered documentation uri using publicDomainTemplate or deckhouse.io domains - -#### Usage - -`{{ include "helm_lib_module_documentation_uri" (list . "") }} ` - - -## Module Ephemeral Storage - -### helm_lib_module_ephemeral_storage_logs_with_extra - - 50Mi for container logs `log-opts.max-file * log-opts.max-size` would be added to passed value - returns ephemeral-storage size for logs with extra space - -#### Usage - -`{{ include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 }} ` - -#### Arguments - -- Extra space in mebibytes - - -### helm_lib_module_ephemeral_storage_only_logs - - 50Mi for container logs `log-opts.max-file * log-opts.max-size` would be requested - returns ephemeral-storage size for only logs - -#### Usage - -`{{ include "helm_lib_module_ephemeral_storage_only_logs" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - -## Module Gateway - -### helm_lib_module_gateway - - accepts a dict that is updated with current gateway name and namespace - -#### Usage - -`{{- include "helm_lib_module_gateway" (list . $gateway) ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- An empty dict to update with current default gateway name and namespace - -## Module Generate Common Name - -### helm_lib_module_generate_common_name - - returns the commonName parameter for use in the Certificate custom resource(cert-manager) - -#### Usage - -`{{ include "helm_lib_module_generate_common_name" (list . "") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Name portion - -## Module Https - -### helm_lib_module_uri_scheme - - return module uri scheme "http" or "https" - -#### Usage - -`{{ include "helm_lib_module_uri_scheme" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_https_mode - - returns https mode for module - -#### Usage - -`{{ if (include "helm_lib_module_https_mode" .) }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_https_cert_manager_cluster_issuer_name - - returns cluster issuer name - -#### Usage - -`{{ include "helm_lib_module_https_cert_manager_cluster_issuer_name" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_https_ingress_tls_enabled - - returns not empty string if tls should be enabled for the ingress - -#### Usage - -`{{ if (include "helm_lib_module_https_ingress_tls_enabled" .) }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_https_route_tls_enabled - - returns not empty string if tls should be enabled for the route - -#### Usage - -`{{ if (include "helm_lib_module_https_route_tls_enabled" .) }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_https_copy_custom_certificate - - Renders secret with [custom certificate](https://deckhouse.io/products/kubernetes-platform/documentation/v1/reference/api/global.html#parameters-modules-https-customcertificate) - in passed namespace with passed prefix - -#### Usage - -`{{ include "helm_lib_module_https_copy_custom_certificate" (list . "namespace" "secret_name_prefix") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Namespace -- Secret name prefix - - -### helm_lib_module_https_secret_name - - returns custom certificate name - -#### Usage - -`{{ include "helm_lib_module_https_secret_name (list . "secret_name_prefix") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Secret name prefix - -## Module Image - -### helm_lib_module_image - - returns image name - -#### Usage - -`{{ include "helm_lib_module_image" (list . "" "(optional)") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Container name - - -### helm_lib_module_image_no_fail - - returns image name if found - -#### Usage - -`{{ include "helm_lib_module_image_no_fail" (list . "") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Container name - - -### helm_lib_module_common_image - - returns image name from common module - -#### Usage - -`{{ include "helm_lib_module_common_image" (list . "") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Container name - - -### helm_lib_module_common_image_no_fail - - returns image name from common module if found - -#### Usage - -`{{ include "helm_lib_module_common_image_no_fail" (list . "") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Container name - - -### helm_lib_module_image_digest - - returns image digest - -#### Usage - -`{{ include "helm_lib_module_image_digest" (list . "" "(optional)") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Container name - - -### helm_lib_module_image_digest_no_fail - - returns image digest if found - -#### Usage - -`{{ include "helm_lib_module_image_digest_no_fail" (list . "" "(optional)") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Container name - -## Module Ingress Class - -### helm_lib_module_ingress_class - - returns ingress class from module settings or if not exists from global config - -#### Usage - -`{{ include "helm_lib_module_ingress_class" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - -## Module Ingress Snippets - -### helm_lib_module_ingress_configuration_snippet - - returns nginx ingress additional headers (e.g. HSTS) if HTTPS is enabled - -#### Usage - -`nginx.ingress.kubernetes.io/configuration-snippet: | {{ include "helm_lib_module_ingress_configuration_snippet" . | nindent 6 }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - -## Module Init Container - -### helm_lib_module_init_container_chown_nobody_volume - - ### Migration 11.12.2020: Remove this helper with all its usages after this commit reached RockSolid - returns initContainer which chowns recursively all files and directories in passed volume - -#### Usage - -`{{ include "helm_lib_module_init_container_chown_nobody_volume" (list . "volume-name") }} ` - - - -### helm_lib_module_init_container_chown_deckhouse_volume - - returns initContainer which chowns recursively all files and directories in passed volume - -#### Usage - -`{{ include "helm_lib_module_init_container_chown_deckhouse_volume" (list . "volume-name") }} ` - - - -### helm_lib_module_init_container_check_linux_kernel - - returns initContainer which checks the kernel version on the node for compliance to semver constraint - -#### Usage - -`{{ include "helm_lib_module_init_container_check_linux_kernel" (list . ">= 4.9.17") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Semver constraint - - -### helm_lib_module_init_container_iptables_wrapper - - returns initContainer with iptables-wrapper - -#### Usage - -`{{ include "helm_lib_module_init_container_iptables_wrapper" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - -## Module Labels - -### helm_lib_module_labels - - returns deckhouse labels - -#### Usage - -`{{ include "helm_lib_module_labels" (list . (dict "app" "test" "component" "testing")) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Additional labels dict - -## Module Public Domain - -### helm_lib_module_public_domain - - returns rendered publicDomainTemplate to service fqdn - -#### Usage - -`{{ include "helm_lib_module_public_domain" (list . "") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Name portion - -## Module Security Context - -### helm_lib_module_pod_security_context_run_as_user_custom - - returns PodSecurityContext parameters for Pod with custom user and group - -#### Usage - -`{{ include "helm_lib_module_pod_security_context_run_as_user_custom" (list . 1000 1000) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- User id -- Group id - - -### helm_lib_module_pod_security_context_run_as_user_nobody - - returns PodSecurityContext parameters for Pod with user and group "nobody" - -#### Usage - -`{{ include "helm_lib_module_pod_security_context_run_as_user_nobody" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_pod_security_context_run_as_user_nobody_with_writable_fs - - returns PodSecurityContext parameters for Pod with user and group "nobody" with write access to mounted volumes - -#### Usage - -`{{ include "helm_lib_module_pod_security_context_run_as_user_nobody_with_writable_fs" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_pod_security_context_run_as_user_deckhouse - - returns PodSecurityContext parameters for Pod with user and group "deckhouse" - -#### Usage - -`{{ include "helm_lib_module_pod_security_context_run_as_user_deckhouse" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_pod_security_context_run_as_user_deckhouse_with_writable_fs - - returns PodSecurityContext parameters for Pod with user and group "deckhouse" with write access to mounted volumes - -#### Usage - -`{{ include "helm_lib_module_pod_security_context_run_as_user_deckhouse_with_writable_fs" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_container_security_context_run_as_user_deckhouse_pss_restricted - - returns SecurityContext parameters for Container with user and group "deckhouse" plus minimal required settings to comply with the Restricted mode of the Pod Security Standards - -#### Usage - -`{{ include "helm_lib_module_container_security_context_run_as_user_deckhouse_pss_restricted" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_container_security_context_pss_restricted_flexible - - SecurityContext for Deckhouse UID/GID 64535 (or root), PSS Restricted - Optional keys: - .ro – bool, read-only root FS (default true) - .caps – []string, capabilities.add (default empty) - .uid – int, runAsUser/runAsGroup (default 64535) - .runAsNonRoot – bool, run as Deckhouse user when true, root when false (default true) - .seccompProfile – bool, disable seccompProfile when false (default true) - -#### Usage - -`include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" false "caps" (list "NET_ADMIN" "SYS_TIME") "uid" 1001 "seccompProfile" false "runAsNonRoot" true) ` - - - -### helm_lib_module_pod_security_context_run_as_user_root - - returns PodSecurityContext parameters for Pod with user and group 0 - -#### Usage - -`{{ include "helm_lib_module_pod_security_context_run_as_user_root" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_pod_security_context_runtime_default - - returns PodSecurityContext parameters for Pod with seccomp profile RuntimeDefault - -#### Usage - -`{{ include "helm_lib_module_pod_security_context_runtime_default" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_container_security_context_not_allow_privilege_escalation - - returns SecurityContext parameters for Container with allowPrivilegeEscalation false - -#### Usage - -`{{ include "helm_lib_module_container_security_context_not_allow_privilege_escalation" . }} ` - - - -### helm_lib_module_container_security_context_read_only_root_filesystem_with_selinux - - returns SecurityContext parameters for Container with read only root filesystem and options for SELinux compatibility - -#### Usage - -`{{ include "helm_lib_module_container_security_context_read_only_root_filesystem_with_selinux" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_container_security_context_read_only_root_filesystem - - returns SecurityContext parameters for Container with read only root filesystem - -#### Usage - -`{{ include "helm_lib_module_container_security_context_read_only_root_filesystem" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_container_security_context_privileged - - returns SecurityContext parameters for Container running privileged - -#### Usage - -`{{ include "helm_lib_module_container_security_context_privileged" . }} ` - - - -### helm_lib_module_container_security_context_escalated_sys_admin_privileged - - returns SecurityContext parameters for Container running privileged with escalation and sys_admin - -#### Usage - -`{{ include "helm_lib_module_container_security_context_escalated_sys_admin_privileged" . }} ` - - - -### helm_lib_module_container_security_context_privileged_read_only_root_filesystem - - returns SecurityContext parameters for Container running privileged with read only root filesystem - -#### Usage - -`{{ include "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all - - returns SecurityContext for Container with read only root filesystem and all capabilities dropped - -#### Usage - -`{{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add - - returns SecurityContext parameters for Container with read only root filesystem, all dropped and some added capabilities - -#### Usage - -`{{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- List of capabilities - - -### helm_lib_module_container_security_context_capabilities_drop_all_and_add - - returns SecurityContext parameters for Container with all dropped and some added capabilities - -#### Usage - -`{{ include "helm_lib_module_container_security_context_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- List of capabilities - - -### helm_lib_module_container_security_context_capabilities_drop_all_and_run_as_user_custom - - returns SecurityContext parameters for Container with read only root filesystem, all dropped, and custom user ID - -#### Usage - -`{{ include "helm_lib_module_container_security_context_capabilities_drop_all_and_run_as_user_custom" (list . 1000 1000) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- User id -- Group id - - -### helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted - - returns SecurityContext parameters for Container with minimal required settings to comply with the Restricted mode of the Pod Security Standards - -#### Usage - -`{{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - -## Module Storage Class - -### helm_lib_module_storage_class_annotations - - return module StorageClass annotations - -#### Usage - -`{{ include "helm_lib_module_storage_class_annotations" (list $ $index $storageClass.name) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Storage class index -- Storage class name - -## Monitoring Grafana Dashboards - -### helm_lib_grafana_dashboard_definitions_recursion - - returns all the dashboard-definintions from / - current dir is optional — used for recursion but you can use it for partially generating dashboards - -#### Usage - -`{{ include "helm_lib_grafana_dashboard_definitions_recursion" (list . [current dir]) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Dashboards root dir -- Dashboards current dir - - -### helm_lib_grafana_dashboard_definitions - - returns dashboard-definintions from monitoring/grafana-dashboards/ - -#### Usage - -`{{ include "helm_lib_grafana_dashboard_definitions" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_single_dashboard - - renders a single dashboard - -#### Usage - -`{{ include "helm_lib_single_dashboard" (list . "dashboard-name" "folder" $dashboard) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Dashboard name -- Folder -- Dashboard definition - -## Monitoring Prometheus Rules - -### helm_lib_prometheus_rules_recursion - - returns all the prometheus rules from / - current dir is optional — used for recursion but you can use it for partially generating rules - file list is optional - list of files to include (filters all files if provided) - -#### Usage - -`{{ include "helm_lib_prometheus_rules_recursion" (list . [current dir] [file list]) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Namespace for creating rules -- Rules root dir -- Current dir (optional) -- File list for filtering (optional) - - -### helm_lib_prometheus_rules - - returns all the prometheus rules from monitoring/prometheus-rules/ optionally filtered by fileList - -#### Usage - -`{{ include "helm_lib_prometheus_rules" (list . [fileList]) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Namespace for creating rules - - -### helm_lib_prometheus_target_scrape_timeout_seconds - - returns adjust timeout value to scrape interval / - -#### Usage - -`{{ include "helm_lib_prometheus_target_scrape_timeout_seconds" (list . ) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Target timeout in seconds - -## Node Affinity - -### helm_lib_internal_check_node_selector_strategy - - Verify node selector strategy. - - - -### helm_lib_node_selector - - Returns node selector for workloads depend on strategy. - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- strategy, one of "frontend" "monitoring" "system" "master" "any-node" "wildcard" - - -### helm_lib_tolerations - - Returns tolerations for workloads depend on strategy. - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "any-node" "with-uninitialized" "without-storage-problems") }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- base strategy, one of "frontend" "monitoring" "system" any-node" "wildcard" -- list of additional strategies. To add strategy list it with prefix "with-", to remove strategy list it with prefix "without-". - - -### _helm_lib_cloud_or_hybrid_cluster - - Check cluster type. - Returns not empty string if this is cloud or hybrid cluster - - - -### helm_lib_internal_check_tolerations_strategy - - Verify base strategy. - Fails if strategy not in allowed list - - - -### _helm_lib_any_node_tolerations - - Base strategy for any uncordoned node in cluster. - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "any-node") }} ` - - - -### _helm_lib_wildcard_tolerations - - Base strategy that tolerates all. - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "wildcard") }} ` - - - -### _helm_lib_monitoring_tolerations - - Base strategy that tolerates nodes with "dedicated.deckhouse.io: monitoring" and "dedicated.deckhouse.io: system" taints. - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "monitoring") }} ` - - - -### _helm_lib_frontend_tolerations - - Base strategy that tolerates nodes with "dedicated.deckhouse.io: frontend" taints. - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "frontend") }} ` - - - -### _helm_lib_system_tolerations - - Base strategy that tolerates nodes with "dedicated.deckhouse.io: system" taints. - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "system") }} ` - - - -### _helm_lib_additional_tolerations_uninitialized - - Additional strategy "uninitialized" - used for CNI's and kube-proxy to allow cni components scheduled on node after CCM initialization. - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "any-node" "with-uninitialized") }} ` - - - -### _helm_lib_additional_tolerations_node_problems - - Additional strategy "node-problems" - used for shedule critical components on non-ready nodes or nodes under pressure. - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "any-node" "with-node-problems") }} ` - - - -### _helm_lib_additional_tolerations_storage_problems - - Additional strategy "storage-problems" - used for shedule critical components on nodes with drbd problems. This additional strategy enabled by default in any base strategy except "wildcard". - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "any-node" "without-storage-problems") }} ` - - - -### _helm_lib_additional_tolerations_no_csi - - Additional strategy "no-csi" - used for any node with no CSI: any node, which was initialized by deckhouse, but have no csi-node driver registered on it. - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "any-node" "with-no-csi") }} ` - - - -### _helm_lib_additional_tolerations_cloud_provider_uninitialized - - Additional strategy "cloud-provider-uninitialized" - used for any node which is not initialized by CCM. - -#### Usage - -`{{ include "helm_lib_tolerations" (tuple . "any-node" "with-cloud-provider-uninitialized") }} ` - - -## Pod Disruption Budget - -### helm_lib_pdb_daemonset - - Returns PDB max unavailable - -#### Usage - -`{{ include "helm_lib_pdb_daemonset" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - -## Priority Class - -### helm_lib_priority_class - - returns priority class - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Priority class name - -## Resources Management - -### helm_lib_resources_management_pod_resources - - returns rendered resources section based on configuration if it is - -#### Usage - -`{{ include "helm_lib_resources_management_pod_resources" (list [ephemeral storage requests]) }} ` - -#### Arguments - -list: -- VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) -- Ephemeral storage requests - - -### helm_lib_resources_management_original_pod_resources - - returns rendered resources section based on configuration if it is present - -#### Usage - -`{{ include "helm_lib_resources_management_original_pod_resources" }} ` - -#### Arguments - -- VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) - - -### helm_lib_resources_management_vpa_spec - - returns rendered vpa spec based on configuration and target reference - -#### Usage - -`{{ include "helm_lib_resources_management_vpa_spec" (list ) }} ` - -#### Arguments - -list: -- Target API version -- Target Kind -- Target Name -- Target container name -- VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) - - -### helm_lib_resources_management_cpu_units_to_millicores - - helper for converting cpu units to millicores - -#### Usage - -`{{ include "helm_lib_resources_management_cpu_units_to_millicores" }} ` - - - -### helm_lib_resources_management_memory_units_to_bytes - - helper for converting memory units to bytes - -#### Usage - -`{{ include "helm_lib_resources_management_memory_units_to_bytes" }} ` - - - -### helm_lib_vpa_kube_rbac_proxy_resources - - helper for VPA resources for kube_rbac_proxy - -#### Usage - -`{{ include "helm_lib_vpa_kube_rbac_proxy_resources" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_container_kube_rbac_proxy_resources - - helper for container resources for kube_rbac_proxy - -#### Usage - -`{{ include "helm_lib_container_kube_rbac_proxy_resources" . }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - -## Spec For High Availability - -### helm_lib_pod_anti_affinity_for_ha - - returns pod affinity spec - -#### Usage - -`{{ include "helm_lib_pod_anti_affinity_for_ha" (list . (dict "app" "test")) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Match labels for podAntiAffinity label selector - - -### helm_lib_pod_affinity - - Returns affinity spec that combines: podAntiAffinity by provided labels when HA is enabled and optional nodeAffinity that schedules pods only on specified architectures. If the list of architectures is not provided or empty, node affinity is not rendered. - -#### Usage - -`{{- include "helm_lib_pod_affinity" (list . (dict "app" "test") (list "amd64")) }} ` - -#### Arguments - -list: -- Template context with .Values, .Chart, etc -- Match labels for podAntiAffinity label selector - - -### helm_lib_deployment_on_master_strategy_and_replicas_for_ha - - returns deployment strategy and replicas for ha components running on master nodes - -#### Usage - -`{{ include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc - - -### helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha - - returns deployment with custom strategy and replicas for ha components running on master nodes - -#### Usage - -`{{ include "helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha" (list . (dict "strategy" "strategy_type")) }} ` - - - -### helm_lib_deployment_strategy_and_replicas_for_ha - - returns deployment strategy and replicas for ha components running not on master nodes - -#### Usage - -`{{ include "helm_lib_deployment_strategy_and_replicas_for_ha" }} ` - -#### Arguments - -- Template context with .Values, .Chart, etc diff --git a/charts/helm_lib/templates/_admission_client_ca.tpl b/charts/helm_lib/templates/_admission_client_ca.tpl deleted file mode 100644 index b599179fe..000000000 --- a/charts/helm_lib/templates/_admission_client_ca.tpl +++ /dev/null @@ -1,21 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_admission_webhook_client_ca_certificate" (list . "namespace") }} */ -}} -{{- /* Renders configmap with admission webhook client CA certificate which uses to verify the AdmissionReview requests. */ -}} -{{- define "helm_lib_admission_webhook_client_ca_certificate" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -{{- /* Namespace where CA configmap will be created */ -}} - {{- $context := index . 0 }} - {{- $namespace := index . 1 }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: admission-client-ca.crt - namespace: {{ $namespace }} - annotations: - kubernetes.io/description: | - Contains a CA bundle that can be used to verify the admission webhook client. - {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} -data: - ca.crt: | - {{ $context.Values.global.internal.modules.admissionWebhookClientCA.cert | nindent 4 }} -{{- end }} diff --git a/charts/helm_lib/templates/_api_version_and_kind.tpl b/charts/helm_lib/templates/_api_version_and_kind.tpl deleted file mode 100644 index 4de8a8a36..000000000 --- a/charts/helm_lib/templates/_api_version_and_kind.tpl +++ /dev/null @@ -1,36 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_kind_exists" (list . "") }} */ -}} -{{- /* returns true if the specified resource kind (case-insensitive) is represented in the cluster */ -}} -{{- define "helm_lib_kind_exists" }} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $kind_name := index . 1 -}} {{- /* Kind name portion */ -}} - {{- if eq (len $context.Capabilities.APIVersions) 0 -}} - {{- fail "Helm reports no capabilities" -}} - {{- end -}} - {{ range $cap := $context.Capabilities.APIVersions }} - {{- if hasSuffix (lower (printf "/%s" $kind_name)) (lower $cap) }} - found - {{- break }} - {{- end }} - {{- end }} -{{- end -}} - -{{- /* Usage: {{ include "helm_lib_get_api_version_by_kind" (list . "") }} */ -}} -{{- /* returns current apiVersion string, based on available helm capabilities, for the provided kind (not all kinds are supported) */ -}} -{{- define "helm_lib_get_api_version_by_kind" }} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $kind_name := index . 1 -}} {{- /* Kind name portion */ -}} - {{- if eq (len $context.Capabilities.APIVersions) 0 -}} - {{- fail "Helm reports no capabilities" -}} - {{- end -}} - {{- if or (eq $kind_name "ValidatingAdmissionPolicy") (eq $kind_name "ValidatingAdmissionPolicyBinding") -}} - {{- if $context.Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" -}} -admissionregistration.k8s.io/v1 - {{- else if $context.Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1beta1/ValidatingAdmissionPolicy" -}} -admissionregistration.k8s.io/v1beta1 - {{- else -}} -admissionregistration.k8s.io/v1alpha1 - {{- end -}} - {{- else -}} - {{- fail (printf "Kind '%s' isn't supported by the 'helm_lib_get_api_version_by_kind' helper" $kind_name) -}} - {{- end -}} -{{- end -}} diff --git a/charts/helm_lib/templates/_application_image.tpl b/charts/helm_lib/templates/_application_image.tpl deleted file mode 100644 index c1aa9ef87..000000000 --- a/charts/helm_lib/templates/_application_image.tpl +++ /dev/null @@ -1,23 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_application_image" (list . "") }} */ -}} -{{- /* returns image name in format "registry/package@digest" */ -}} -{{- define "helm_lib_application_image" }} - {{- $context := index . 0 }} - - {{- $image := index . 1 | trimAll "\"" }} - {{- $imageDigest := index $context.Application.Package.Digests $image }} - {{- if not $imageDigest }} - {{- fail (printf "Image %s has no digest" $image) }} - {{- end }} - - {{- $registryBase := $context.Application.Package.Registry.repository }} - {{- if not $registryBase }} - {{- fail "Registry base is not set" }} - {{- end }} - - {{- $packageName := $context.Application.Package.Name }} - {{- if not $packageName }} - {{- fail "Package name is not set" }} - {{- end }} - - {{- printf "%s/%s@%s" $registryBase $packageName $imageDigest }} -{{- end }} diff --git a/charts/helm_lib/templates/_application_security_context.tpl b/charts/helm_lib/templates/_application_security_context.tpl deleted file mode 100644 index fd2108359..000000000 --- a/charts/helm_lib/templates/_application_security_context.tpl +++ /dev/null @@ -1,263 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_custom" (list . 1000 1000) }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with custom user and group */ -}} -{{- define "helm_lib_application_pod_security_context_run_as_user_custom" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -{{- /* User id */ -}} -{{- /* Group id */ -}} -securityContext: - runAsNonRoot: true - runAsUser: {{ index . 1 }} - runAsGroup: {{ index . 2 }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_nobody" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with user and group "nobody" */ -}} -{{- define "helm_lib_v_pod_security_context_run_as_user_nobody" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_nobody_with_writable_fs" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with user and group "nobody" with write access to mounted volumes */ -}} -{{- define "helm_lib_application_pod_security_context_run_as_user_nobody_with_writable_fs" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 - fsGroup: 65534 -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_deckhouse" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with user and group "deckhouse" */ -}} -{{- define "helm_lib_application_pod_security_context_run_as_user_deckhouse" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - runAsNonRoot: true - runAsUser: 64535 - runAsGroup: 64535 -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_deckhouse_with_writable_fs" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with user and group "deckhouse" with write access to mounted volumes */ -}} -{{- define "helm_lib_application_pod_security_context_run_as_user_deckhouse_with_writable_fs" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - runAsNonRoot: true - runAsUser: 64535 - runAsGroup: 64535 - fsGroup: 64535 -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_run_as_user_deckhouse_pss_restricted" . }} */ -}} -{{- /* returns SecurityContext parameters for Container with user and group "deckhouse" plus minimal required settings to comply with the Restricted mode of the Pod Security Standards */ -}} -{{- define "helm_lib_application_container_security_context_run_as_user_deckhouse_pss_restricted" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - all - runAsGroup: 64535 - runAsNonRoot: true - runAsUser: 64535 - seccompProfile: - type: RuntimeDefault -{{- end }} - - -{{- /* SecurityContext for Deckhouse UID/GID 64535 (or root), PSS Restricted */ -}} -{{- /* Optional keys: */ -}} -{{- /* .ro – bool, read-only root FS (default true) */ -}} -{{- /* .caps – []string, capabilities.add (default empty) */ -}} -{{- /* .uid – int, runAsUser/runAsGroup (default 64535) */ -}} -{{- /* .runAsNonRoot – bool, run as Deckhouse user when true, root when false (default true) */ -}} -{{- /* .seccompProfile – bool, disable seccompProfile when false (default true) */ -}} -{{- /* Usage: include "helm_lib_application_container_security_context_pss_restricted_flexible" (dict "ro" false "caps" (list "NET_ADMIN" "SYS_TIME") "uid" 1001 "seccompProfile" false "runAsNonRoot" true) */ -}} -{{- define "helm_lib_application_container_security_context_pss_restricted_flexible" -}} -{{- $ro := true -}} -{{- if hasKey . "ro" -}} - {{- $ro = .ro -}} -{{- end -}} -{{- $seccompProfile := true -}} -{{- if hasKey . "seccompProfile" -}} - {{- $seccompProfile = .seccompProfile -}} -{{- end -}} -{{- $caps := default (list) .caps -}} -{{- $uid := default 64535 .uid -}} -{{- $runAsNonRoot := true -}} -{{- if hasKey . "runAsNonRoot" -}} - {{- $runAsNonRoot = .runAsNonRoot -}} -{{- end -}} - -securityContext: - readOnlyRootFilesystem: {{ $ro }} - allowPrivilegeEscalation: {{ not $runAsNonRoot }} -{{- if $runAsNonRoot }} - privileged: false -{{- end }} - capabilities: - drop: - - ALL -{{- if $caps }} - add: {{ $caps | toJson }} -{{- end }} - runAsUser: {{ ternary $uid 0 $runAsNonRoot }} - runAsGroup: {{ ternary $uid 0 $runAsNonRoot }} - runAsNonRoot: {{ $runAsNonRoot }} -{{- if $seccompProfile }} - seccompProfile: - type: RuntimeDefault -{{- end }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_application_pod_security_context_run_as_user_root" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with user and group 0 */ -}} -{{- define "helm_lib_application_pod_security_context_run_as_user_root" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - runAsNonRoot: false - runAsUser: 0 - runAsGroup: 0 -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_pod_security_context_runtime_default" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with seccomp profile RuntimeDefault */ -}} -{{- define "helm_lib_application_pod_security_context_runtime_default" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - seccompProfile: - type: RuntimeDefault -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_not_allow_privilege_escalation" . }} */ -}} -{{- /* returns SecurityContext parameters for Container with allowPrivilegeEscalation false */ -}} -{{- define "helm_lib_application_container_security_context_not_allow_privilege_escalation" -}} -securityContext: - allowPrivilegeEscalation: false -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_read_only_root_filesystem_with_selinux" . }} */ -}} -{{- /* returns SecurityContext parameters for Container with read only root filesystem and options for SELinux compatibility*/ -}} -{{- define "helm_lib_application_container_security_context_read_only_root_filesystem_with_selinux" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - seLinuxOptions: - level: 's0' - type: 'spc_t' -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_read_only_root_filesystem" . }} */ -}} -{{- /* returns SecurityContext parameters for Container with read only root filesystem */ -}} -{{- define "helm_lib_application_container_security_context_read_only_root_filesystem" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_privileged" . }} */ -}} -{{- /* returns SecurityContext parameters for Container running privileged */ -}} -{{- define "helm_lib_application_container_security_context_privileged" -}} -securityContext: - privileged: true -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_escalated_sys_admin_privileged" . }} */ -}} -{{- /* returns SecurityContext parameters for Container running privileged with escalation and sys_admin */ -}} -{{- define "helm_lib_application_container_security_context_escalated_sys_admin_privileged" -}} -securityContext: - allowPrivilegeEscalation: true - readOnlyRootFilesystem: true - capabilities: - add: - - SYS_ADMIN - privileged: true -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_privileged_read_only_root_filesystem" . }} */ -}} -{{- /* returns SecurityContext parameters for Container running privileged with read only root filesystem */ -}} -{{- define "helm_lib_application_container_security_context_privileged_read_only_root_filesystem" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - privileged: true - readOnlyRootFilesystem: true -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all" . }} */ -}} -{{- /* returns SecurityContext for Container with read only root filesystem and all capabilities dropped */ -}} -{{- define "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} */ -}} -{{- /* returns SecurityContext parameters for Container with read only root filesystem, all dropped and some added capabilities */ -}} -{{- define "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -{{- /* List of capabilities */ -}} -securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - add: {{ index . 1 | toJson }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} */ -}} -{{- /* returns SecurityContext parameters for Container with all dropped and some added capabilities */ -}} -{{- define "helm_lib_application_container_security_context_capabilities_drop_all_and_add" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -{{- /* List of capabilities */ -}} -securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - add: {{ index . 1 | toJson }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_capabilities_drop_all_and_run_as_user_custom" (list . 1000 1000) }} */ -}} -{{- /* returns SecurityContext parameters for Container with read only root filesystem, all dropped, and custom user ID */ -}} -{{- define "helm_lib_application_container_security_context_capabilities_drop_all_and_run_as_user_custom" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -{{- /* User id */ -}} -{{- /* Group id */ -}} -securityContext: - runAsUser: {{ index . 1 }} - runAsGroup: {{ index . 2 }} - runAsNonRoot: true - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault -{{- end }} - -{{- /* Usage: {{ include "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . }} */ -}} -{{- /* returns SecurityContext parameters for Container with minimal required settings to comply with the Restricted mode of the Pod Security Standards */ -}} -{{- define "helm_lib_application_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault -{{- end }} diff --git a/charts/helm_lib/templates/_capi_controller_manager.tpl b/charts/helm_lib/templates/_capi_controller_manager.tpl deleted file mode 100644 index 97d993500..000000000 --- a/charts/helm_lib/templates/_capi_controller_manager.tpl +++ /dev/null @@ -1,249 +0,0 @@ -{{- define "capi_controller_manager_resources" -}} -cpu: 25m -memory: 50Mi -{{- end -}} - -{{- define "capi_controller_manager_max_allowed_resources" -}} -cpu: 50m -memory: 50Mi -{{- end -}} - -{{- define "capi_controller_manager_liveness_probe" -}} -httpGet: - path: /healthz - port: 8081 -initialDelaySeconds: 15 -periodSeconds: 20 -{{- end -}} - -{{- define "capi_controller_manager_readiness_probe" -}} -httpGet: - path: /readyz - port: 8081 -initialDelaySeconds: 5 -periodSeconds: 10 -{{- end -}} - -{{- /* Usage: {{ include "helm_lib_capi_controller_manager_manifests" (list . $config) }} */ -}} -{{- /* Renders common manifests for provider-specific CAPI Controller Managers. */ -}} -{{- /* Includes Deployment, VerticalPodAutoscaler (optional) and PodDisruptionBudget (optional). */ -}} -{{- /* Supported configuration parameters: */ -}} -{{- /* + fullname (required) — resource base name used for Deployment, PDB, VPA, and by default for the main container name. */ -}} -{{- /* + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. */ -}} -{{- /* + image (required) — image for the main container. */ -}} -{{- /* + capiProviderName (required) — value for the cluster.x-k8s.io/provider label in selectors and pod labels. */ -}} -{{- /* + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. */ -}} -{{- /* + priorityClassName (optional, default: `"system-cluster-critical"`) — Pod priority class name. */ -}} -{{- /* + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. */ -}} -{{- /* + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. */ -}} -{{- /* + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. */ -}} -{{- /* + terminationGracePeriodSeconds (optional, default: `10`) — Pod termination grace period. */ -}} -{{- /* + hostNetwork (optional, default: `false`) — enables host networking for the Pod. */ -}} -{{- /* + dnsPolicy (optional, default: `nil`) — Pod DNS policy; if not set, the field is omitted. */ -}} -{{- /* + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. */ -}} -{{- /* + tolerationsStrategies (optional, default: `["any-node", "uninitialized"]`) — arguments passed to helm_lib_tolerations. */ -}} -{{- /* + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 8081}, initialDelaySeconds: 15, periodSeconds: 20}`) — liveness probe configuration for the main container. */ -}} -{{- /* + readinessProbe (optional, default: `{httpGet: {path: /readyz, port: 8081}, initialDelaySeconds: 5, periodSeconds: 10}`) — readiness probe configuration for the main container. */ -}} -{{- /* + additionalArgs (optional, default: `[]`) — extra args for the main container. */ -}} -{{- /* + additionalEnv (optional, default: `[]`) — extra environment variables for the main container. */ -}} -{{- /* + additionalPorts (optional, default: `[]`) — extra container ports for the main container. */ -}} -{{- /* + additionalInitContainers (optional, default: `[]`) — extra initContainers for the Pod. */ -}} -{{- /* + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. */ -}} -{{- /* + additionalVolumes (optional, default: `[]`) — extra Pod volumes. */ -}} -{{- /* + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. */ -}} -{{- /* + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. */ -}} -{{- /* + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. */ -}} -{{- /* + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. */ -}} -{{- /* + vpaEnabled (optional, default: `false`) — enables VerticalPodAutoscaler rendering. */ -}} -{{- /* + vpaUpdateMode (optional, default: `"InPlaceOrRecreate"`) — VPA update mode. */ -}} -{{- /* + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. */ -}} -{{- /* + securityPolicyExceptionEnabled (optional, default: `false`) — enables SecurityPolicyException rendering and adds the related pod label. */ -}} -{{- define "helm_lib_capi_controller_manager_manifests" -}} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc. */ -}} - {{- $config := index . 1 -}} {{- /* Configuration dict for the CAPI Controller Manager. */ -}} - - {{- $fullname := required "helm_lib_capi_controller_manager_manifests: fullname is required" $config.fullname -}} - {{- $namespace := dig "namespace" (printf "d8-%s" $context.Chart.Name) $config -}} - {{- $image := required "helm_lib_capi_controller_manager_manifests: image is required" $config.image -}} - {{- $capiProviderName := required "helm_lib_capi_controller_manager_manifests: $capiProviderName is required" $config.capiProviderName -}} - {{- $resources := dig "resources" (include "capi_controller_manager_resources" $context | fromYaml) $config -}} - {{- $priorityClassName := dig "priorityClassName" "system-cluster-critical" $config -}} - {{- $serviceAccountName := dig "serviceAccountName" $fullname $config -}} - {{- $automountServiceAccountToken := dig "automountServiceAccountToken" true $config -}} - {{- $revisionHistoryLimit := dig "revisionHistoryLimit" 2 $config -}} - {{- $terminationGracePeriodSeconds := dig "terminationGracePeriodSeconds" 10 $config -}} - {{- $hostNetwork := dig "hostNetwork" false $config -}} - {{- $dnsPolicy := dig "dnsPolicy" nil $config -}} - {{- $nodeSelectorStrategy := dig "nodeSelectorStrategy" "master" $config -}} - {{- $tolerationsStrategies := dig "tolerationsStrategies" (list "any-node" "uninitialized") $config -}} - {{- $livenessProbe := dig "livenessProbe" (include "capi_controller_manager_liveness_probe" $context | fromYaml) $config }} - {{- $readinessProbe := dig "readinessProbe" (include "capi_controller_manager_readiness_probe" $context | fromYaml) $config }} - {{- $additionalArgs := dig "additionalArgs" (list) $config -}} - {{- $additionalEnv := dig "additionalEnv" (list) $config -}} - {{- $additionalPorts := dig "additionalPorts" (list) $config -}} - {{- $additionalInitContainers := dig "additionalInitContainers" (list) $config -}} - {{- $additionalVolumeMounts := dig "additionalVolumeMounts" (list) $config -}} - {{- $additionalVolumes := dig "additionalVolumes" (list) $config -}} - {{- $additionalPodLabels := dig "additionalPodLabels" (dict) $config -}} - {{- $additionalPodAnnotations := dig "additionalPodAnnotations" (dict) $config -}} - {{- $pdbEnabled := dig "pdbEnabled" true $config -}} - {{- $pdbMaxUnavailable := dig "pdbMaxUnavailable" 1 $config -}} - {{- $vpaEnabled := dig "vpaEnabled" false $config -}} - {{- $vpaUpdateMode := dig "vpaUpdateMode" "InPlaceOrRecreate" $config -}} - {{- $vpaMaxAllowed := dig "vpaMaxAllowed" (include "capi_controller_manager_max_allowed_resources" $context | fromYaml) $config -}} - {{- $securityPolicyExceptionEnabled := dig "securityPolicyExceptionEnabled" false $config }} - -{{- if and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} ---- -apiVersion: autoscaling.k8s.io/v1 -kind: VerticalPodAutoscaler -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -spec: - targetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ $fullname }} - updatePolicy: - updateMode: {{ $vpaUpdateMode | quote }} - resourcePolicy: - containerPolicies: - - containerName: {{ $fullname | quote }} - minAllowed: - {{- toYaml $resources | nindent 10 }} - maxAllowed: - {{- toYaml $vpaMaxAllowed | nindent 10 }} -{{- end }} - -{{- if $pdbEnabled }} ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -spec: - maxUnavailable: {{ $pdbMaxUnavailable }} - selector: - matchLabels: - app: {{ $fullname }} -{{- end }} - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -spec: - {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" $context | nindent 2 }} - revisionHistoryLimit: {{ $revisionHistoryLimit }} - selector: - matchLabels: - app: {{ $fullname }} - cluster.x-k8s.io/provider: {{ $capiProviderName }} - control-plane: controller-manager - template: - metadata: - labels: - app: {{ $fullname }} - cluster.x-k8s.io/provider: {{ $capiProviderName }} - control-plane: controller-manager - {{- with $additionalPodLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $additionalPodAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - imagePullSecrets: - - name: deckhouse-registry - {{- include "helm_lib_pod_anti_affinity_for_ha" (list $context (dict "app" $fullname)) | nindent 6 }} - {{- include "helm_lib_priority_class" (tuple $context $priorityClassName) | nindent 6 }} - {{- include "helm_lib_node_selector" (tuple $context $nodeSelectorStrategy) | nindent 6 }} - {{- include "helm_lib_tolerations" (concat (list $context) $tolerationsStrategies) | nindent 6 }} - {{- include "helm_lib_module_pod_security_context_run_as_user_deckhouse" $context | nindent 6 }} - automountServiceAccountToken: {{ $automountServiceAccountToken }} - serviceAccountName: {{ $serviceAccountName }} - terminationGracePeriodSeconds: {{ $terminationGracePeriodSeconds }} - hostNetwork: {{ $hostNetwork }} - {{- with $dnsPolicy }} - dnsPolicy: {{ . }} - {{- end }} - {{- with $additionalInitContainers }} - initContainers: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: {{ $fullname }} - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" dict | nindent 8 }} - image: {{ $image }} - imagePullPolicy: IfNotPresent - args: - - --leader-elect - {{- with $additionalArgs }} - {{- toYaml . | nindent 10 }} - {{- end }} - {{- with $additionalEnv }} - env: - {{- toYaml . | nindent 10 }} - {{- end }} - {{- with $additionalPorts }} - ports: - {{- toYaml . | nindent 10 }} - {{- end }} - livenessProbe: - {{- with $livenessProbe }} - {{- toYaml . | nindent 10 }} - {{- end }} - readinessProbe: - {{- with $readinessProbe }} - {{- toYaml . | nindent 10 }} - {{- end }} - {{- with $additionalVolumeMounts }} - volumeMounts: - {{- toYaml . | nindent 10 }} - {{- end }} - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- if not (and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd")) }} - {{- toYaml $resources | nindent 12 }} - {{- end }} - {{- with $additionalVolumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - -{{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} -{{- if $hostNetwork }} ---- -apiVersion: deckhouse.io/v1alpha1 -kind: SecurityPolicyException -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} -spec: - network: - {{- if $additionalPorts }} - hostPorts: - {{- range $additionalPorts }} - - port: {{ .containerPort }} - protocol: {{ .protocol | default "TCP" }} - {{- end }} - {{- end }} - hostNetwork: - allowedValue: true - metadata: - description: | - Allow host network access for CAPI infrastructure controller manager. - The CAPI infrastructure controller manager requires host network access to continue infrastructure reconciliation even if the CNI or pod network is unavailable. -{{- end }} -{{- end }} - -{{- end -}} \ No newline at end of file diff --git a/charts/helm_lib/templates/_cloud_controller_manager.tpl b/charts/helm_lib/templates/_cloud_controller_manager.tpl deleted file mode 100644 index e8820a778..000000000 --- a/charts/helm_lib/templates/_cloud_controller_manager.tpl +++ /dev/null @@ -1,280 +0,0 @@ -{{- define "cloud_controller_manager_resources" }} -cpu: 25m -memory: 50Mi -{{- end }} - -{{- define "cloud_controller_manager_max_allowed_resources" }} -cpu: 50m -memory: 50Mi -{{- end }} - -{{- define "cloud_controller_manager_liveness_probe" -}} -httpGet: - path: /healthz - port: 10471 - host: 127.0.0.1 - scheme: HTTPS -{{- end -}} - -{{- define "cloud_controller_manager_readiness_probe" -}} -httpGet: - path: /healthz - port: 10471 - host: 127.0.0.1 - scheme: HTTPS -{{- end -}} - -{{- /* Usage: {{ include "helm_lib_cloud_controller_manager_manifests" (list . $config) }} */ -}} -{{- /* Renders common manifests for provider-specific Cloud Controller Managers. */ -}} -{{- /* Includes Deployment, VerticalPodAutoscaler (optional), PodDisruptionBudget (optional), and SecurityPolicyException (optional). */ -}} -{{- /* Supported configuration parameters: */ -}} -{{- /* + fullname (optional, default: `"cloud-controller-manager"`) — resource base name used for Deployment, PDB, VPA, SecurityPolicyException, and the main container name by default. */ -}} -{{- /* + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. */ -}} -{{- /* + image (required) — image for the main container. */ -}} -{{- /* + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. */ -}} -{{- /* + priorityClassName (optional, default: `"system-cluster-critical"`) — Pod priority class name. */ -}} -{{- /* + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. */ -}} -{{- /* + tolerationsStrategies (optional, default: ["wildcard"]) — strategies passed to helm_lib_tolerations. */ -}} -{{- /* + hostNetwork (optional, default: `true`) — enables host networking for the Pod and SecurityPolicyException network rule generation. */ -}} -{{- /* + dnsPolicy (optional, default: `"Default"`) — Pod DNS policy. */ -}} -{{- /* + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. */ -}} -{{- /* + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. */ -}} -{{- /* + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. */ -}} -{{- /* + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 10471, host: 127.0.0.1, scheme: HTTPS}}`) — liveness probe configuration for the main container. */ -}} -{{- /* + readinessProbe (optional, default: `{httpGet: {path: /healthz, port: 10471, host: 127.0.0.1, scheme: HTTPS}}`) — readiness probe configuration for the main container. */ -}} -{{- /* + additionalEnvs (optional, default: `[]`) — extra environment variables for the main container. */ -}} -{{- /* + additionalArgs (optional, default: `nil`) — extra args for the main container. */ -}} -{{- /* + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. */ -}} -{{- /* + additionalVolumes (optional, default: `[]`) — extra Pod volumes; hostPath volumes are also used to build SecurityPolicyException rules when enabled. */ -}} -{{- /* + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. */ -}} -{{- /* + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. */ -}} -{{- /* + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. */ -}} -{{- /* + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. */ -}} -{{- /* + additionalPDBAnnotations (optional, default: `{}`) — extra annotations added to PodDisruptionBudget metadata. */ -}} -{{- /* + vpaEnabled (optional, default: `true`) — enables VerticalPodAutoscaler rendering. */ -}} -{{- /* + vpaUpdateMode (optional, default: `"InPlaceOrRecreate"`) — VPA update mode. */ -}} -{{- /* + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. */ -}} -{{- /* + securityPolicyExceptionEnabled (optional, default: `false`) — enables SecurityPolicyException rendering and adds the related pod label. */ -}} -{{- define "helm_lib_cloud_controller_manager_manifests" }} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc. */ -}} - {{- $config := index . 1 -}} {{- /* Configuration dict for the Cloud Controller Manager. */ -}} - - {{- $fullname := dig "fullname" "cloud-controller-manager" $config }} - {{- $namespace := dig "namespace" (printf "d8-%s" $context.Chart.Name) $config -}} - {{- $image := $config.image | required "image is required" }} - {{- $resources := dig "resources" (include "cloud_controller_manager_resources" $context | fromYaml) $config }} - {{- $priorityClassName := dig "priorityClassName" "system-cluster-critical" $config }} - {{- $nodeSelectorStrategy := dig "nodeSelectorStrategy" "master" $config -}} - {{- $tolerationsStrategies := dig "tolerationsStrategies" (list "wildcard") $config -}} - {{- $hostNetwork := dig "hostNetwork" true $config }} - {{- $dnsPolicy := dig "dnsPolicy" "Default" $config }} - {{- $automountServiceAccountToken := dig "automountServiceAccountToken" true $config }} - {{- $serviceAccountName := dig "serviceAccountName" $fullname $config }} - {{- $revisionHistoryLimit := dig "revisionHistoryLimit" 2 $config }} - {{- $livenessProbe := dig "livenessProbe" (include "cloud_controller_manager_liveness_probe" $context | fromYaml) $config }} - {{- $readinessProbe := dig "readinessProbe" (include "cloud_controller_manager_readiness_probe" $context | fromYaml) $config }} - {{- $additionalEnvs := dig "additionalEnvs" (list) $config }} - {{- $additionalArgs := dig "additionalArgs" nil $config }} - {{- $additionalVolumeMounts := dig "additionalVolumeMounts" (list) $config }} - {{- $additionalVolumes := dig "additionalVolumes" (list) $config }} - {{- $additionalPodLabels := dig "additionalPodLabels" (dict) $config }} - {{- $additionalPodAnnotations := dig "additionalPodAnnotations" (dict) $config }} - {{- $pdbEnabled := dig "pdbEnabled" true $config }} - {{- $pdbMaxUnavailable := dig "pdbMaxUnavailable" 1 $config }} - {{- $additionalPDBAnnotations := dig "additionalPDBAnnotations" (dict) $config }} - {{- $vpaEnabled := dig "vpaEnabled" true $config }} - {{- $vpaUpdateMode := dig "vpaUpdateMode" "InPlaceOrRecreate" $config }} - {{- $vpaMaxAllowed := dig "vpaMaxAllowed" (include "cloud_controller_manager_max_allowed_resources" $context | fromYaml) $config }} - {{- $securityPolicyExceptionEnabled := dig "securityPolicyExceptionEnabled" false $config }} - -{{- if and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} ---- -apiVersion: autoscaling.k8s.io/v1 -kind: VerticalPodAutoscaler -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -spec: - targetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ $fullname }} - updatePolicy: - updateMode: {{ $vpaUpdateMode }} - resourcePolicy: - containerPolicies: - - containerName: {{ $fullname | quote }} - minAllowed: - {{- toYaml $resources | nindent 10 }} - maxAllowed: - {{- toYaml $vpaMaxAllowed | nindent 10 }} -{{- end }} - -{{- if $pdbEnabled }} ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} - {{- with $additionalPDBAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - maxUnavailable: {{ $pdbMaxUnavailable }} - selector: - matchLabels: - app: {{ $fullname }} -{{- end }} - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -spec: - {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" $context | nindent 2 }} - revisionHistoryLimit: {{ $revisionHistoryLimit }} - selector: - matchLabels: - app: {{ $fullname }} - template: - metadata: - labels: - app: {{ $fullname }} - {{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} - security.deckhouse.io/security-policy-exception: {{ $fullname }} - {{- end }} - {{- with $additionalPodLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $additionalPodAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - imagePullSecrets: - - name: deckhouse-registry - {{- include "helm_lib_pod_anti_affinity_for_ha" (list $context (dict "app" $fullname)) | nindent 6 }} - {{- include "helm_lib_priority_class" (tuple $context $priorityClassName) | nindent 6 }} - {{- include "helm_lib_node_selector" (tuple $context $nodeSelectorStrategy) | nindent 6 }} - {{- include "helm_lib_tolerations" (concat (list $context) $tolerationsStrategies) | nindent 6 }} - {{- include "helm_lib_module_pod_security_context_run_as_user_deckhouse" $context | nindent 6 }} - automountServiceAccountToken: {{ $automountServiceAccountToken }} - hostNetwork: {{ $hostNetwork }} - dnsPolicy: {{ $dnsPolicy }} - serviceAccountName: {{ $serviceAccountName }} - containers: - - name: {{ $fullname }} - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" dict | nindent 10 }} - image: {{ $image }} - args: - - --leader-elect=true - - --bind-address=127.0.0.1 - - --secure-port=10471 - {{- with $additionalArgs }} - {{- toYaml . | nindent 12 }} - {{- end }} - env: - {{- if not $context.Values.global.clusterIsBootstrapped }} - - name: KUBERNETES_SERVICE_HOST - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - - name: KUBERNETES_SERVICE_PORT - value: "6443" - {{- end }} - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - {{- with $additionalEnvs }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- include "helm_lib_envs_for_proxy" $context | nindent 12 }} - livenessProbe: - {{- with $livenessProbe }} - {{- toYaml . | nindent 12 }} - {{- end }} - readinessProbe: - {{- with $readinessProbe }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with $additionalVolumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 14 }} - {{- if not (and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd")) }} - {{- toYaml $resources | nindent 14 }} - {{- end }} - {{- with $additionalVolumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - -{{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} ---- -apiVersion: deckhouse.io/v1alpha1 -kind: SecurityPolicyException -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} -spec: - {{- if $hostNetwork }} - network: - hostNetwork: - allowedValue: true - metadata: - description: | - Allow host network access for Cloud Controller Manager. - The Cloud Controller Manager requires host network access to communicate with the API for managing infrastructure resources, including load balancer configuration, node lifecycle management, and routing operations. - {{- end }} - - {{- $hasHostPathVolumes := false }} - {{- if $additionalVolumes }} - {{- range $volume := $additionalVolumes }} - {{- if $volume.hostPath }} - {{- $hasHostPathVolumes = true }} - {{- end }} - {{- end }} - {{- end }} - {{- if $hasHostPathVolumes }} - volumes: - types: - allowedValues: - - hostPath - metadata: - description: | - Allow hostPath volume type for Cloud Controller Manager. - The Cloud Controller Manager requires hostPath volumes for accessing host-level resources needed for cloud provider integration and infrastructure management operations. - hostPath: - allowedValues: - {{- range $volume := $additionalVolumes }} - {{- if $volume.hostPath }} - {{- $readOnly := false }} - {{- range $volumeMount := $additionalVolumeMounts }} - {{- if eq $volumeMount.name $volume.name }} - {{- $readOnly = (default false $volumeMount.readOnly) }} - {{- end }} - {{- end }} - - path: {{ $volume.hostPath.path }} - readOnly: {{ $readOnly }} - metadata: - description: | - Allow access to additional hostPath volume at {{ $volume.hostPath.path }}. - This additional hostPath volume is required by the Cloud Controller Manager for provider-specific infrastructure management operations. - {{- end }} - {{- end }} - {{- end }} -{{- end }} - -{{- end }} diff --git a/charts/helm_lib/templates/_cloud_data_discoverer.tpl b/charts/helm_lib/templates/_cloud_data_discoverer.tpl deleted file mode 100644 index c9c6eb50a..000000000 --- a/charts/helm_lib/templates/_cloud_data_discoverer.tpl +++ /dev/null @@ -1,311 +0,0 @@ -{{- define "cloud_data_discoverer_resources" -}} -cpu: 25m -memory: 50Mi -{{- end -}} - -{{- define "cloud_data_discoverer_max_allowed_resources" -}} -cpu: 50m -memory: 50Mi -{{- end -}} - -{{- define "cloud_data_discoverer_liveness_probe" -}} -httpGet: - path: /healthz - port: 8080 - scheme: HTTPS -{{- end -}} - -{{- define "cloud_data_discoverer_readiness_probe" -}} -httpGet: - path: /healthz - port: 8080 - scheme: HTTPS -{{- end -}} - -{{- /* Usage: {{ include "helm_lib_cloud_data_discoverer_manifests" (list . $config) }} */ -}} -{{- /* Renders common manifests for provider-specific Cloud Data Discoverers. */ -}} -{{- /* Includes Deployment, VerticalPodAutoscaler (optional) and PodDisruptionBudget (optional). */ -}} -{{- /* Supported configuration parameters: */ -}} -{{- /* + fullname (optional, default: `"cloud-data-discoverer"`) — resource base name used for Deployment, PDB, VPA, and the main container name by default. */ -}} -{{- /* + namespace (optional, default: `d8-{{ $context.Chart.Name }}`) — resource base namespace. */ -}} -{{- /* + image (required) — image for the main container. */ -}} -{{- /* + resources (optional, default: `{cpu: 25m, memory: 50Mi}`) — main container resource requests used when VPA is disabled. */ -}} -{{- /* + replicas (optional, default: `1`) — number of Deployment replicas. */ -}} -{{- /* + revisionHistoryLimit (optional, default: `2`) — number of old ReplicaSets retained by the Deployment. */ -}} -{{- /* + serviceAccountName (optional, default: `$config.fullname`) — ServiceAccount name used by the Pod. */ -}} -{{- /* + automountServiceAccountToken (optional, default: `true`) — controls whether the service account token is mounted into the Pod. */ -}} -{{- /* + priorityClassName (optional, default: `"cluster-low"`) — Pod priority class name. */ -}} -{{- /* + nodeSelectorStrategy (optional, default: `"master"`) — strategy passed to helm_lib_node_selector. */ -}} -{{- /* + tolerationsStrategies (optional, default: `["any-node", "with-uninitialized"]`) — strategies passed to helm_lib_tolerations. */ -}} -{{- /* + livenessProbe (optional, default: `{httpGet: {path: /healthz, port: 8080, scheme: HTTPS}}`) — liveness probe configuration for the main container. */ -}} -{{- /* + readinessProbe (optional, default: `{httpGet: {path: /healthz, port: 8080, scheme: HTTPS}}`) — readiness probe configuration for the main container. */ -}} -{{- /* + additionalArgs (optional, default: `[]`) — extra args for the main container. */ -}} -{{- /* + additionalEnv (optional, default: `[]`) — extra environment variables for the main container. */ -}} -{{- /* + additionalPodLabels (optional, default: `{}`) — extra labels added to the pod template metadata. */ -}} -{{- /* + additionalPodAnnotations (optional, default: `{}`) — extra annotations added to the pod template metadata. */ -}} -{{- /* + additionalInitContainers (optional, default: `[]`) — extra initContainers for the Pod. */ -}} -{{- /* + additionalVolumes (optional, default: `[]`) — extra Pod volumes. */ -}} -{{- /* + additionalVolumeMounts (optional, default: `[]`) — extra volumeMounts for the main container. */ -}} -{{- /* + pdbEnabled (optional, default: `true`) — enables PodDisruptionBudget rendering. */ -}} -{{- /* + pdbMaxUnavailable (optional, default: `1`) — maxUnavailable value for PodDisruptionBudget. */ -}} -{{- /* + vpaEnabled (optional, default: `true`) — enables VerticalPodAutoscaler rendering. */ -}} -{{- /* + vpaUpdateMode (optional, default: `"Initial"`) — VPA update mode. */ -}} -{{- /* + vpaMaxAllowed (optional, default: `{cpu: 50m, memory: 50Mi}`) — maximum resource values used in VPA policy. */ -}} -{{- define "helm_lib_cloud_data_discoverer_manifests" -}} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc. */ -}} - {{- $config := index . 1 -}} {{- /* Configuration dict for the Cloud Data Discoverer. */ -}} - - {{- $fullname := dig "fullname" "cloud-data-discoverer" $config -}} - {{- $namespace := dig "namespace" (printf "d8-%s" $context.Chart.Name) $config -}} - {{- $image := required "helm_lib_cloud_data_discoverer_manifests: image is required" $config.image -}} - {{- $resources := dig "resources" (include "cloud_data_discoverer_resources" $context | fromYaml) $config -}} - {{- $replicas := dig "replicas" 1 $config -}} - {{- $revisionHistoryLimit := dig "revisionHistoryLimit" 2 $config -}} - {{- $serviceAccountName := dig "serviceAccountName" $fullname $config -}} - {{- $automountServiceAccountToken := dig "automountServiceAccountToken" true $config -}} - {{- $priorityClassName := dig "priorityClassName" "cluster-low" $config -}} - {{- $nodeSelectorStrategy := dig "nodeSelectorStrategy" "master" $config -}} - {{- $tolerationsStrategies := dig "tolerationsStrategies" (list "any-node" "with-uninitialized") $config -}} - {{- $livenessProbe := dig "livenessProbe" (include "cloud_data_discoverer_liveness_probe" $context | fromYaml) $config }} - {{- $readinessProbe := dig "readinessProbe" (include "cloud_data_discoverer_readiness_probe" $context | fromYaml) $config }} - {{- $additionalArgs := dig "additionalArgs" (list) $config -}} - {{- $additionalEnv := dig "additionalEnv" (list) $config -}} - {{- $additionalPodLabels := dig "additionalPodLabels" (dict) $config }} - {{- $additionalPodAnnotations := dig "additionalPodAnnotations" (dict) $config }} - {{- $additionalInitContainers := dig "additionalInitContainers" (list) $config -}} - {{- $additionalVolumes := dig "additionalVolumes" (list) $config -}} - {{- $additionalVolumeMounts := dig "additionalVolumeMounts" (list) $config -}} - {{- $pdbEnabled := dig "pdbEnabled" true $config -}} - {{- $pdbMaxUnavailable := dig "pdbMaxUnavailable" 1 $config -}} - {{- $vpaEnabled := dig "vpaEnabled" true $config -}} - {{- $vpaUpdateMode := dig "vpaUpdateMode" "Initial" $config -}} - {{- $vpaMaxAllowed := dig "vpaMaxAllowed" (include "cloud_data_discoverer_max_allowed_resources" $context | fromYaml) $config -}} - -{{- if and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} ---- -apiVersion: autoscaling.k8s.io/v1 -kind: VerticalPodAutoscaler -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -spec: - targetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ $fullname }} - updatePolicy: - updateMode: {{ $vpaUpdateMode | quote }} - resourcePolicy: - containerPolicies: - - containerName: {{ $fullname | quote }} - minAllowed: - {{- toYaml $resources | nindent 8 }} - maxAllowed: - {{- toYaml $vpaMaxAllowed | nindent 8 }} - {{- include "helm_lib_vpa_kube_rbac_proxy_resources" $context | nindent 4 }} -{{- end }} - -{{- if $pdbEnabled }} ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -spec: - maxUnavailable: {{ $pdbMaxUnavailable }} - selector: - matchLabels: - app: {{ $fullname }} -{{- end }} - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $fullname }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -spec: - replicas: {{ $replicas }} - revisionHistoryLimit: {{ $revisionHistoryLimit }} - strategy: - type: Recreate - selector: - matchLabels: - app: {{ $fullname }} - template: - metadata: - labels: - app: {{ $fullname }} - {{- with $additionalPodLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - annotations: - kubectl.kubernetes.io/default-exec-container: {{ $fullname }} - kubectl.kubernetes.io/default-logs-container: {{ $fullname }} - {{- with $additionalPodAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - imagePullSecrets: - - name: deckhouse-registry - {{- include "helm_lib_priority_class" (tuple $context $priorityClassName) | nindent 6 }} - {{- include "helm_lib_node_selector" (tuple $context $nodeSelectorStrategy) | nindent 6 }} - {{- include "helm_lib_tolerations" (concat (list $context) $tolerationsStrategies) | nindent 6 }} - {{- include "helm_lib_module_pod_security_context_run_as_user_deckhouse" $context | nindent 6 }} - dnsPolicy: {{ include "helm_lib_dns_policy_bootstraping_state" (list $context "Default" "ClusterFirstWithHostNet") }} - automountServiceAccountToken: {{ $automountServiceAccountToken }} - serviceAccountName: {{ $serviceAccountName }} - {{- with $additionalInitContainers }} - initContainers: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: {{ $fullname }} - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" dict | nindent 8 }} - image: {{ $image }} - args: - - --discovery-period=1h - - --listen-address=127.0.0.1:8081 - {{- with $additionalArgs }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $additionalEnv }} - env: - {{- toYaml . | nindent 10 }} - {{- end }} - livenessProbe: - {{- with $livenessProbe }} - {{- toYaml . | nindent 10 }} - {{- end }} - readinessProbe: - {{- with $readinessProbe }} - {{- toYaml . | nindent 10 }} - {{- end }} - {{- with $additionalVolumeMounts }} - volumeMounts: - {{- toYaml . | nindent 10 }} - {{- end }} - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 12 }} - {{- if not (and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd")) }} - {{- toYaml $resources | nindent 12 }} - {{- end }} - - name: kube-rbac-proxy - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" dict | nindent 8 }} - image: {{ include "helm_lib_module_common_image" (list $context "kubeRbacProxy") }} - args: - - "--secure-listen-address=$(KUBE_RBAC_PROXY_LISTEN_ADDRESS):8080" - - "--v=2" - - "--logtostderr=true" - - "--stale-cache-interval=1h30m" - - "--livez-path=/livez" - ports: - - containerPort: 8080 - name: https-metrics - env: - - name: KUBE_RBAC_PROXY_LISTEN_ADDRESS - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBE_RBAC_PROXY_CONFIG - value: | - excludePaths: - - /healthz - upstreams: - - upstream: http://127.0.0.1:8081/ - path: / - authorization: - resourceAttributes: - namespace: {{ $namespace }} - apiGroup: apps - apiVersion: v1 - resource: deployments - subresource: prometheus-metrics - name: {{ $fullname }} - livenessProbe: - httpGet: - path: /livez - port: 8080 - scheme: HTTPS - readinessProbe: - httpGet: - path: /livez - port: 8080 - scheme: HTTPS - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 12 }} - {{- if not (and $vpaEnabled ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd")) }} - {{- include "helm_lib_container_kube_rbac_proxy_resources" $context | nindent 12 }} - {{- end }} - {{- with $additionalVolumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_cloud_data_discoverer_pod_monitor" (list . $config) }} */ -}} -{{- /* Renders PodMonitor manifest for provider-specific Cloud Data Discoverers. */ -}} -{{- /* Supported configuration parameters: */ -}} -{{- /* + fullname (optional, default: `"cloud-data-discoverer"`) — PodMonitor base name. */ -}} -{{- /* + targetNamespace (required) — target pod namespace for selector. */ -}} -{{- /* + additionalRelabelings (optional, default: `[]`) — additional rules for labels rewriting. */ -}} -{{- define "helm_lib_cloud_data_discoverer_pod_monitor" -}} - {{- $context := index . 0 -}} - {{- $config := index . 1 -}} - - {{- $fullname := dig "fullname" "cloud-data-discoverer" $config -}} - {{- $targetNamespace := required "helm_lib_cloud_data_discoverer_pod_monitor: targetNamespace is required" $config.targetNamespace -}} - {{- $additionalRelabelings := dig "additionalRelabelings" (list) $config -}} - -{{- if ($context.Values.global.enabledModules | has "operator-prometheus-crd") -}} ---- -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: {{ $fullname }}-metrics - namespace: d8-monitoring - {{- include "helm_lib_module_labels" (list $context (dict "prometheus" "main" "app" $fullname)) | nindent 2 }} -spec: - jobLabel: app - podMetricsEndpoints: - - port: https-metrics - path: /metrics - scheme: https - bearerTokenSecret: - name: prometheus-token - key: token - tlsConfig: - insecureSkipVerify: true - honorLabels: true - scrapeTimeout: {{ include "helm_lib_prometheus_target_scrape_timeout_seconds" (list $context 25) }} - relabelings: - - regex: "endpoint|pod|container" - action: labeldrop - - targetLabel: job - replacement: {{ $fullname }} - - sourceLabels: [__meta_kubernetes_pod_node_name] - targetLabel: node - - targetLabel: tier - replacement: cluster - - sourceLabels: [__meta_kubernetes_pod_ready] - regex: "true" - action: keep - {{- with $additionalRelabelings }} - {{- toYaml . | nindent 6 }} - {{- end }} - selector: - matchLabels: - app: {{ $fullname }} - namespaceSelector: - matchNames: - - {{ $targetNamespace }} - - {{- end -}} -{{- end -}} \ No newline at end of file diff --git a/charts/helm_lib/templates/_cloud_provider_user_authz_roles.tpl b/charts/helm_lib/templates/_cloud_provider_user_authz_roles.tpl deleted file mode 100644 index 2de128a71..000000000 --- a/charts/helm_lib/templates/_cloud_provider_user_authz_roles.tpl +++ /dev/null @@ -1,83 +0,0 @@ -{{- /* Usage: {{- include "helm_lib_cloud_provider_user_authz_cluster_roles" (list . $config) }} */ -}} -{{- /* Renders user-authz ClusterRoles for provider-specific cloud resources. */ -}} -{{- /* Includes User and ClusterAdmin ClusterRoles. */ -}} -{{- /* Supported configuration parameters: */ -}} -{{- /* + providerName (required) — provider name segment used in ClusterRole names. */ -}} -{{- /* + instanceClassResource (required) — Deckhouse instance class resource name granted by the rules. */ -}} -{{- /* + capiResources (optional, default: `[]`) — CAPI infrastructure resource names granted by the rules. */ -}} -{{- /* + additionalUserRules (optional, default: `[]`) — extra rules appended to the User ClusterRole. */ -}} -{{- /* + additionalClusterAdminRules (optional, default: `[]`) — extra rules appended to the ClusterAdmin ClusterRole. */ -}} -{{- define "helm_lib_cloud_provider_user_authz_cluster_roles" -}} - {{- $context := index . 0 -}} - {{- $config := index . 1 -}} - {{- $providerName := required "helm_lib_cloud_provider_user_authz_cluster_roles: providerName is required" (get $config "providerName") -}} - {{- $instanceClassResource := required "helm_lib_cloud_provider_user_authz_cluster_roles: instanceClassResource is required" (get $config "instanceClassResource") -}} - {{- $capiResources := dig "capiResources" (list) $config -}} - {{- $additionalUserRules := dig "additionalUserRules" (list) $config -}} - {{- $additionalClusterAdminRules := dig "additionalClusterAdminRules" (list) $config -}} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - user-authz.deckhouse.io/access-level: User - name: d8:user-authz:{{ $providerName }}:user - {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} -rules: -- apiGroups: - - deckhouse.io - resources: - - {{ $instanceClassResource }} - verbs: - - get - - list - - watch -{{- if $capiResources }} -- apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - {{- range $capiResources }} - - {{ . }} - {{- end }} - verbs: - - get - - list - - watch -{{- end }} -{{- with $additionalUserRules }} -{{ toYaml . }} -{{- end }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - user-authz.deckhouse.io/access-level: ClusterAdmin - name: d8:user-authz:{{ $providerName }}:cluster-admin - {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} -rules: -- apiGroups: - - deckhouse.io - resources: - - {{ $instanceClassResource }} - verbs: - - create - - delete - - deletecollection - - patch - - update -{{- if $capiResources }} -- apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - {{- range $capiResources }} - - {{ . }} - {{- end }} - verbs: - - patch - - update -{{- end }} -{{- with $additionalClusterAdminRules }} -{{ toYaml . }} -{{- end }} -{{- end -}} diff --git a/charts/helm_lib/templates/_csi_controller.tpl b/charts/helm_lib/templates/_csi_controller.tpl deleted file mode 100644 index 8d717d959..000000000 --- a/charts/helm_lib/templates/_csi_controller.tpl +++ /dev/null @@ -1,986 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_csi_image_with_common_fallback" (list . "" "") }} */ -}} -{{- /* returns image name from storage foundation module if enabled, otherwise from common module */ -}} -{{- define "helm_lib_csi_image_with_common_fallback" }} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $rawContainerName := index . 1 | trimAll "\"" }} {{- /* Container raw name */ -}} - {{- $kubernetesSemVer := index . 2 }} {{- /* Kubernetes semantic version */ -}} - {{- $imageDigest := "" }} - {{- $registryBase := $context.Values.global.modulesImages.registry.base }} - {{- /* Try to get from storage foundation module if enabled */}} - {{- if $context.Values.global.enabledModules | has "storage-foundation" }} - {{- $registryBase = join "/" (list $context.Values.storageFoundation.registry.base "modules" "storage-foundation" ) }} - {{- $storageFoundationDigests := index $context.Values.global.modulesImages.digests "storageFoundation" | default dict }} - {{- $currentMinor := int $kubernetesSemVer.Minor }} - {{- $kubernetesMajor := int $kubernetesSemVer.Major }} - {{- /* Iterate from currentMinor down to 0: use offset from 0 to currentMinor, then calculate minorVersion = currentMinor - offset */}} - {{- range $offset := until (int (add $currentMinor 1)) }} - {{- if not $imageDigest }} - {{- $minorVersion := int (sub $currentMinor $offset) }} - {{- $containerName := join "" (list $rawContainerName "ForK8SGE" $kubernetesMajor $minorVersion) }} - {{- $digest := index $storageFoundationDigests $containerName | default "" }} - {{- if $digest }} - {{- $imageDigest = $digest }} - {{- end }} - {{- end }} - {{- end }} - {{- /* Fallback to base container name if no versioned image found (when minor reached 0) */}} - {{- if not $imageDigest }} - {{- $imageDigest = index $storageFoundationDigests $rawContainerName | default "" }} - {{- end }} - {{- /* Fallback to common module if storage foundation module is not enabled */}} - {{- else }} - {{- $containerName := join "" (list $rawContainerName $kubernetesSemVer.Major $kubernetesSemVer.Minor) }} - {{- $imageDigest = index $context.Values.global.modulesImages.digests "common" $containerName | default "" }} - {{- end }} - {{- if $imageDigest }} - {{- printf "%s@%s" $registryBase $imageDigest }} - {{- end }} -{{- end }} - - -{{- define "attacher_resources" }} -cpu: 10m -memory: 25Mi -{{- end }} - -{{- define "provisioner_resources" }} -cpu: 10m -memory: 25Mi -{{- end }} - -{{- define "resizer_resources" }} -cpu: 10m -memory: 25Mi -{{- end }} - -{{- define "syncer_resources" }} -cpu: 10m -memory: 25Mi -{{- end }} - -{{- define "snapshotter_resources" }} -cpu: 10m -memory: 25Mi -{{- end }} - -{{- define "livenessprobe_resources" }} -cpu: 10m -memory: 25Mi -{{- end }} - -{{- define "controller_resources" }} -cpu: 10m -memory: 50Mi -{{- end }} - -{{- /* Usage: {{ include "helm_lib_csi_controller_manifests" (list . $config) }} */ -}} -{{- define "helm_lib_csi_controller_manifests" }} - {{- $context := index . 0 }} - - {{- $config := index . 1 }} - {{- $fullname := $config.fullname | default "csi-controller" }} - {{- $snapshotterEnabled := dig "snapshotterEnabled" true $config }} - {{- $snapshotterSnapshotNamePrefix := dig "snapshotterSnapshotNamePrefix" false $config }} - {{- $resizerEnabled := dig "resizerEnabled" true $config }} - {{- $syncerEnabled := dig "syncerEnabled" false $config }} - {{- $topologyEnabled := dig "topologyEnabled" true $config }} - {{- $capacityEnabled := dig "capacityEnabled" true $config }} - {{- $runAsRootUser := dig "runAsRootUser" false $config }} - {{- $extraCreateMetadataEnabled := dig "extraCreateMetadataEnabled" false $config }} - {{- $controllerImage := $config.controllerImage | required "$config.controllerImage is required" }} - {{- $provisionerTimeout := $config.provisionerTimeout | default "600s" }} - {{- $attacherTimeout := $config.attacherTimeout | default "600s" }} - {{- $resizerTimeout := $config.resizerTimeout | default "600s" }} - {{- $snapshotterTimeout := $config.snapshotterTimeout | default "600s" }} - {{- $provisionerWorkers := $config.provisionerWorkers | default "10" }} - {{- $volumeNamePrefix := $config.volumeNamePrefix }} - {{- $volumeNameUUIDLength := $config.volumeNameUUIDLength }} - {{- $attacherWorkers := $config.attacherWorkers | default "10" }} - {{- $resizerWorkers := $config.resizerWorkers | default "10" }} - {{- $snapshotterWorkers := $config.snapshotterWorkers | default "10" }} - {{- $csiControllerHaMode := $config.csiControllerHaMode | default false }} - {{- $additionalCsiControllerPodAnnotations := $config.additionalCsiControllerPodAnnotations | default false }} - {{- $additionalControllerEnvs := $config.additionalControllerEnvs }} - {{- $additionalSyncerEnvs := $config.additionalSyncerEnvs }} - {{- $additionalControllerArgs := $config.additionalControllerArgs }} - {{- $additionalControllerVolumes := $config.additionalControllerVolumes }} - {{- $additionalControllerVolumeMounts := $config.additionalControllerVolumeMounts }} - {{- $additionalControllerVPA := $config.additionalControllerVPA }} - {{- $additionalControllerPorts := $config.additionalControllerPorts }} - {{- $additionalContainers := $config.additionalContainers }} - {{- $csiControllerHostNetwork := $config.csiControllerHostNetwork | default "true" }} - {{- $csiControllerHostPID := $config.csiControllerHostPID | default "false" }} - {{- $livenessProbePort := $config.livenessProbePort | default 9808 }} - {{- $livenessProbeTimeoutSeconds := $config.livenessProbeTimeoutSeconds | default 5 }} - {{- $livenessProbeInitialDelaySeconds := $config.livenessProbeInitialDelaySeconds | default 5 }} - {{- $initContainers := $config.initContainers }} - {{- $customNodeSelector := $config.customNodeSelector }} - {{- $additionalPullSecrets := $config.additionalPullSecrets }} - {{- $forceCsiControllerPrivilegedContainer := $config.forceCsiControllerPrivilegedContainer | default false }} - {{- $dnsPolicy := $config.dnsPolicy | default "ClusterFirstWithHostNet" }} - {{- $securityPolicyExceptionEnabled := $config.securityPolicyExceptionEnabled | default false }} - - {{- $kubernetesSemVer := semver $context.Values.global.discovery.kubernetesVersion }} - - {{- $provisionerImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiExternalProvisioner" $kubernetesSemVer) }} - - {{- $attacherImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiExternalAttacher" $kubernetesSemVer) }} - - {{- $resizerImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiExternalResizer" $kubernetesSemVer) }} - - {{- $syncerImageName := join "" (list "csiVsphereSyncer" $kubernetesSemVer.Major $kubernetesSemVer.Minor) }} - {{- $syncerImage := include "helm_lib_module_common_image_no_fail" (list $context $syncerImageName) }} - - {{- $snapshotterImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiExternalSnapshotter" $kubernetesSemVer) }} - - {{- $livenessprobeImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiLivenessprobe" $kubernetesSemVer) }} - - {{- if $provisionerImage }} - {{- if ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} ---- -apiVersion: autoscaling.k8s.io/v1 -kind: VerticalPodAutoscaler -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" "csi-controller")) | nindent 2 }} -spec: - targetRef: - apiVersion: "apps/v1" - kind: Deployment - name: {{ $fullname }} - updatePolicy: - updateMode: "InPlaceOrRecreate" - resourcePolicy: - containerPolicies: - - containerName: "provisioner" - minAllowed: - {{- include "provisioner_resources" $context | nindent 8 }} - maxAllowed: - cpu: 20m - memory: 50Mi - - containerName: "attacher" - minAllowed: - {{- include "attacher_resources" $context | nindent 8 }} - maxAllowed: - cpu: 20m - memory: 50Mi - {{- if $resizerEnabled }} - - containerName: "resizer" - minAllowed: - {{- include "resizer_resources" $context | nindent 8 }} - maxAllowed: - cpu: 20m - memory: 50Mi - {{- end }} - {{- if $syncerEnabled }} - - containerName: "syncer" - minAllowed: - {{- include "syncer_resources" $context | nindent 8 }} - maxAllowed: - cpu: 20m - memory: 50Mi - {{- end }} - {{- if $snapshotterEnabled }} - - containerName: "snapshotter" - minAllowed: - {{- include "snapshotter_resources" $context | nindent 8 }} - maxAllowed: - cpu: 20m - memory: 50Mi - {{- end }} - - containerName: "livenessprobe" - minAllowed: - {{- include "livenessprobe_resources" $context | nindent 8 }} - maxAllowed: - cpu: 20m - memory: 50Mi - - containerName: "controller" - minAllowed: - {{- include "controller_resources" $context | nindent 8 }} - maxAllowed: - cpu: 20m - memory: 100Mi - {{- if $additionalControllerVPA }} - {{- $additionalControllerVPA | toYaml | nindent 4 }} - {{- end }} - {{- end }} ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" "csi-controller")) | nindent 2 }} -spec: - maxUnavailable: 1 - selector: - matchLabels: - app: {{ $fullname }} ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" "csi-controller")) | nindent 2 }} - -spec: - {{- if $csiControllerHaMode }} - {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" $context | nindent 2 }} - {{- else }} - replicas: 1 - strategy: - type: Recreate - {{- end }} - revisionHistoryLimit: 2 - selector: - matchLabels: - app: {{ $fullname }} - template: - metadata: - labels: - app: {{ $fullname }} - {{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} - security.deckhouse.io/security-policy-exception: {{ $fullname }} - {{- end }} - {{- if or (hasPrefix "cloud-provider-" $context.Chart.Name) ($additionalCsiControllerPodAnnotations) }} - annotations: - {{- if hasPrefix "cloud-provider-" $context.Chart.Name }} - cloud-config-checksum: {{ include (print $context.Template.BasePath "/cloud-controller-manager/secret.yaml") $context | sha256sum }} - {{- end }} - {{- if $additionalCsiControllerPodAnnotations }} - {{- $additionalCsiControllerPodAnnotations | toYaml | nindent 8 }} - {{- end }} - {{- end }} - spec: - {{- if $csiControllerHaMode }} - {{- include "helm_lib_pod_anti_affinity_for_ha" (list $context (dict "app" $fullname)) | nindent 6 }} - {{- end }} - hostNetwork: {{ $csiControllerHostNetwork }} - hostPID: {{ $csiControllerHostPID }} - {{- if eq $csiControllerHostNetwork "true" }} - dnsPolicy: {{ $dnsPolicy | quote }} - {{- end }} - imagePullSecrets: - - name: deckhouse-registry - {{- if $additionalPullSecrets }} - {{- $additionalPullSecrets | toYaml | nindent 6 }} - {{- end }} - {{- include "helm_lib_priority_class" (tuple $context "system-cluster-critical") | nindent 6 }} - {{- if $customNodeSelector }} - nodeSelector: - {{- $customNodeSelector | toYaml | nindent 8 }} - {{- else }} - {{- include "helm_lib_node_selector" (tuple $context "master") | nindent 6 }} - {{- end }} - {{- include "helm_lib_tolerations" (tuple $context "any-node" "with-uninitialized") | nindent 6 }} - {{- if $runAsRootUser }} - {{- include "helm_lib_module_pod_security_context_run_as_user_root" . | nindent 6 }} - {{- else }} - {{- include "helm_lib_module_pod_security_context_run_as_user_deckhouse" . | nindent 6 }} - {{- end }} - serviceAccountName: csi - automountServiceAccountToken: true - containers: - - name: provisioner - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} - image: {{ $provisionerImage | quote }} - args: - - "--timeout={{ $provisionerTimeout }}" - - "--v=5" - - "--csi-address=$(ADDRESS)" - {{- if $volumeNamePrefix }} - - "--volume-name-prefix={{ $volumeNamePrefix }}" - {{- end }} - {{- if $volumeNameUUIDLength }} - - "--volume-name-uuid-length={{ $volumeNameUUIDLength }}" - {{- end }} - {{- if $topologyEnabled }} - - "--feature-gates=Topology=true" - - "--strict-topology" - {{- else }} - - "--feature-gates=Topology=false" - {{- end }} - - "--default-fstype=ext4" - - "--leader-election=true" - - "--leader-election-namespace=$(NAMESPACE)" - - "--leader-election-lease-duration=30s" - - "--leader-election-renew-deadline=20s" - - "--leader-election-retry-period=5s" - {{- if $capacityEnabled }} - - "--enable-capacity" - - "--capacity-ownerref-level=2" - {{- end }} - {{- if $extraCreateMetadataEnabled }} - - "--extra-create-metadata=true" - {{- end }} - - "--worker-threads={{ $provisionerWorkers }}" - env: - - name: ADDRESS - value: /csi/csi.sock - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - volumeMounts: - - name: socket-dir - mountPath: /csi - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "provisioner_resources" $context | nindent 12 }} - {{- end }} - - name: attacher - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} - image: {{ $attacherImage | quote }} - args: - - "--timeout={{ $attacherTimeout }}" - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--leader-election=true" - - "--leader-election-namespace=$(NAMESPACE)" - - "--leader-election-lease-duration=30s" - - "--leader-election-renew-deadline=20s" - - "--leader-election-retry-period=5s" - - "--worker-threads={{ $attacherWorkers }}" - env: - - name: ADDRESS - value: /csi/csi.sock - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - volumeMounts: - - name: socket-dir - mountPath: /csi - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "attacher_resources" $context | nindent 12 }} - {{- end }} - {{- if $resizerEnabled }} - - name: resizer - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} - image: {{ $resizerImage | quote }} - args: - - "--timeout={{ $resizerTimeout }}" - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--leader-election=true" - - "--leader-election-namespace=$(NAMESPACE)" - - "--leader-election-lease-duration=30s" - - "--leader-election-renew-deadline=20s" - - "--leader-election-retry-period=5s" - - "--workers={{ $resizerWorkers }}" - env: - - name: ADDRESS - value: /csi/csi.sock - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - volumeMounts: - - name: socket-dir - mountPath: /csi - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "resizer_resources" $context | nindent 12 }} - {{- end }} - {{- end }} - {{- if $syncerEnabled }} - - name: syncer - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} - image: {{ $syncerImage | quote }} - args: - - "--leader-election" - - "--leader-election-lease-duration=30s" - - "--leader-election-renew-deadline=20s" - - "--leader-election-retry-period=10s" - {{- if $additionalControllerArgs }} - {{- $additionalControllerArgs | toYaml | nindent 8 }} - {{- end }} - {{- if $additionalSyncerEnvs }} - env: - {{- $additionalSyncerEnvs | toYaml | nindent 8 }} - {{- end }} - {{- if $additionalControllerVolumeMounts }} - volumeMounts: - {{- $additionalControllerVolumeMounts | toYaml | nindent 8 }} - {{- end }} - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "syncer_resources" $context | nindent 12 }} - {{- end }} - {{- end }} - {{- if $snapshotterEnabled }} - - name: snapshotter - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} - image: {{ $snapshotterImage | quote }} - args: - - "--timeout={{ $snapshotterTimeout }}" - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--leader-election=true" - - "--leader-election-namespace=$(NAMESPACE)" - - "--leader-election-lease-duration=30s" - - "--leader-election-renew-deadline=20s" - - "--leader-election-retry-period=5s" - - "--worker-threads={{ $snapshotterWorkers }}" - {{- if $snapshotterSnapshotNamePrefix }} - - "--snapshot-name-prefix={{ $snapshotterSnapshotNamePrefix }}" - {{- end }} - env: - - name: ADDRESS - value: /csi/csi.sock - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - volumeMounts: - - name: socket-dir - mountPath: /csi - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "snapshotter_resources" $context | nindent 12 }} - {{- end }} - {{- end }} - - name: livenessprobe - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} - image: {{ $livenessprobeImage | quote }} - args: - - "--csi-address=$(ADDRESS)" - - "--probe-timeout={{ $livenessProbeTimeoutSeconds }}s" - {{- if eq $csiControllerHostNetwork "true" }} - - "--http-endpoint=$(HOST_IP):{{ $livenessProbePort }}" - {{- else }} - - "--http-endpoint=$(POD_IP):{{ $livenessProbePort }}" - {{- end }} - env: - - name: ADDRESS - value: /csi/csi.sock - {{- if eq $csiControllerHostNetwork "true" }} - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - {{- else }} - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - {{- end }} - volumeMounts: - - name: socket-dir - mountPath: /csi - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "livenessprobe_resources" $context | nindent 12 }} - {{- end }} - - name: controller -{{- if $forceCsiControllerPrivilegedContainer }} - {{- include "helm_lib_module_container_security_context_escalated_sys_admin_privileged" . | nindent 8 }} -{{- else }} - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 8 }} -{{- end }} - image: {{ $controllerImage | quote }} - args: - {{- if $additionalControllerArgs }} - {{- $additionalControllerArgs | toYaml | nindent 8 }} - {{- end }} - {{- if $additionalControllerEnvs }} - env: - {{- $additionalControllerEnvs | toYaml | nindent 8 }} - {{- end }} - livenessProbe: - httpGet: - path: /healthz - port: {{ $livenessProbePort }} - initialDelaySeconds: {{ $livenessProbeInitialDelaySeconds }} - timeoutSeconds: {{ $livenessProbeTimeoutSeconds }} - - {{- if $additionalControllerPorts }} - ports: - {{- $additionalControllerPorts | toYaml | nindent 8 }} - {{- end }} - volumeMounts: - - name: socket-dir - mountPath: /csi - {{- /* For an unknown reason vSphere csi-controller won't start without `/tmp` directory */ -}} - {{- if eq $context.Chart.Name "cloud-provider-vsphere" }} - - name: tmp - mountPath: /tmp - {{- end }} - {{- if $additionalControllerVolumeMounts }} - {{- $additionalControllerVolumeMounts | toYaml | nindent 8 }} - {{- end }} - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "controller_resources" $context | nindent 12 }} - {{- end }} - {{- if $additionalContainers }} - {{- $additionalContainers | toYaml | nindent 6 }} - {{- end }} - - {{- if $initContainers }} - initContainers: - {{- range $initContainer := $initContainers }} - - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- $initContainer | toYaml | nindent 8 }} - {{- end }} - {{- end }} - - volumes: - - name: socket-dir - emptyDir: {} - {{- /* For an unknown reason vSphere csi-controller won't start without `/tmp` directory */ -}} - {{- if eq $context.Chart.Name "cloud-provider-vsphere" }} - - name: tmp - emptyDir: {} - {{- end }} - - {{- if $additionalControllerVolumes }} - {{- $additionalControllerVolumes | toYaml | nindent 6 }} - {{- end }} - -{{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} ---- -apiVersion: deckhouse.io/v1alpha1 -kind: SecurityPolicyException -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} -spec: -{{- if or $forceCsiControllerPrivilegedContainer $runAsRootUser (eq $csiControllerHostNetwork "true") (ne $csiControllerHostPID "false") }} - {{- if or $forceCsiControllerPrivilegedContainer $runAsRootUser }} - securityContext: - {{- if $forceCsiControllerPrivilegedContainer }} - privileged: - allowedValue: true - metadata: - description: | - Allow privileged mode for CSI Controller. - The CSI Controller requires privileged access to perform storage management operations that need direct access to host resources, including device management and volume lifecycle control. - capabilities: - allowedValues: - add: - - SYS_ADMIN - metadata: - description: | - Allow SYS_ADMIN capability for CSI Controller. - The CSI Controller requires SYS_ADMIN capability to perform privileged storage management operations such as volume provisioning, snapshotting, and direct interaction with the storage backend. - {{- end }} - {{- if $runAsRootUser }} - runAsUser: - allowedValues: - - 0 - metadata: - description: | - Allow running as root user (UID 0) for CSI Controller. - The CSI Controller requires root privileges to interact with storage backends, manage volume lifecycle operations, and access cloud provider APIs. - runAsNonRoot: - allowedValue: false - metadata: - description: | - Allow running as root for CSI Controller. - The CSI Controller requires root access for storage management operations that need elevated privileges to interact with the underlying infrastructure. - {{- end }} - {{- end }} - {{- if or (eq $csiControllerHostNetwork "true") (ne $csiControllerHostPID "false") }} - network: - {{- if eq $csiControllerHostNetwork "true" }} - hostNetwork: - allowedValue: true - metadata: - description: | - Allow host network access for CSI Controller. - The CSI Controller requires host network access to communicate with cloud provider API endpoints for volume provisioning, attachment, snapshot, and lifecycle management operations. - {{- end }} - {{- if ne $csiControllerHostPID "false" }} - hostPID: - allowedValue: true - metadata: - description: | - Allow host PID namespace access for CSI Controller. - The CSI Controller requires host PID namespace access to interact with host processes for storage operations and coordinate with system-level services. - {{- end }} - {{- end }} - {{- $hasHostPathVolumes := false }} - {{- if $additionalControllerVolumes }} - {{- range $vol := $additionalControllerVolumes }} - {{- if $vol.hostPath }} - {{- $hasHostPathVolumes = true }} - {{- end }} - {{- end }} - {{- end }} - {{- if $hasHostPathVolumes }} - volumes: - types: - allowedValues: - - hostPath - metadata: - description: | - Allow hostPath volume type for CSI Controller. - The CSI Controller requires hostPath volumes for accessing host-level resources needed for storage management operations specific to the cloud provider implementation. - hostPath: - allowedValues: - {{- range $volume := $additionalControllerVolumes }} - {{- if $volume.hostPath }} - {{- $readOnly := false }} - {{- range $volumeMount := $additionalControllerVolumeMounts }} - {{- if eq $volumeMount.name $volume.name }} - {{- $readOnly = (default false $volumeMount.readOnly) }} - {{- end }} - {{- end }} - - path: {{ $volume.hostPath.path }} - readOnly: {{ $readOnly }} - metadata: - description: | - Allow access to additional hostPath volume at {{ $volume.hostPath.path }}. - This hostPath volume is required by the CSI Controller for storage management operations specific to the cloud provider implementation. - {{- end }} - {{- end }} - {{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_csi_controller_rbac" . }} */ -}} -{{- define "helm_lib_csi_controller_rbac" }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi - namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -automountServiceAccountToken: false - -# =========== -# provisioner -# =========== -# Source https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: d8:{{ .Chart.Name }}:csi:controller:external-provisioner - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -rules: -- apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] -- apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] -- apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] -- apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] -# Access to volumeattachments is only needed when the CSI driver -# has the PUBLISH_UNPUBLISH_VOLUME controller capability. -# In that case, external-provisioner will watch volumeattachments -# to determine when it is safe to delete a volume. -- apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: d8:{{ .Chart.Name }}:csi:controller:external-provisioner - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -subjects: -- kind: ServiceAccount - name: csi - namespace: d8-{{ .Chart.Name }} -roleRef: - kind: ClusterRole - name: d8:{{ .Chart.Name }}:csi:controller:external-provisioner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi:controller:external-provisioner - namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -rules: -# Only one of the following rules for endpoints or leases is required based on -# what is set for `--leader-election-type`. Endpoints are deprecated in favor of Leases. -- apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] -# Permissions for CSIStorageCapacity are only needed enabling the publishing -# of storage capacity information. -- apiGroups: ["storage.k8s.io"] - resources: ["csistoragecapacities"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -# The GET permissions below are needed for walking up the ownership chain -# for CSIStorageCapacity. They are sufficient for deployment via -# StatefulSet (only needs to get Pod) and Deployment (needs to get -# Pod and then ReplicaSet to find the Deployment). -- apiGroups: [""] - resources: ["pods"] - verbs: ["get"] -- apiGroups: ["apps"] - resources: ["replicasets"] - verbs: ["get"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi:controller:external-provisioner - namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -subjects: -- kind: ServiceAccount - name: csi - namespace: d8-{{ .Chart.Name }} -roleRef: - kind: Role - name: csi:controller:external-provisioner - apiGroup: rbac.authorization.k8s.io - -# ======== -# attacher -# ======== -# Source https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: d8:{{ .Chart.Name }}:csi:controller:external-attacher - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -rules: -- apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] -- apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] -- apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] -- apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: d8:{{ .Chart.Name }}:csi:controller:external-attacher - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -subjects: -- kind: ServiceAccount - name: csi - namespace: d8-{{ .Chart.Name }} -roleRef: - kind: ClusterRole - name: d8:{{ .Chart.Name }}:csi:controller:external-attacher - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi:controller:external-attacher - namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -rules: -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi:controller:external-attacher - namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -subjects: -- kind: ServiceAccount - name: csi - namespace: d8-{{ .Chart.Name }} -roleRef: - kind: Role - name: csi:controller:external-attacher - apiGroup: rbac.authorization.k8s.io - -# ======= -# resizer -# ======= -# Source https://github.com/kubernetes-csi/external-resizer/blob/master/deploy/kubernetes/rbac.yaml ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: d8:{{ .Chart.Name }}:csi:controller:external-resizer - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -rules: -- apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "patch"] -- apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: d8:{{ .Chart.Name }}:csi:controller:external-resizer - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -subjects: -- kind: ServiceAccount - name: csi - namespace: d8-{{ .Chart.Name }} -roleRef: - kind: ClusterRole - name: d8:{{ .Chart.Name }}:csi:controller:external-resizer - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi:controller:external-resizer - namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -rules: -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi:controller:external-resizer - namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -subjects: -- kind: ServiceAccount - name: csi - namespace: d8-{{ .Chart.Name }} -roleRef: - kind: Role - name: csi:controller:external-resizer - apiGroup: rbac.authorization.k8s.io -# ======== -# snapshotter -# ======== -# Source https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: d8:{{ .Chart.Name }}:csi:controller:external-snapshotter - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -rules: -- apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: d8:{{ .Chart.Name }}:csi:controller:external-snapshotter - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -subjects: -- kind: ServiceAccount - name: csi - namespace: d8-{{ .Chart.Name }} -roleRef: - kind: ClusterRole - name: d8:{{ .Chart.Name }}:csi:controller:external-snapshotter - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi:controller:external-snapshotter - namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -rules: -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi:controller:external-snapshotter - namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "app" "csi-controller")) | nindent 2 }} -subjects: -- kind: ServiceAccount - name: csi - namespace: d8-{{ .Chart.Name }} -roleRef: - kind: Role - name: csi:controller:external-snapshotter - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/helm_lib/templates/_csi_node.tpl b/charts/helm_lib/templates/_csi_node.tpl deleted file mode 100644 index 318b4bd4d..000000000 --- a/charts/helm_lib/templates/_csi_node.tpl +++ /dev/null @@ -1,397 +0,0 @@ -{{- define "node_driver_registrar_resources" }} -cpu: 12m -memory: 25Mi -{{- end }} - -{{- define "node_resources" }} -cpu: 12m -memory: 25Mi -{{- end }} - -{{- /* Usage: {{ include "helm_lib_csi_node_manifests" (list . $config) }} */ -}} -{{- define "helm_lib_csi_node_manifests" }} - {{- $context := index . 0 }} - - {{- $config := index . 1 }} - {{- $fullname := $config.fullname | default "csi-node" }} - {{- $nodeImage := $config.nodeImage | required "$config.nodeImage is required" }} - {{- $driverFQDN := $config.driverFQDN | required "$config.driverFQDN is required" }} - {{- $serviceAccount := $config.serviceAccount | default "" }} - {{- $additionalNodeVPA := $config.additionalNodeVPA }} - {{- $additionalNodeEnvs := $config.additionalNodeEnvs }} - {{- $additionalNodeArgs := $config.additionalNodeArgs }} - {{- $additionalNodeVolumes := $config.additionalNodeVolumes }} - {{- $additionalNodeVolumeMounts := $config.additionalNodeVolumeMounts }} - {{- $additionalNodeLivenessProbesCmd := $config.additionalNodeLivenessProbesCmd }} - {{- $livenessProbePort := $config.livenessProbePort }} - {{- $additionalNodeSelectorTerms := $config.additionalNodeSelectorTerms }} - {{- $customNodeSelector := $config.customNodeSelector }} - {{- $forceCsiNodeAndStaticNodesDepoloy := $config.forceCsiNodeAndStaticNodesDepoloy | default false }} - {{- $setSysAdminCapability := $config.setSysAdminCapability | default false }} - {{- $additionalContainers := $config.additionalContainers }} - {{- $initContainers := $config.initContainers }} - {{- $additionalPullSecrets := $config.additionalPullSecrets }} - {{- $csiNodeLifecycle := $config.csiNodeLifecycle | default false }} - {{- $csiNodeDriverRegistrarLifecycle := $config.csiNodeDriverRegistrarLifecycle | default false }} - {{- $additionalCsiNodePodAnnotations := $config.additionalCsiNodePodAnnotations | default false }} - {{- $csiNodeHostNetwork := $config.csiNodeHostNetwork | default "true" }} - {{- $csiNodeHostPID := $config.csiNodeHostPID | default "false" }} - {{- $dnsPolicy := $config.dnsPolicy | default "ClusterFirstWithHostNet" }} - {{- $securityPolicyExceptionEnabled := $config.securityPolicyExceptionEnabled | default false }} - {{- $kubernetesSemVer := semver $context.Values.global.discovery.kubernetesVersion }} - {{- $driverRegistrarImage := include "helm_lib_csi_image_with_common_fallback" (list $context "csiNodeDriverRegistrar" $kubernetesSemVer) }} - {{- if $driverRegistrarImage }} - {{- if or $forceCsiNodeAndStaticNodesDepoloy (include "_helm_lib_cloud_or_hybrid_cluster" $context) }} - {{- if ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} ---- -apiVersion: autoscaling.k8s.io/v1 -kind: VerticalPodAutoscaler -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" "csi-node")) | nindent 2 }} -spec: - targetRef: - apiVersion: "apps/v1" - kind: DaemonSet - name: {{ $fullname }} - updatePolicy: - updateMode: "InPlaceOrRecreate" - resourcePolicy: - containerPolicies: - {{- if $additionalNodeVPA }} - {{- $additionalNodeVPA | toYaml | nindent 4 }} - {{- end }} - - containerName: "node-driver-registrar" - minAllowed: - {{- include "node_driver_registrar_resources" $context | nindent 8 }} - maxAllowed: - cpu: 25m - memory: 50Mi - - containerName: "node" - minAllowed: - {{- include "node_resources" $context | nindent 8 }} - maxAllowed: - cpu: 25m - memory: 50Mi - {{- end }} ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" "csi-node")) | nindent 2 }} -spec: - updateStrategy: - type: RollingUpdate - selector: - matchLabels: - app: {{ $fullname }} - template: - metadata: - labels: - app: {{ $fullname }} - {{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} - security.deckhouse.io/security-policy-exception: {{ $fullname }} - {{- end }} - {{- if or (hasPrefix "cloud-provider-" $context.Chart.Name) ($additionalCsiNodePodAnnotations) }} - annotations: - {{- if hasPrefix "cloud-provider-" $context.Chart.Name }} - cloud-config-checksum: {{ include (print $context.Template.BasePath "/cloud-controller-manager/secret.yaml") $context | sha256sum }} - {{- end }} - {{- if $additionalCsiNodePodAnnotations }} - {{- $additionalCsiNodePodAnnotations | toYaml | nindent 8 }} - {{- end }} - {{- end }} - spec: - {{- if $customNodeSelector }} - nodeSelector: - {{- $customNodeSelector | toYaml | nindent 8 }} - {{- else }} - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - operator: In - key: node.deckhouse.io/type - values: - - CloudEphemeral - - CloudPermanent - - CloudStatic - {{- if $forceCsiNodeAndStaticNodesDepoloy }} - - Static - {{- end }} - {{- if $additionalNodeSelectorTerms }} - {{- $additionalNodeSelectorTerms | toYaml | nindent 14 }} - {{- end }} - {{- end }} - imagePullSecrets: - - name: deckhouse-registry - {{- if $additionalPullSecrets }} - {{- $additionalPullSecrets | toYaml | nindent 6 }} - {{- end }} - {{- include "helm_lib_priority_class" (tuple $context "system-node-critical") | nindent 6 }} - {{- include "helm_lib_tolerations" (tuple $context "any-node" "with-no-csi" "with-uninitialized") | nindent 6 }} - {{- include "helm_lib_module_pod_security_context_run_as_user_root" . | nindent 6 }} - hostNetwork: {{ $csiNodeHostNetwork }} - hostPID: {{ $csiNodeHostPID }} - {{- if eq $csiNodeHostNetwork "true" }} - dnsPolicy: {{ $dnsPolicy | quote }} - {{- end }} - containers: - - name: node-driver-registrar - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true "uid" "0" "runAsNonRoot" false) | nindent 8 }} - image: {{ $driverRegistrarImage | quote }} - args: - - "--v=5" - - "--csi-address=$(CSI_ENDPOINT)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - {{- if $livenessProbePort }} - - "--http-endpoint=:{{ $livenessProbePort }}" - {{- end }} - env: - - name: CSI_ENDPOINT - value: "/csi/csi.sock" - - name: DRIVER_REG_SOCK_PATH - value: "/var/lib/kubelet/csi-plugins/{{ $driverFQDN }}/csi.sock" - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- if $csiNodeDriverRegistrarLifecycle }} - lifecycle: - {{- $csiNodeDriverRegistrarLifecycle | toYaml | nindent 10 }} - {{- end }} - {{- if $additionalNodeLivenessProbesCmd }} - livenessProbe: - initialDelaySeconds: 3 - exec: - command: - {{- $additionalNodeLivenessProbesCmd | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_only_logs" 10 | nindent 12 }} - {{- if not ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "node_driver_registrar_resources" $context | nindent 12 }} - {{- end }} - - name: node - securityContext: - allowPrivilegeEscalation: true - privileged: true - readOnlyRootFilesystem: true - seccompProfile: - type: RuntimeDefault - capabilities: - drop: - - ALL - {{- if $setSysAdminCapability }} - add: - - SYS_ADMIN - {{- end }} - image: {{ $nodeImage }} - args: - {{- if $additionalNodeArgs }} - {{- $additionalNodeArgs | toYaml | nindent 8 }} - {{- end }} - {{- if $additionalNodeEnvs }} - env: - {{- $additionalNodeEnvs | toYaml | nindent 8 }} - {{- end }} - {{- if $csiNodeLifecycle }} - lifecycle: - {{- $csiNodeLifecycle | toYaml | nindent 10 }} - {{- end }} - {{- if $livenessProbePort }} - livenessProbe: - httpGet: - path: /healthz - port: {{ $livenessProbePort }} - initialDelaySeconds: 5 - timeoutSeconds: 5 - {{- end }} - volumeMounts: - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" - - name: plugin-dir - mountPath: /csi - - name: device-dir - mountPath: /dev - {{- if $additionalNodeVolumeMounts }} - {{- $additionalNodeVolumeMounts | toYaml | nindent 8 }} - {{- end }} - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- if not ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "node_resources" $context | nindent 12 }} - {{- end }} - - {{- if $additionalContainers }} - {{- $additionalContainers | toYaml | nindent 6 }} - {{- end }} - - {{- if $initContainers }} - initContainers: - {{- range $initContainer := $initContainers }} - - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 12 }} - {{- $initContainer | toYaml | nindent 8 }} - {{- end }} - {{- end }} - - serviceAccount: {{ $serviceAccount | quote }} - serviceAccountName: {{ $serviceAccount | quote }} - automountServiceAccountToken: true - volumes: - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/csi-plugins/{{ $driverFQDN }}/ - type: DirectoryOrCreate - - name: device-dir - hostPath: - path: /dev - type: Directory - - {{- if $additionalNodeVolumes }} - {{- $additionalNodeVolumes | toYaml | nindent 6 }} - {{- end }} - -{{- if and $securityPolicyExceptionEnabled ($context.Values.global.enabledModules | has "admission-policy-engine-crd") }} ---- -apiVersion: deckhouse.io/v1alpha1 -kind: SecurityPolicyException -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} -spec: - securityContext: - privileged: - allowedValue: true - metadata: - description: | - Allow privileged mode for CSI Node Driver. - The CSI Node Driver requires privileged access to perform critical storage operations such as mounting/unmounting volumes, formatting block devices, and interacting directly with the host kernel for disk management. - runAsNonRoot: - allowedValue: false - metadata: - description: | - Allow running as root for CSI Node Driver. - The CSI Node Driver requires root access to perform privileged storage operations on the host, including device management and filesystem mounting. - allowPrivilegeEscalation: - allowedValue: true - metadata: - description: | - Allow privilege escalation for CSI Node Driver. - The node plugin may need to escalate privileges during mount, unmount, and device operations that rely on setuid helpers or additional capabilities beyond the container's initial security context. - runAsUser: - allowedValues: - - 0 - metadata: - description: | - Allow running as root user (UID 0) for CSI Node Driver. - The CSI Node Driver and node-driver-registrar require root privileges to perform storage operations, interact with host devices, and manage volume mounts. - {{- if $setSysAdminCapability }} - capabilities: - allowedValues: - add: - - SYS_ADMIN - metadata: - description: | - Allow SYS_ADMIN capability for CSI Node Driver. - The CSI Node Driver requires SYS_ADMIN capability to perform privileged filesystem operations such as mounting, unmounting, and managing block devices on the host. - {{- end }} - - {{- if or (eq $csiNodeHostNetwork "true") (ne $csiNodeHostPID "false") }} - network: - {{- if eq $csiNodeHostNetwork "true" }} - hostNetwork: - allowedValue: true - metadata: - description: | - Allow host network access for CSI Node Driver. - The CSI Node Driver requires host network access to communicate with the CSI Controller, coordinate volume attachment operations, and synchronize storage metadata across the cluster. - {{- end }} - {{- if ne $csiNodeHostPID "false" }} - hostPID: - allowedValue: true - metadata: - description: | - Allow host PID namespace access for CSI Node Driver. - The CSI Node Driver requires host PID namespace access to interact with host processes for storage operations such as detecting mount points and managing device attachments. - {{- end }} - {{- end }} - - volumes: - types: - allowedValues: - - hostPath - metadata: - description: | - Allow hostPath volume type for CSI Node. - The CSI Node Driver requires hostPath volumes to access host filesystem paths for proper operation, including communication with the container runtime and access to block devices. - hostPath: - allowedValues: - - path: /var/lib/kubelet/plugins_registry/ - readOnly: false - metadata: - description: | - Allow access to the CSI plugin registry directory. - CSI Node Driver registers itself in this directory to enable dynamic discovery and communication with the kubelet. - - path: /var/lib/kubelet - readOnly: false - metadata: - description: | - Allow access to the kubelet root directory. - Required for CSI Node Driver to manage volume mounts and access kubelet data structures. - - path: /var/lib/kubelet/csi-plugins/{{ $driverFQDN }}/ - readOnly: false - metadata: - description: | - Allow access to the CSI plugin directory. - This directory contains the CSI driver socket and persistent data required for volume attachment and mounting operations. - - path: /dev - readOnly: false - metadata: - description: | - Allow access to host device directory. - CSI Node Driver requires access to /dev to manage block devices and perform disk operations for persistent volumes. - {{- if $additionalNodeVolumes }} - {{- range $volume := $additionalNodeVolumes }} - {{- if $volume.hostPath }} - {{- $readOnly := false }} - {{- range $volumeMount := $additionalNodeVolumeMounts }} - {{- if eq $volumeMount.name $volume.name }} - {{- $readOnly = (default false $volumeMount.readOnly) }} - {{- end }} - {{- end }} - - path: {{ $volume.hostPath.path }} - readOnly: {{ $readOnly }} - metadata: - description: | - Allow access to additional hostPath volume at {{ $volume.hostPath.path }}. - This additional hostPath volume is required by the CSI Node Driver for extended storage operations specific to the cloud provider implementation. - {{- end }} - {{- end }} - {{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/helm_lib/templates/_dns_policy.tpl b/charts/helm_lib/templates/_dns_policy.tpl deleted file mode 100644 index 31d3e73ec..000000000 --- a/charts/helm_lib/templates/_dns_policy.tpl +++ /dev/null @@ -1,12 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_dns_policy_bootstraping_state" (list . "Default" "ClusterFirstWithHostNet") }} */ -}} -{{- /* returns the proper dnsPolicy value depending on the cluster bootstrap phase */ -}} -{{- define "helm_lib_dns_policy_bootstraping_state" }} -{{- $context := index . 0 }} -{{- $valueDuringBootstrap := index . 1 }} -{{- $valueAfterBootstrap := index . 2 }} -{{- if $context.Values.global.clusterIsBootstrapped }} -{{- printf $valueAfterBootstrap }} -{{- else }} -{{- printf $valueDuringBootstrap }} -{{- end }} -{{- end }} diff --git a/charts/helm_lib/templates/_enable_ds_eviction.tpl b/charts/helm_lib/templates/_enable_ds_eviction.tpl deleted file mode 100644 index b912c050d..000000000 --- a/charts/helm_lib/templates/_enable_ds_eviction.tpl +++ /dev/null @@ -1,6 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_prevent_ds_eviction_annotation" . }} */ -}} -{{- /* Adds `cluster-autoscaler.kubernetes.io/enable-ds-eviction` annotation to manage DaemonSet eviction by the Cluster Autoscaler. */ -}} -{{- /* This is important to prevent the eviction of DaemonSet pods during cluster scaling. */ -}} -{{- define "helm_lib_prevent_ds_eviction_annotation" -}} -cluster-autoscaler.kubernetes.io/enable-ds-eviction: "false" -{{- end }} diff --git a/charts/helm_lib/templates/_envs_for_proxy.tpl b/charts/helm_lib/templates/_envs_for_proxy.tpl deleted file mode 100644 index 398d7c43c..000000000 --- a/charts/helm_lib/templates/_envs_for_proxy.tpl +++ /dev/null @@ -1,43 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_envs_for_proxy" . }} or {{ include "helm_lib_envs_for_proxy" (list . (list "extra1" "extra2")) }} */ -}} -{{- /* Add HTTP_PROXY, HTTPS_PROXY and NO_PROXY environment variables for container */ -}} -{{- /* depends on [proxy settings](https://deckhouse.io/products/kubernetes-platform/documentation/v1/reference/api/global.html#parameters-modules-proxy) */ -}} -{{- define "helm_lib_envs_for_proxy" }} - {{- /* Template context with .Values, .Chart, etc */ -}} - {{- /* List of additional NO_PROXY entries (optional) */ -}} - {{- $context := . -}} - {{- $extraNoProxy := list -}} - - {{- /* If a list is passed, then the first element is the context, and the second is the extraNoProxy list. */ -}} - {{- if kindIs "slice" . }} - {{- $context = index . 0 -}} - {{- $extraNoProxy = index . 1 -}} - {{- end }} - - {{- if $context.Values.global.clusterConfiguration }} - {{- if $context.Values.global.clusterConfiguration.proxy }} - {{- if $context.Values.global.clusterConfiguration.proxy.httpProxy }} -- name: HTTP_PROXY - value: {{ $context.Values.global.clusterConfiguration.proxy.httpProxy | quote }} -- name: http_proxy - value: {{ $context.Values.global.clusterConfiguration.proxy.httpProxy | quote }} - {{- end }} - {{- if $context.Values.global.clusterConfiguration.proxy.httpsProxy }} -- name: HTTPS_PROXY - value: {{ $context.Values.global.clusterConfiguration.proxy.httpsProxy | quote }} -- name: https_proxy - value: {{ $context.Values.global.clusterConfiguration.proxy.httpsProxy | quote }} - {{- end }} - {{- $noProxy := list "127.0.0.1" "169.254.169.254" "registry.d8-system.svc" $context.Values.global.clusterConfiguration.clusterDomain $context.Values.global.clusterConfiguration.podSubnetCIDR $context.Values.global.clusterConfiguration.serviceSubnetCIDR }} - {{- if $context.Values.global.clusterConfiguration.proxy.noProxy }} - {{- $noProxy = concat $noProxy $context.Values.global.clusterConfiguration.proxy.noProxy }} - {{- end }} - {{- if $extraNoProxy }} - {{- $noProxy = concat $noProxy $extraNoProxy }} - {{- end }} -- name: NO_PROXY - value: {{ $noProxy | join "," | quote }} -- name: no_proxy - value: {{ $noProxy | join "," | quote }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/helm_lib/templates/_high_availability.tpl b/charts/helm_lib/templates/_high_availability.tpl deleted file mode 100644 index 8c7da23ac..000000000 --- a/charts/helm_lib/templates/_high_availability.tpl +++ /dev/null @@ -1,39 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_is_ha_to_value" (list . yes no) }} */ -}} -{{- /* returns value "yes" if cluster is highly available, else — returns "no" */ -}} -{{- define "helm_lib_is_ha_to_value" }} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $yes := index . 1 -}} {{- /* Yes value */ -}} - {{- $no := index . 2 -}} {{- /* No value */ -}} - - {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) }} - - {{- if hasKey $module_values "highAvailability" -}} - {{- if $module_values.highAvailability -}} {{- $yes -}} {{- else -}} {{- $no -}} {{- end -}} - {{- else if hasKey $context.Values.global "highAvailability" -}} - {{- if $context.Values.global.highAvailability -}} {{- $yes -}} {{- else -}} {{- $no -}} {{- end -}} - {{- else -}} - {{- if $context.Values.global.discovery.clusterControlPlaneIsHighlyAvailable -}} {{- $yes -}} {{- else -}} {{- $no -}} {{- end -}} - {{- end -}} -{{- end }} - -{{- /* Usage: {{- if (include "helm_lib_ha_enabled" .) }} */ -}} -{{- /* returns empty value, which is treated by go template as false */ -}} -{{- define "helm_lib_ha_enabled" }} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - - {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) }} - - {{- if hasKey $module_values "highAvailability" -}} - {{- if $module_values.highAvailability -}} - "not empty string" - {{- end -}} - {{- else if hasKey $context.Values.global "highAvailability" -}} - {{- if $context.Values.global.highAvailability -}} - "not empty string" - {{- end -}} - {{- else -}} - {{- if $context.Values.global.discovery.clusterControlPlaneIsHighlyAvailable -}} - "not empty string" - {{- end -}} - {{- end -}} -{{- end -}} diff --git a/charts/helm_lib/templates/_kube_rbac_proxy.tpl b/charts/helm_lib/templates/_kube_rbac_proxy.tpl deleted file mode 100644 index af9f7a447..000000000 --- a/charts/helm_lib/templates/_kube_rbac_proxy.tpl +++ /dev/null @@ -1,21 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_kube_rbac_proxy_ca_certificate" (list . "namespace") }} */ -}} -{{- /* Renders configmap with kube-rbac-proxy CA certificate which uses to verify the kube-rbac-proxy clients. */ -}} -{{- define "helm_lib_kube_rbac_proxy_ca_certificate" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -{{- /* Namespace where CA configmap will be created */ -}} - {{- $context := index . 0 }} - {{- $namespace := index . 1 }} ---- -apiVersion: v1 -data: - ca.crt: | - {{ $context.Values.global.internal.modules.kubeRBACProxyCA.cert | nindent 4 }} -kind: ConfigMap -metadata: - annotations: - kubernetes.io/description: | - Contains a CA bundle that can be used to verify the kube-rbac-proxy clients. - {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} - name: kube-rbac-proxy-ca.crt - namespace: {{ $namespace }} -{{- end }} diff --git a/charts/helm_lib/templates/_module_controller.tpl b/charts/helm_lib/templates/_module_controller.tpl deleted file mode 100644 index 3e855392e..000000000 --- a/charts/helm_lib/templates/_module_controller.tpl +++ /dev/null @@ -1,532 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_controller_resources" . }} */ -}} -{{- /* Returns default controller resources */ -}} -{{- define "helm_lib_module_controller_resources" }} -cpu: 10m -memory: 25Mi -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_webhooks_resources" . }} */ -}} -{{- /* Returns default webhooks resources */ -}} -{{- define "helm_lib_module_webhooks_resources" }} -cpu: 10m -memory: 50Mi -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_controller_manifests" (list . $config) }} */ -}} -{{- /* - Generates controller Deployment, VPA and PDB manifests. - - $config parameters: - - fullname: name for the deployment (default: "controller") - - valuesKey: key to access module values (e.g., "csiHpe", "csiS3") - - webhookEnabled: whether to include webhook container (default: false) - - webhookCertPath: path to webhook cert in values (e.g., "internal.customWebhookCert") - - onMasterNode: use master node selector and tolerations (default: true) - - priorityClass: priority class name (default: "system-cluster-critical") - - podSecurityContext: "deckhouse" or "nobody" (default: "nobody") - - additionalLabels: additional labels for VPA - - controllerMaxCpu: max CPU for controller VPA (default: "200m") - - controllerMaxMemory: max memory for controller VPA (default: "100Mi") - - webhooksMaxCpu: max CPU for webhooks VPA (default: "20m") - - webhooksMaxMemory: max memory for webhooks VPA (default: "100Mi") - - additionalContainers: additional containers to add to the pod - - additionalVolumes: additional volumes to add to the pod - - additionalControllerVolumeMounts: additional volume mounts for controller - - additionalControllerEnvs: additional environment variables for controller - - controllerImage: custom controller image (default: uses helm_lib_module_image) - - controllerImageName: image name for helm_lib_module_image (default: "controller") - - webhooksImageName: image name for webhooks (default: "webhooks") - - webhooksPort: port for webhooks container (default: 8443) - - webhooksCertMountPath: mount path for webhook certs (default: "/etc/webhook/certs") - - webhooksCommand: command for webhooks container - - controllerPort: port for controller probes (default: 8081) - - controllerMetricsPort: port for controller metrics (optional, no port exposed if not set) -*/ -}} -{{- define "helm_lib_module_controller_manifests" }} - {{- $context := index . 0 }} - {{- $config := index . 1 }} - - {{- $fullname := $config.fullname | default "controller" }} - {{- $valuesKey := $config.valuesKey | required "$config.valuesKey is required" }} - {{- $webhookEnabled := dig "webhookEnabled" false $config }} - {{- $webhookCertPath := $config.webhookCertPath | default "internal.customWebhookCert" }} - {{- $onMasterNode := dig "onMasterNode" true $config }} - {{- $priorityClass := $config.priorityClass | default "system-cluster-critical" }} - {{- $podSecurityContext := $config.podSecurityContext | default "nobody" }} - {{- $additionalLabels := $config.additionalLabels | default dict }} - {{- $controllerMaxCpu := $config.controllerMaxCpu | default "200m" }} - {{- $controllerMaxMemory := $config.controllerMaxMemory | default "100Mi" }} - {{- $webhooksMaxCpu := $config.webhooksMaxCpu | default "20m" }} - {{- $webhooksMaxMemory := $config.webhooksMaxMemory | default "100Mi" }} - {{- $additionalContainers := $config.additionalContainers }} - {{- $additionalVolumes := $config.additionalVolumes }} - {{- $additionalControllerVolumeMounts := $config.additionalControllerVolumeMounts }} - {{- $additionalControllerEnvs := $config.additionalControllerEnvs }} - {{- $controllerImage := $config.controllerImage }} - {{- $controllerImageName := $config.controllerImageName | default "controller" }} - {{- $webhooksImageName := $config.webhooksImageName | default "webhooks" }} - {{- $webhooksPort := $config.webhooksPort | default 8443 }} - {{- $webhooksCertMountPath := $config.webhooksCertMountPath | default "/etc/webhook/certs" }} - {{- $webhooksCommand := $config.webhooksCommand }} - {{- $controllerPort := $config.controllerPort | default 8081 }} - {{- $controllerMetricsPort := $config.controllerMetricsPort }} - - {{- /* Get module values */ -}} - {{- $moduleValues := index $context.Values $valuesKey }} - - {{- /* Build VPA labels */ -}} - {{- $vpaLabels := dict "app" $fullname }} - {{- if $onMasterNode }} - {{- $vpaLabels = merge $vpaLabels (dict "workload-resource-policy.deckhouse.io" "master") }} - {{- end }} - {{- $vpaLabels = merge $vpaLabels $additionalLabels }} - - {{- if ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} ---- -apiVersion: autoscaling.k8s.io/v1 -kind: VerticalPodAutoscaler -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context $vpaLabels) | nindent 2 }} -spec: - targetRef: - apiVersion: "apps/v1" - kind: Deployment - name: {{ $fullname }} - updatePolicy: - updateMode: "Initial" - resourcePolicy: - containerPolicies: - - containerName: "controller" - minAllowed: - {{- include "helm_lib_module_controller_resources" $context | nindent 8 }} - maxAllowed: - cpu: {{ $controllerMaxCpu }} - memory: {{ $controllerMaxMemory }} - {{- if $webhookEnabled }} - - containerName: "webhooks" - minAllowed: - {{- include "helm_lib_module_webhooks_resources" $context | nindent 8 }} - maxAllowed: - cpu: {{ $webhooksMaxCpu }} - memory: {{ $webhooksMaxMemory }} - {{- end }} - {{- end }} ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -spec: - minAvailable: {{ include "helm_lib_is_ha_to_value" (list $context 1 0) }} - selector: - matchLabels: - app: {{ $fullname }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -spec: - revisionHistoryLimit: 2 - {{- if $onMasterNode }} - {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" $context | nindent 2 }} - {{- else }} - {{- include "helm_lib_deployment_strategy_and_replicas_for_ha" $context | nindent 2 }} - {{- end }} - selector: - matchLabels: - app: {{ $fullname }} - template: - metadata: - {{- if $webhookEnabled }} - {{- $certPath := printf "%s.ca" $webhookCertPath }} - {{- $certValue := $moduleValues }} - {{- range $part := (split "." $certPath) }} - {{- $certValue = index $certValue $part }} - {{- end }} - annotations: - checksum/ca: {{ $certValue | sha256sum | quote }} - {{- end }} - labels: - app: {{ $fullname }} - spec: - {{- include "helm_lib_priority_class" (tuple $context $priorityClass) | nindent 6 }} - {{- if $onMasterNode }} - {{- include "helm_lib_tolerations" (tuple $context "any-node" "with-uninitialized" "with-cloud-provider-uninitialized") | nindent 6 }} - {{- include "helm_lib_node_selector" (tuple $context "master") | nindent 6 }} - {{- else }} - {{- include "helm_lib_node_selector" (tuple $context "system") | nindent 6 }} - {{- include "helm_lib_tolerations" (tuple $context "system") | nindent 6 }} - {{- end }} - {{- include "helm_lib_pod_anti_affinity_for_ha" (list $context (dict "app" $fullname)) | nindent 6 }} - {{- if eq $podSecurityContext "deckhouse" }} - {{- include "helm_lib_module_pod_security_context_run_as_user_deckhouse" $context | nindent 6 }} - {{- else }} - {{- include "helm_lib_module_pod_security_context_run_as_user_nobody" $context | nindent 6 }} - {{- end }} - imagePullSecrets: - - name: {{ $context.Chart.Name }}-module-registry - serviceAccountName: {{ $fullname }} - containers: - - name: controller - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 10 }} - {{- if $controllerImage }} - image: {{ $controllerImage | quote }} - {{- else }} - image: {{ include "helm_lib_module_image" (list $context $controllerImageName) }} - {{- end }} - imagePullPolicy: IfNotPresent - readinessProbe: - httpGet: - path: /readyz - port: {{ $controllerPort }} - scheme: HTTP - initialDelaySeconds: 5 - failureThreshold: 2 - periodSeconds: 1 - livenessProbe: - httpGet: - path: /healthz - port: {{ $controllerPort }} - scheme: HTTP - periodSeconds: 1 - failureThreshold: 3 - {{- if $controllerMetricsPort }} - ports: - - name: metrics - containerPort: {{ $controllerMetricsPort }} - protocol: TCP - {{- end }} - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 14 }} -{{- if not ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "helm_lib_module_controller_resources" $context | nindent 14 }} -{{- end }} - env: - - name: LOG_LEVEL - value: {{ include "helm_lib_module_controller_log_level" (list $context $valuesKey) | quote }} - - name: CONTROLLER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - {{- if $additionalControllerEnvs }} - {{- $additionalControllerEnvs | toYaml | nindent 12 }} - {{- end }} - {{- if $additionalControllerVolumeMounts }} - volumeMounts: - {{- $additionalControllerVolumeMounts | toYaml | nindent 12 }} - {{- end }} - {{- if $webhookEnabled }} - - name: webhooks - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true "seccompProfile" true) | nindent 10 }} - {{- if $webhooksCommand }} - command: - {{- $webhooksCommand | toYaml | nindent 12 }} - {{- else }} - command: - - /webhooks - - -tls-cert-file={{ $webhooksCertMountPath }}/tls.crt - - -tls-key-file={{ $webhooksCertMountPath }}/tls.key - {{- end }} - image: {{ include "helm_lib_module_image" (list $context $webhooksImageName) }} - imagePullPolicy: IfNotPresent - volumeMounts: - - name: webhook-certs - mountPath: {{ $webhooksCertMountPath }} - readOnly: true - readinessProbe: - httpGet: - path: /healthz - port: {{ $webhooksPort }} - scheme: HTTPS - initialDelaySeconds: 5 - failureThreshold: 2 - periodSeconds: 1 - livenessProbe: - httpGet: - path: /healthz - port: {{ $webhooksPort }} - scheme: HTTPS - periodSeconds: 1 - failureThreshold: 3 - ports: - - name: https - containerPort: {{ $webhooksPort }} - protocol: TCP - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 14 }} -{{- if not ($context.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "helm_lib_module_webhooks_resources" $context | nindent 14 }} -{{- end }} - {{- end }} - {{- if $additionalContainers }} - {{- $additionalContainers | toYaml | nindent 8 }} - {{- end }} - {{- if or $webhookEnabled $additionalVolumes }} - volumes: - {{- if $webhookEnabled }} - - name: webhook-certs - secret: - secretName: webhooks-https-certs - {{- end }} - {{- if $additionalVolumes }} - {{- $additionalVolumes | toYaml | nindent 8 }} - {{- end }} - {{- end }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_module_controller_log_level" (list . "csiHpe") }} */ -}} -{{- /* Returns numeric log level from module values */ -}} -{{- define "helm_lib_module_controller_log_level" }} - {{- $context := index . 0 }} - {{- $valuesKey := index . 1 }} - {{- $moduleValues := index $context.Values $valuesKey }} - {{- $logLevel := $moduleValues.logLevel | default "INFO" }} - {{- if eq $logLevel "ERROR" -}} -0 - {{- else if eq $logLevel "WARN" -}} -1 - {{- else if eq $logLevel "INFO" -}} -2 - {{- else if eq $logLevel "DEBUG" -}} -3 - {{- else if eq $logLevel "TRACE" -}} -4 - {{- else -}} -2 - {{- end -}} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_module_webhook_service" (list . $config) }} */ -}} -{{- /* - Generates webhook Service manifest. - - $config parameters: - - fullname: name of the service (default: "webhooks") - - selectorApp: app label for selector (default: "controller") - - targetPort: target port name or number (default: "https") - - additionalPorts: list of additional ports (optional), each port is a dict with: - - name: port name (required) - - port: service port (required) - - targetPort: target port name or number (required) - - protocol: protocol (default: "TCP") -*/ -}} -{{- define "helm_lib_module_webhook_service" }} - {{- $context := index . 0 }} - {{- $config := index . 1 | default dict }} - - {{- $fullname := $config.fullname | default "webhooks" }} - {{- $selectorApp := $config.selectorApp | default "controller" }} - {{- $targetPort := $config.targetPort | default "https" }} - {{- $additionalPorts := $config.additionalPorts }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $selectorApp)) | nindent 2 }} -spec: - type: ClusterIP - ports: - - port: 443 - targetPort: {{ $targetPort }} - protocol: TCP - name: https - {{- range $additionalPorts }} - - name: {{ .name }} - port: {{ .port }} - targetPort: {{ .targetPort }} - protocol: {{ .protocol | default "TCP" }} - {{- end }} - selector: - app: {{ $selectorApp }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_module_validating_webhook_configuration" (list . $config) }} */ -}} -{{- /* - Generates ValidatingWebhookConfiguration manifest. - - $config parameters: - - name: webhook configuration name (required, e.g., "sc-validation") - - webhookName: webhook name suffix (required, e.g., "sc-validation") - - valuesKey: key to access module values (required, e.g., "csiHpe") - - webhookCertPath: path to webhook cert in values (default: "internal.customWebhookCert") - - serviceName: service name (default: "webhooks") - - path: webhook path (required, e.g., "/sc-validate") - - rules: webhook rules (required) - - matchConditions: optional match conditions - - sideEffects: side effects (default: "None") - - timeoutSeconds: timeout (default: 5) -*/ -}} -{{- define "helm_lib_module_validating_webhook_configuration" }} - {{- $context := index . 0 }} - {{- $config := index . 1 }} - - {{- $name := $config.name | required "$config.name is required" }} - {{- $webhookName := $config.webhookName | required "$config.webhookName is required" }} - {{- $valuesKey := $config.valuesKey | required "$config.valuesKey is required" }} - {{- $webhookCertPath := $config.webhookCertPath | default "internal.customWebhookCert" }} - {{- $serviceName := $config.serviceName | default "webhooks" }} - {{- $path := $config.path | required "$config.path is required" }} - {{- $rules := $config.rules | required "$config.rules is required" }} - {{- $matchConditions := $config.matchConditions }} - {{- $sideEffects := $config.sideEffects | default "None" }} - {{- $timeoutSeconds := $config.timeoutSeconds | default 5 }} - - {{- /* Get module values and cert */ -}} - {{- $moduleValues := index $context.Values $valuesKey }} - {{- $certPath := printf "%s.ca" $webhookCertPath }} - {{- $certValue := $moduleValues }} - {{- range $part := (split "." $certPath) }} - {{- $certValue = index $certValue $part }} - {{- end }} ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: "d8-{{ $context.Chart.Name }}-{{ $name }}" - {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} -webhooks: - - name: "d8-{{ $context.Chart.Name }}-{{ $webhookName }}.deckhouse.io" - rules: - {{- $rules | toYaml | nindent 6 }} - clientConfig: - service: - namespace: "d8-{{ $context.Chart.Name }}" - name: {{ $serviceName | quote }} - path: {{ $path | quote }} - caBundle: {{ $certValue | b64enc | quote }} - admissionReviewVersions: ["v1", "v1beta1"] - sideEffects: {{ $sideEffects }} - timeoutSeconds: {{ $timeoutSeconds }} - {{- if $matchConditions }} - matchConditions: - {{- $matchConditions | toYaml | nindent 6 }} - {{- end }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_module_webhook_certs_secret" (list . $config) }} */ -}} -{{- /* - Generates webhook TLS certificates Secret manifest. - - $config parameters: - - fullname: name of the secret (default: "webhooks-https-certs") - - valuesKey: key to access module values (required, e.g., "csiHpe", "csiS3") - - webhookCertPath: path to webhook cert in values (default: "internal.customWebhookCert") - - appLabel: app label for the secret (default: "webhooks") -*/ -}} -{{- define "helm_lib_module_webhook_certs_secret" }} - {{- $context := index . 0 }} - {{- $config := index . 1 }} - - {{- $fullname := $config.fullname | default "webhooks-https-certs" }} - {{- $valuesKey := $config.valuesKey | required "$config.valuesKey is required" }} - {{- $webhookCertPath := $config.webhookCertPath | default "internal.customWebhookCert" }} - {{- $appLabel := $config.appLabel | default "webhooks" }} - - {{- /* Get module values and certs */ -}} - {{- $moduleValues := index $context.Values $valuesKey }} - {{- $certValue := $moduleValues }} - {{- range $part := (split "." $webhookCertPath) }} - {{- $certValue = index $certValue $part }} - {{- end }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $appLabel)) | nindent 2 }} -type: kubernetes.io/tls -data: - ca.crt: {{ $certValue.ca | b64enc | quote }} - tls.crt: {{ $certValue.crt | b64enc | quote }} - tls.key: {{ $certValue.key | b64enc | quote }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_module_controller_rbac" (list . $config) }} */ -}} -{{- /* - Generates controller RBAC manifests (ServiceAccount, Role, ClusterRole, RoleBinding, ClusterRoleBinding). - - $config parameters: - - fullname: name for the resources (default: "controller") - - roleRules: rules for namespaced Role (required) - - clusterRoleRules: rules for ClusterRole (required) -*/ -}} -{{- define "helm_lib_module_controller_rbac" -}} - {{- $context := index . 0 -}} - {{- $config := index . 1 -}} - {{- $fullname := $config.fullname | default "controller" -}} - {{- $roleRules := $config.roleRules | required "$config.roleRules is required" -}} - {{- $clusterRoleRules := $config.clusterRoleRules | required "$config.clusterRoleRules is required" }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -rules: - {{- $roleRules | toYaml | nindent 2 }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: d8:{{ $context.Chart.Name }}:{{ $fullname }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -rules: - {{- $clusterRoleRules | toYaml | nindent 2 }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -subjects: - - kind: ServiceAccount - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} -roleRef: - kind: Role - name: {{ $fullname }} - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: d8:{{ $context.Chart.Name }}:{{ $fullname }} - {{- include "helm_lib_module_labels" (list $context (dict "app" $fullname)) | nindent 2 }} -subjects: - - kind: ServiceAccount - name: {{ $fullname }} - namespace: d8-{{ $context.Chart.Name }} -roleRef: - kind: ClusterRole - name: d8:{{ $context.Chart.Name }}:{{ $fullname }} - apiGroup: rbac.authorization.k8s.io -{{- end }} - - - diff --git a/charts/helm_lib/templates/_module_documentation_uri.tpl b/charts/helm_lib/templates/_module_documentation_uri.tpl deleted file mode 100644 index c1b94812d..000000000 --- a/charts/helm_lib/templates/_module_documentation_uri.tpl +++ /dev/null @@ -1,15 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_documentation_uri" (list . "") }} */ -}} -{{- /* returns rendered documentation uri using publicDomainTemplate or deckhouse.io domains*/ -}} -{{- define "helm_lib_module_documentation_uri" }} - {{- $default_doc_prefix := "https://deckhouse.io" -}} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $path_portion := index . 1 -}} {{- /* Path to the document */ -}} - {{- $uri := "" -}} - {{- if $context.Values.global.modules.publicDomainTemplate }} - {{- $uri = printf "%s://%s%s" (include "helm_lib_module_uri_scheme" $context) (include "helm_lib_module_public_domain" (list $context "documentation")) $path_portion -}} - {{- else }} - {{- $uri = printf "%s%s" $default_doc_prefix $path_portion -}} - {{- end -}} - - {{ $uri }} -{{- end }} diff --git a/charts/helm_lib/templates/_module_ephemeral_storage.tpl b/charts/helm_lib/templates/_module_ephemeral_storage.tpl deleted file mode 100644 index 4b2dd02ed..000000000 --- a/charts/helm_lib/templates/_module_ephemeral_storage.tpl +++ /dev/null @@ -1,15 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 }} */ -}} -{{- /* 50Mi for container logs `log-opts.max-file * log-opts.max-size` would be added to passed value */ -}} -{{- /* returns ephemeral-storage size for logs with extra space */ -}} -{{- define "helm_lib_module_ephemeral_storage_logs_with_extra" -}} -{{- /* Extra space in mebibytes */ -}} -ephemeral-storage: {{ add . 50 }}Mi -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_ephemeral_storage_only_logs" . }} */ -}} -{{- /* 50Mi for container logs `log-opts.max-file * log-opts.max-size` would be requested */ -}} -{{- /* returns ephemeral-storage size for only logs */ -}} -{{- define "helm_lib_module_ephemeral_storage_only_logs" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -ephemeral-storage: 50Mi -{{- end }} diff --git a/charts/helm_lib/templates/_module_gateway.tpl b/charts/helm_lib/templates/_module_gateway.tpl deleted file mode 100644 index 2ad4e63f1..000000000 --- a/charts/helm_lib/templates/_module_gateway.tpl +++ /dev/null @@ -1,22 +0,0 @@ -{{- /* Usage: {{- include "helm_lib_module_gateway" (list . $gateway) */ -}} -{{- /* accepts a dict that is updated with current gateway name and namespace */ -}} -{{- define "helm_lib_module_gateway" -}} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $result := index . 1 -}} {{- /* An empty dict to update with current default gateway name and namespace */ -}} - {{- $g := dict -}} - - {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} - - {{- if hasKey $module_values "gatewayAPIGateway" -}} - {{- $g = $module_values.gatewayAPIGateway -}} - {{- else if hasKey $context.Values.global.modules "gatewayAPIGateway" -}} - {{- $g = $context.Values.global.modules.gatewayAPIGateway -}} - {{- else if and (hasKey $context.Values.global "discovery") (hasKey $context.Values.global.discovery "gatewayAPIDefaultGateway") -}} - {{- $g = $context.Values.global.discovery.gatewayAPIDefaultGateway -}} - {{- end -}} - - {{- if and $g.name $g.namespace -}} - {{- $_ := set $result "name" $g.name -}} - {{- $_ := set $result "namespace" $g.namespace -}} - {{- end -}} -{{- end -}} diff --git a/charts/helm_lib/templates/_module_generate_common_name.tpl b/charts/helm_lib/templates/_module_generate_common_name.tpl deleted file mode 100644 index fb142f824..000000000 --- a/charts/helm_lib/templates/_module_generate_common_name.tpl +++ /dev/null @@ -1,13 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_generate_common_name" (list . "") }} */ -}} -{{- /* returns the commonName parameter for use in the Certificate custom resource(cert-manager) */ -}} -{{- define "helm_lib_module_generate_common_name" }} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $name_portion := index . 1 -}} {{- /* Name portion */ -}} - - {{- $domain := include "helm_lib_module_public_domain" (list $context $name_portion) -}} - - {{- $domain_length := len $domain -}} - {{- if le $domain_length 64 -}} -commonName: {{ $domain }} - {{- end -}} -{{- end }} diff --git a/charts/helm_lib/templates/_module_https.tpl b/charts/helm_lib/templates/_module_https.tpl deleted file mode 100644 index 24c5d3735..000000000 --- a/charts/helm_lib/templates/_module_https.tpl +++ /dev/null @@ -1,201 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_uri_scheme" . }} */ -}} -{{- /* return module uri scheme "http" or "https" */ -}} -{{- define "helm_lib_module_uri_scheme" -}} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $mode := "" -}} - - {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} - {{- if hasKey $module_values "https" -}} - {{- if hasKey $module_values.https "mode" -}} - {{- $mode = $module_values.https.mode -}} - {{- else }} - {{- $mode = $context.Values.global.modules.https.mode | default "" -}} - {{- end }} - {{- else }} - {{- $mode = $context.Values.global.modules.https.mode | default "" -}} - {{- end }} - - - {{- if eq "Disabled" $mode -}} - http - {{- else -}} - https - {{- end -}} -{{- end -}} - -{{- /* Usage: {{ $https_values := include "helm_lib_https_values" . | fromYaml }} */ -}} -{{- define "helm_lib_https_values" -}} - {{- $context := . -}} - {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} - {{- $mode := "" -}} - {{- $certManagerClusterIssuerName := "" -}} - - {{- if hasKey $module_values "https" -}} - {{- if hasKey $module_values.https "mode" -}} - {{- $mode = $module_values.https.mode -}} - {{- if eq $mode "CertManager" -}} - {{- if not (hasKey $module_values.https "certManager") -}} - {{- cat ".https.certManager.clusterIssuerName is mandatory when .https.mode is set to CertManager" | fail -}} - {{- end -}} - {{- if hasKey $module_values.https.certManager "clusterIssuerName" -}} - {{- $certManagerClusterIssuerName = $module_values.https.certManager.clusterIssuerName -}} - {{- else -}} - {{- cat ".https.certManager.clusterIssuerName is mandatory when .https.mode is set to CertManager" | fail -}} - {{- end -}} - {{- end -}} - {{- else -}} - {{- cat ".https.mode is mandatory when .https is defined" | fail -}} - {{- end -}} - {{- end -}} - - {{- if empty $mode -}} - {{- $mode = $context.Values.global.modules.https.mode -}} - {{- if eq $mode "CertManager" -}} - {{- $certManagerClusterIssuerName = $context.Values.global.modules.https.certManager.clusterIssuerName -}} - {{- end -}} - {{- end -}} - - {{- if not (has $mode (list "Disabled" "CertManager" "CustomCertificate" "OnlyInURI")) -}} - {{- cat "Unknown https.mode:" $mode | fail -}} - {{- end -}} - - {{- if and (eq $mode "CertManager") (not ($context.Values.global.enabledModules | has "cert-manager")) -}} - {{- cat "https.mode has value CertManager but cert-manager module not enabled" | fail -}} - {{- end -}} - -mode: {{ $mode }} - {{- if eq $mode "CertManager" }} -certManager: - clusterIssuerName: {{ $certManagerClusterIssuerName }} - {{- end -}} - -{{- end -}} - -{{- /* Usage: {{ if (include "helm_lib_module_https_mode" .) }} */ -}} -{{- /* returns https mode for module */ -}} -{{- define "helm_lib_module_https_mode" -}} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $https_values := include "helm_lib_https_values" $context | fromYaml -}} - {{- $https_values.mode -}} -{{- end -}} - -{{- /* Usage: {{ include "helm_lib_module_https_cert_manager_cluster_issuer_name" . }} */ -}} -{{- /* returns cluster issuer name */ -}} -{{- define "helm_lib_module_https_cert_manager_cluster_issuer_name" -}} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $https_values := include "helm_lib_https_values" $context | fromYaml -}} - {{- $https_values.certManager.clusterIssuerName -}} -{{- end -}} - -{{- /* Usage: {{ if (include "helm_lib_module_https_cert_manager_cluster_issuer_is_dns01_challenge_solver" .) }} */ -}} -{{- define "helm_lib_module_https_cert_manager_cluster_issuer_is_dns01_challenge_solver" -}} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- if has (include "helm_lib_module_https_cert_manager_cluster_issuer_name" $context) (list "route53" "cloudflare" "digitalocean" "clouddns") }} - "not empty string" - {{- end -}} -{{- end -}} - -{{- /* Usage: {{ include "helm_lib_module_https_cert_manager_acme_solver_challenge_settings" . | nindent 4 }} */ -}} -{{- define "helm_lib_module_https_cert_manager_acme_solver_challenge_settings" -}} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- if (include "helm_lib_module_https_cert_manager_cluster_issuer_is_dns01_challenge_solver" $context) }} -- dns01: - provider: {{ include "helm_lib_module_https_cert_manager_cluster_issuer_name" $context }} - {{- else }} -- http01: - ingressClass: {{ include "helm_lib_module_ingress_class" $context | quote }} - {{- end }} -{{- end -}} - -{{- /* Usage: {{ if (include "helm_lib_module_https_ingress_tls_enabled" .) }} */ -}} -{{- /* returns not empty string if tls should be enabled for the ingress */ -}} -{{- define "helm_lib_module_https_ingress_tls_enabled" -}} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - - {{- $mode := include "helm_lib_module_https_mode" $context -}} - - {{- if or (eq "CertManager" $mode) (eq "CustomCertificate" $mode) -}} - not empty string - {{- end -}} -{{- end -}} - -{{- /* Usage: {{ if (include "helm_lib_module_https_route_tls_enabled" .) }} */ -}} -{{- /* returns not empty string if tls should be enabled for the route */ -}} -{{- define "helm_lib_module_https_route_tls_enabled" -}} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - - {{- $mode := include "helm_lib_module_https_mode" $context -}} - - {{- if or (eq "CertManager" $mode) (eq "CustomCertificate" $mode) -}} - not empty string - {{- end -}} -{{- end -}} - -{{- /* Usage: {{ include "helm_lib_module_https_copy_custom_certificate" (list . "namespace" "secret_name_prefix") }} */ -}} -{{- /* Renders secret with [custom certificate](https://deckhouse.io/products/kubernetes-platform/documentation/v1/reference/api/global.html#parameters-modules-https-customcertificate) */ -}} -{{- /* in passed namespace with passed prefix */ -}} -{{- define "helm_lib_module_https_copy_custom_certificate" -}} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $namespace := index . 1 -}} {{- /* Namespace */ -}} - {{- $secret_name_prefix := index . 2 -}} {{- /* Secret name prefix */ -}} - {{- $mode := include "helm_lib_module_https_mode" $context -}} - {{- if eq $mode "CustomCertificate" -}} - {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} - {{- $customCertificateData := $module_values.internal.customCertificateData -}} - {{- if not $customCertificateData -}} - {{- fail (printf "internal.customCertificateData is required to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} - {{- end -}} - {{- if not (kindIs "map" $customCertificateData) -}} - {{- fail (printf "internal.customCertificateData must be a map to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} - {{- end -}} - {{- $tlsCrt := index $customCertificateData "tls.crt" -}} - {{- $tlsKey := index $customCertificateData "tls.key" -}} - {{- if not $tlsCrt -}} - {{- fail (printf "internal.customCertificateData.tls.crt is required to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} - {{- end -}} - {{- if not $tlsKey -}} - {{- fail (printf "internal.customCertificateData.tls.key is required to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} - {{- end -}} - {{- if not (kindIs "string" $tlsCrt) -}} - {{- fail (printf "internal.customCertificateData.tls.crt must be a string to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} - {{- end -}} - {{- if not (kindIs "string" $tlsKey) -}} - {{- fail (printf "internal.customCertificateData.tls.key must be a string to copy custom certificate for secret prefix '%s'" $secret_name_prefix) -}} - {{- end -}} - {{- $secret_name := include "helm_lib_module_https_secret_name" (list $context $secret_name_prefix) -}} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ $secret_name }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context) | nindent 2 }} -type: kubernetes.io/tls -data: -{{- if (hasKey $customCertificateData "ca.crt") }} - {{- $caCrt := index $customCertificateData "ca.crt" -}} - {{- if and $caCrt (kindIs "string" $caCrt) }} - ca.crt: {{ $caCrt | b64enc }} - {{- end }} -{{- end }} - tls.crt: {{ $tlsCrt | b64enc }} - tls.key: {{ $tlsKey | b64enc }} - {{- end -}} -{{- end -}} - -{{- /* Usage: {{ include "helm_lib_module_https_secret_name (list . "secret_name_prefix") }} */ -}} -{{- /* returns custom certificate name */ -}} -{{- define "helm_lib_module_https_secret_name" -}} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $secret_name_prefix := index . 1 -}} {{- /* Secret name prefix */ -}} - {{- $mode := include "helm_lib_module_https_mode" $context -}} - {{- if eq $mode "CertManager" -}} - {{- $secret_name_prefix -}} - {{- else -}} - {{- if eq $mode "CustomCertificate" -}} - {{- printf "%s-customcertificate" $secret_name_prefix -}} - {{- else -}} - {{- fail "https.mode must be CustomCertificate or CertManager" -}} - {{- end -}} - {{- end -}} -{{- end -}} diff --git a/charts/helm_lib/templates/_module_image.tpl b/charts/helm_lib/templates/_module_image.tpl deleted file mode 100644 index 74ab35499..000000000 --- a/charts/helm_lib/templates/_module_image.tpl +++ /dev/null @@ -1,136 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_image" (list . "" "(optional)") }} */ -}} -{{- /* returns image name */ -}} -{{- define "helm_lib_module_image" }} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} - - {{- /* New approach: use module package values */}} - {{- if and $context.Module $context.Module.Package }} - {{- $registryBase := $context.Module.Package.Registry.repository }} - {{- if not $registryBase }} - {{- fail "Registry base is not set" }} - {{- end }} - - {{- $packageName := $context.Module.Package.Name }} - {{- if not $packageName }} - {{- fail "Package name is not set" }} - {{- end }} - - {{- $imageDigest := index $context.Module.Package.Digests $containerName }} - {{- if not $imageDigest }} - {{- fail (printf "Image %s has no digest" $containerName) }} - {{- end }} - - {{- printf "%s/%s@%s" $registryBase $packageName $imageDigest }} - - {{- /* Legacy fallback: use global modulesImages values */}} - {{- else }} - {{- $rawModuleName := $context.Chart.Name }} - {{- if ge (len .) 3 }} - {{- $rawModuleName = (index . 2) }} {{- /* Optional module name */ -}} - {{- end }} - {{- $moduleName := (include "helm_lib_module_camelcase_name" $rawModuleName) }} - - {{- $imageDigest := index $context.Values.global.modulesImages.digests $moduleName $containerName }} - {{- if not $imageDigest }} - {{- $error := (printf "Image %s.%s has no digest" $moduleName $containerName ) }} - {{- fail $error }} - {{- end }} - - {{- $registryBase := $context.Values.global.modulesImages.registry.base }} - {{- /* handle external modules registry */}} - {{- if index $context.Values $moduleName }} - {{- if index $context.Values $moduleName "registry" }} - {{- if index $context.Values $moduleName "registry" "base" }} - {{- $host := trimAll "/" (index $context.Values $moduleName "registry" "base") }} - {{- $path := trimAll "/" (include "helm_lib_module_kebabcase_name" $rawModuleName) }} - {{- $registryBase = join "/" (list $host $path) }} - {{- end }} - {{- end }} - {{- end }} - {{- /* end of external module handling block */}} - {{- printf "%s@%s" $registryBase $imageDigest }} - {{- end }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_image_no_fail" (list . "") }} */ -}} -{{- /* returns image name if found */ -}} -{{- define "helm_lib_module_image_no_fail" }} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} - {{- $moduleName := (include "helm_lib_module_camelcase_name" $context) }} - {{- if ge (len .) 3 }} - {{- $moduleName = (include "helm_lib_module_camelcase_name" (index . 2)) }} {{- /* Optional module name */ -}} - {{- end }} - {{- $imageDigest := index $context.Values.global.modulesImages.digests $moduleName $containerName }} - {{- if $imageDigest }} - {{- $registryBase := $context.Values.global.modulesImages.registry.base }} - {{- if index $context.Values $moduleName }} - {{- if index $context.Values $moduleName "registry" }} - {{- if index $context.Values $moduleName "registry" "base" }} - {{- $host := trimAll "/" (index $context.Values $moduleName "registry" "base") }} - {{- $path := trimAll "/" $context.Chart.Name }} - {{- $registryBase = join "/" (list $host $path) }} - {{- end }} - {{- end }} - {{- end }} - {{- printf "%s@%s" $registryBase $imageDigest }} - {{- end }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_common_image" (list . "") }} */ -}} -{{- /* returns image name from common module */ -}} -{{- define "helm_lib_module_common_image" }} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} - {{- $imageDigest := index $context.Values.global.modulesImages.digests "common" $containerName }} - {{- if not $imageDigest }} - {{- $error := (printf "Image %s.%s has no digest" "common" $containerName ) }} - {{- fail $error }} - {{- end }} - {{- printf "%s@%s" $context.Values.global.modulesImages.registry.base $imageDigest }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_common_image_no_fail" (list . "") }} */ -}} -{{- /* returns image name from common module if found */ -}} -{{- define "helm_lib_module_common_image_no_fail" }} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} - {{- $imageDigest := index $context.Values.global.modulesImages.digests "common" $containerName }} - {{- if $imageDigest }} - {{- printf "%s@%s" $context.Values.global.modulesImages.registry.base $imageDigest }} - {{- end }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_image_digest" (list . "" "(optional)") }} */ -}} -{{- /* returns image digest */ -}} -{{- define "helm_lib_module_image_digest" }} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} - {{- $rawModuleName := $context.Chart.Name }} - {{- if ge (len .) 3 }} - {{- $rawModuleName = (index . 2) }} {{- /* Optional module name */ -}} - {{- end }} - {{- $moduleName := (include "helm_lib_module_camelcase_name" $rawModuleName) }} - {{- $moduleMap := index $context.Values.global.modulesImages.digests $moduleName | default dict }} - {{- $imageDigest := index $moduleMap $containerName | default "" }} - {{- if not $imageDigest }} - {{- $error := (printf "Image %s.%s has no digest" $moduleName $containerName ) }} - {{- fail $error }} - {{- end }} - {{- printf "%s" $imageDigest }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_image_digest_no_fail" (list . "" "(optional)") }} */ -}} -{{- /* returns image digest if found */ -}} -{{- define "helm_lib_module_image_digest_no_fail" }} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $containerName := index . 1 | trimAll "\"" }} {{- /* Container name */ -}} - {{- $moduleName := (include "helm_lib_module_camelcase_name" $context) }} - {{- if ge (len .) 3 }} - {{- $moduleName = (include "helm_lib_module_camelcase_name" (index . 2)) }} {{- /* Optional module name */ -}} - {{- end }} - {{- $moduleMap := index $context.Values.global.modulesImages.digests $moduleName | default dict }} - {{- $imageDigest := index $moduleMap $containerName | default "" }} - {{- printf "%s" $imageDigest }} -{{- end }} diff --git a/charts/helm_lib/templates/_module_ingress_class.tpl b/charts/helm_lib/templates/_module_ingress_class.tpl deleted file mode 100644 index db7f50ba7..000000000 --- a/charts/helm_lib/templates/_module_ingress_class.tpl +++ /dev/null @@ -1,13 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_ingress_class" . }} */ -}} -{{- /* returns ingress class from module settings or if not exists from global config */ -}} -{{- define "helm_lib_module_ingress_class" -}} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - - {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} - - {{- if hasKey $module_values "ingressClass" -}} - {{- $module_values.ingressClass -}} - {{- else if hasKey $context.Values.global.modules "ingressClass" -}} - {{- $context.Values.global.modules.ingressClass -}} - {{- end -}} -{{- end -}} diff --git a/charts/helm_lib/templates/_module_ingress_snippets.tpl b/charts/helm_lib/templates/_module_ingress_snippets.tpl deleted file mode 100644 index b2ae8fa1a..000000000 --- a/charts/helm_lib/templates/_module_ingress_snippets.tpl +++ /dev/null @@ -1,11 +0,0 @@ -{{- /* Usage: nginx.ingress.kubernetes.io/configuration-snippet: | {{ include "helm_lib_module_ingress_configuration_snippet" . | nindent 6 }} */ -}} -{{- /* returns nginx ingress additional headers (e.g. HSTS) if HTTPS is enabled */ -}} -{{- define "helm_lib_module_ingress_configuration_snippet" -}} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - - {{- $mode := include "helm_lib_module_https_mode" $context -}} - - {{- if or (eq "CertManager" $mode) (eq "CustomCertificate" $mode) -}} -add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; - {{- end -}} -{{- end -}} diff --git a/charts/helm_lib/templates/_module_init_container.tpl b/charts/helm_lib/templates/_module_init_container.tpl deleted file mode 100644 index 199723c9e..000000000 --- a/charts/helm_lib/templates/_module_init_container.tpl +++ /dev/null @@ -1,101 +0,0 @@ -{{- /* ### Migration 11.12.2020: Remove this helper with all its usages after this commit reached RockSolid */ -}} -{{- /* Usage: {{ include "helm_lib_module_init_container_chown_nobody_volume" (list . "volume-name") }} */ -}} -{{- /* returns initContainer which chowns recursively all files and directories in passed volume */ -}} -{{- define "helm_lib_module_init_container_chown_nobody_volume" }} - {{- $context := index . 0 -}} - {{- $volume_name := index . 1 -}} - {{- $image := "alpine" -}} - {{- if hasKey $context.Values.global.modulesImages.digests.common "init" -}} - {{- $image = "init" -}} - {{- end -}} -- name: chown-volume-{{ $volume_name }} - image: {{ include "helm_lib_module_common_image" (list $context $image) }} - command: ["sh", "-c", "chown -R 65534:65534 /tmp/data"] - securityContext: - runAsNonRoot: false - readOnlyRootFilesystem: true - runAsUser: 0 - runAsGroup: 0 - volumeMounts: - - name: {{ $volume_name }} - mountPath: /tmp/data - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 6 }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_init_container_chown_deckhouse_volume" (list . "volume-name") }} */ -}} -{{- /* returns initContainer which chowns recursively all files and directories in passed volume */ -}} -{{- define "helm_lib_module_init_container_chown_deckhouse_volume" }} - {{- $context := index . 0 -}} - {{- $volume_name := index . 1 -}} - {{- $image := "alpine" -}} - {{- if hasKey $context.Values.global.modulesImages.digests.common "init" -}} - {{- $image = "init" -}} - {{- end -}} -- name: chown-volume-{{ $volume_name }} - image: {{ include "helm_lib_module_common_image" (list $context $image) }} - command: ["sh", "-c", "chown -R 64535:64535 /tmp/data"] - securityContext: - runAsNonRoot: false - readOnlyRootFilesystem: true - runAsUser: 0 - runAsGroup: 0 - volumeMounts: - - name: {{ $volume_name }} - mountPath: /tmp/data - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 6 }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_init_container_check_linux_kernel" (list . ">= 4.9.17") }} */ -}} -{{- /* returns initContainer which checks the kernel version on the node for compliance to semver constraint */ -}} -{{- define "helm_lib_module_init_container_check_linux_kernel" }} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $semver_constraint := index . 1 -}} {{- /* Semver constraint */ -}} -- name: check-linux-kernel - image: {{ include "helm_lib_module_common_image" (list $context "checkKernelVersion") }} - {{- include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" true) | nindent 2 }} - env: - - name: KERNEL_CONSTRAINT - value: {{ $semver_constraint | quote }} - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_only_logs" $context | nindent 6 }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_init_container_iptables_wrapper" . }} */ -}} -{{- /* returns initContainer with iptables-wrapper */ -}} -{{- define "helm_lib_module_init_container_iptables_wrapper" -}} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} - - name: iptables-wrapper-init - {{- include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "NET_ADMIN" "NET_RAW")) | nindent 2 }} - runAsNonRoot: false - runAsUser: 0 - runAsGroup: 0 - seccompProfile: - type: RuntimeDefault - image: {{ include "helm_lib_module_image" (list $context "iptablesWrapperInit") }} - command: - - /bin/bash - - -ec - - | - /usr/bin/cp /iptables-wrapper /sbin/ -rv - /usr/bin/cp /_sbin/* /sbin/ -rv - /usr/bin/cp /relocate/sbin/* /sbin/ -rv - /sbin/iptables --version - /usr/bin/rm /sbin/iptables-wrapper -v - volumeMounts: - - mountPath: /sbin - name: sbin - - name: xtables-lock - mountPath: /run/xtables.lock - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_logs_with_extra" 10 | nindent 6 }} - {{- if not ( $context.Values.global.enabledModules | has "vertical-pod-autoscaler") }} - cpu: 10m - memory: 10Mi - {{- end }} -{{- end }} diff --git a/charts/helm_lib/templates/_module_labels.tpl b/charts/helm_lib/templates/_module_labels.tpl deleted file mode 100644 index 228dcf3c4..000000000 --- a/charts/helm_lib/templates/_module_labels.tpl +++ /dev/null @@ -1,15 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_labels" (list . (dict "app" "test" "component" "testing")) }} */ -}} -{{- /* returns deckhouse labels */ -}} -{{- define "helm_lib_module_labels" }} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- /* Additional labels dict */ -}} -labels: - heritage: deckhouse - module: {{ $context.Chart.Name }} - {{- if eq (len .) 2 }} - {{- $deckhouse_additional_labels := index . 1 }} - {{- range $key, $value := $deckhouse_additional_labels }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/helm_lib/templates/_module_name.tpl b/charts/helm_lib/templates/_module_name.tpl deleted file mode 100644 index 8323c177e..000000000 --- a/charts/helm_lib/templates/_module_name.tpl +++ /dev/null @@ -1,23 +0,0 @@ -{{- define "helm_lib_module_camelcase_name" -}} - -{{- $moduleName := "" -}} -{{- if (kindIs "string" .) -}} -{{- $moduleName = . | trimAll "\"" -}} -{{- else -}} -{{- $moduleName = .Chart.Name -}} -{{- end -}} - -{{ $moduleName | replace "-" "_" | camelcase | untitle }} -{{- end -}} - - -{{- define "helm_lib_module_kebabcase_name" -}} -{{- $moduleName := "" -}} -{{- if (kindIs "string" .) -}} -{{- $moduleName = . | trimAll "\"" -}} -{{- else -}} -{{- $moduleName = .Chart.Name -}} -{{- end -}} - -{{ $moduleName | kebabcase }} -{{- end -}} diff --git a/charts/helm_lib/templates/_module_public_domain.tpl b/charts/helm_lib/templates/_module_public_domain.tpl deleted file mode 100644 index bfbaae743..000000000 --- a/charts/helm_lib/templates/_module_public_domain.tpl +++ /dev/null @@ -1,11 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_public_domain" (list . "") }} */ -}} -{{- /* returns rendered publicDomainTemplate to service fqdn */ -}} -{{- define "helm_lib_module_public_domain" }} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $name_portion := index . 1 -}} {{- /* Name portion */ -}} - - {{- if not (contains "%s" $context.Values.global.modules.publicDomainTemplate) }} - {{ fail "Error!!! global.modules.publicDomainTemplate must contain \"%s\" pattern to render service fqdn!" }} - {{- end }} - {{- printf $context.Values.global.modules.publicDomainTemplate $name_portion }} -{{- end }} diff --git a/charts/helm_lib/templates/_module_security_context.tpl b/charts/helm_lib/templates/_module_security_context.tpl deleted file mode 100644 index 1f510812e..000000000 --- a/charts/helm_lib/templates/_module_security_context.tpl +++ /dev/null @@ -1,263 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_custom" (list . 1000 1000) }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with custom user and group */ -}} -{{- define "helm_lib_module_pod_security_context_run_as_user_custom" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -{{- /* User id */ -}} -{{- /* Group id */ -}} -securityContext: - runAsNonRoot: true - runAsUser: {{ index . 1 }} - runAsGroup: {{ index . 2 }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_nobody" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with user and group "nobody" */ -}} -{{- define "helm_lib_module_pod_security_context_run_as_user_nobody" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_nobody_with_writable_fs" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with user and group "nobody" with write access to mounted volumes */ -}} -{{- define "helm_lib_module_pod_security_context_run_as_user_nobody_with_writable_fs" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 - fsGroup: 65534 -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_deckhouse" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with user and group "deckhouse" */ -}} -{{- define "helm_lib_module_pod_security_context_run_as_user_deckhouse" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - runAsNonRoot: true - runAsUser: 64535 - runAsGroup: 64535 -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_deckhouse_with_writable_fs" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with user and group "deckhouse" with write access to mounted volumes */ -}} -{{- define "helm_lib_module_pod_security_context_run_as_user_deckhouse_with_writable_fs" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - runAsNonRoot: true - runAsUser: 64535 - runAsGroup: 64535 - fsGroup: 64535 -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_run_as_user_deckhouse_pss_restricted" . }} */ -}} -{{- /* returns SecurityContext parameters for Container with user and group "deckhouse" plus minimal required settings to comply with the Restricted mode of the Pod Security Standards */ -}} -{{- define "helm_lib_module_container_security_context_run_as_user_deckhouse_pss_restricted" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - all - runAsGroup: 64535 - runAsNonRoot: true - runAsUser: 64535 - seccompProfile: - type: RuntimeDefault -{{- end }} - - -{{- /* SecurityContext for Deckhouse UID/GID 64535 (or root), PSS Restricted */ -}} -{{- /* Optional keys: */ -}} -{{- /* .ro – bool, read-only root FS (default true) */ -}} -{{- /* .caps – []string, capabilities.add (default empty) */ -}} -{{- /* .uid – int, runAsUser/runAsGroup (default 64535) */ -}} -{{- /* .runAsNonRoot – bool, run as Deckhouse user when true, root when false (default true) */ -}} -{{- /* .seccompProfile – bool, disable seccompProfile when false (default true) */ -}} -{{- /* Usage: include "helm_lib_module_container_security_context_pss_restricted_flexible" (dict "ro" false "caps" (list "NET_ADMIN" "SYS_TIME") "uid" 1001 "seccompProfile" false "runAsNonRoot" true) */ -}} -{{- define "helm_lib_module_container_security_context_pss_restricted_flexible" -}} -{{- $ro := true -}} -{{- if hasKey . "ro" -}} - {{- $ro = .ro -}} -{{- end -}} -{{- $seccompProfile := true -}} -{{- if hasKey . "seccompProfile" -}} - {{- $seccompProfile = .seccompProfile -}} -{{- end -}} -{{- $caps := default (list) .caps -}} -{{- $uid := default 64535 .uid -}} -{{- $runAsNonRoot := true -}} -{{- if hasKey . "runAsNonRoot" -}} - {{- $runAsNonRoot = .runAsNonRoot -}} -{{- end -}} - -securityContext: - readOnlyRootFilesystem: {{ $ro }} - allowPrivilegeEscalation: {{ not $runAsNonRoot }} -{{- if $runAsNonRoot }} - privileged: false -{{- end }} - capabilities: - drop: - - ALL -{{- if $caps }} - add: {{ $caps | toJson }} -{{- end }} - runAsUser: {{ ternary $uid 0 $runAsNonRoot }} - runAsGroup: {{ ternary $uid 0 $runAsNonRoot }} - runAsNonRoot: {{ $runAsNonRoot }} -{{- if $seccompProfile }} - seccompProfile: - type: RuntimeDefault -{{- end }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_module_pod_security_context_run_as_user_root" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with user and group 0 */ -}} -{{- define "helm_lib_module_pod_security_context_run_as_user_root" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - runAsNonRoot: false - runAsUser: 0 - runAsGroup: 0 -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_pod_security_context_runtime_default" . }} */ -}} -{{- /* returns PodSecurityContext parameters for Pod with seccomp profile RuntimeDefault */ -}} -{{- define "helm_lib_module_pod_security_context_runtime_default" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - seccompProfile: - type: RuntimeDefault -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_not_allow_privilege_escalation" . }} */ -}} -{{- /* returns SecurityContext parameters for Container with allowPrivilegeEscalation false */ -}} -{{- define "helm_lib_module_container_security_context_not_allow_privilege_escalation" -}} -securityContext: - allowPrivilegeEscalation: false -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_read_only_root_filesystem_with_selinux" . }} */ -}} -{{- /* returns SecurityContext parameters for Container with read only root filesystem and options for SELinux compatibility*/ -}} -{{- define "helm_lib_module_container_security_context_read_only_root_filesystem_with_selinux" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - seLinuxOptions: - level: 's0' - type: 'spc_t' -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_read_only_root_filesystem" . }} */ -}} -{{- /* returns SecurityContext parameters for Container with read only root filesystem */ -}} -{{- define "helm_lib_module_container_security_context_read_only_root_filesystem" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_privileged" . }} */ -}} -{{- /* returns SecurityContext parameters for Container running privileged */ -}} -{{- define "helm_lib_module_container_security_context_privileged" -}} -securityContext: - privileged: true -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_escalated_sys_admin_privileged" . }} */ -}} -{{- /* returns SecurityContext parameters for Container running privileged with escalation and sys_admin */ -}} -{{- define "helm_lib_module_container_security_context_escalated_sys_admin_privileged" -}} -securityContext: - allowPrivilegeEscalation: true - readOnlyRootFilesystem: true - capabilities: - add: - - SYS_ADMIN - privileged: true -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" . }} */ -}} -{{- /* returns SecurityContext parameters for Container running privileged with read only root filesystem */ -}} -{{- define "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - privileged: true - readOnlyRootFilesystem: true -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all" . }} */ -}} -{{- /* returns SecurityContext for Container with read only root filesystem and all capabilities dropped */ -}} -{{- define "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} */ -}} -{{- /* returns SecurityContext parameters for Container with read only root filesystem, all dropped and some added capabilities */ -}} -{{- define "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -{{- /* List of capabilities */ -}} -securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - add: {{ index . 1 | toJson }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_capabilities_drop_all_and_add" (list . (list "KILL" "SYS_PTRACE")) }} */ -}} -{{- /* returns SecurityContext parameters for Container with all dropped and some added capabilities */ -}} -{{- define "helm_lib_module_container_security_context_capabilities_drop_all_and_add" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -{{- /* List of capabilities */ -}} -securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - add: {{ index . 1 | toJson }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_capabilities_drop_all_and_run_as_user_custom" (list . 1000 1000) }} */ -}} -{{- /* returns SecurityContext parameters for Container with read only root filesystem, all dropped, and custom user ID */ -}} -{{- define "helm_lib_module_container_security_context_capabilities_drop_all_and_run_as_user_custom" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -{{- /* User id */ -}} -{{- /* Group id */ -}} -securityContext: - runAsUser: {{ index . 1 }} - runAsGroup: {{ index . 2 }} - runAsNonRoot: true - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault -{{- end }} - -{{- /* Usage: {{ include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . }} */ -}} -{{- /* returns SecurityContext parameters for Container with minimal required settings to comply with the Restricted mode of the Pod Security Standards */ -}} -{{- define "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" -}} -{{- /* Template context with .Values, .Chart, etc */ -}} -securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault -{{- end }} diff --git a/charts/helm_lib/templates/_module_storage_class.tpl b/charts/helm_lib/templates/_module_storage_class.tpl deleted file mode 100644 index e9c071766..000000000 --- a/charts/helm_lib/templates/_module_storage_class.tpl +++ /dev/null @@ -1,36 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_module_storage_class_annotations" (list $ $index $storageClass.name) }} */ -}} -{{- /* return module StorageClass annotations */ -}} -{{- define "helm_lib_module_storage_class_annotations" -}} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $sc_index := index . 1 -}} {{- /* Storage class index */ -}} - {{- $sc_name := index . 2 -}} {{- /* Storage class name */ -}} - {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) -}} - {{- $annotations := dict -}} - - {{- $volume_expansion_mode_offline := false -}} - {{- range $module_name := list "cloud-provider-azure" "cloud-provider-vsphere" "cloud-provider-vcd"}} - {{- if has $module_name $context.Values.global.enabledModules }} - {{- $volume_expansion_mode_offline = true }} - {{- end }} - {{- end }} - - {{- if $volume_expansion_mode_offline }} - {{- $_ := set $annotations "storageclass.deckhouse.io/volume-expansion-mode" "offline" }} - {{- end }} - - {{- if $context.Values.global.discovery.defaultStorageClass }} - {{- if eq $context.Values.global.discovery.defaultStorageClass $sc_name }} - {{- $_ := set $annotations "storageclass.kubernetes.io/is-default-class" "true" }} - {{- end }} - {{- else }} - {{- /* Annotate first StorageClass in list as default in case there is `global.discovery.defaultStorageClass` and */ -}} - {{- /* `global.defaultClusterStorageClass` NOT defined/empty */ -}} - {{- if (eq $sc_index 0) }} - {{- if or (not (hasKey $context.Values.global "defaultClusterStorageClass")) (and (hasKey $context.Values.global "defaultClusterStorageClass") (not $context.Values.global.defaultClusterStorageClass)) }} - {{- $_ := set $annotations "storageclass.kubernetes.io/is-default-class" "true" }} - {{- end }} - {{- end }} - {{- end }} - -{{- (dict "annotations" $annotations) | toYaml -}} -{{- end -}} diff --git a/charts/helm_lib/templates/_monitoring_grafana_dashboards.tpl b/charts/helm_lib/templates/_monitoring_grafana_dashboards.tpl deleted file mode 100644 index def3d7ad7..000000000 --- a/charts/helm_lib/templates/_monitoring_grafana_dashboards.tpl +++ /dev/null @@ -1,101 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_grafana_dashboard_definitions_recursion" (list . [current dir]) }} */ -}} -{{- /* returns all the dashboard-definintions from / */ -}} -{{- /* current dir is optional — used for recursion but you can use it for partially generating dashboards */ -}} -{{- define "helm_lib_grafana_dashboard_definitions_recursion" -}} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $rootDir := index . 1 }} {{- /* Dashboards root dir */ -}} - {{- /* Dashboards current dir */ -}} - - {{- $currentDir := "" }} - {{- if gt (len .) 2 }} {{- $currentDir = index . 2 }} {{- else }} {{- $currentDir = $rootDir }} {{- end }} - - {{- $currentDirIndex := (sub ($currentDir | splitList "/" | len) 1) }} - {{- $rootDirIndex := (sub ($rootDir | splitList "/" | len) 1) }} - {{- $folderNamesIndex := (add1 $rootDirIndex) }} - - {{- range $path, $_ := $context.Files.Glob (print $currentDir "/*.json") }} - {{- $fileName := ($path | splitList "/" | last ) }} - {{- $definition := ($context.Files.Get $path) }} - - {{- $folder := (index ($currentDir | splitList "/") $folderNamesIndex | replace "-" " " | title) }} - {{- $resourceName := (regexReplaceAllLiteral "\\.json$" $path "") }} - {{- $resourceName = ($resourceName | replace " " "-" | replace "." "-" | replace "_" "-") }} - {{- $resourceName = (slice ($resourceName | splitList "/") $folderNamesIndex | join "-") }} - {{- $resourceName = (printf "%s-%s" $context.Chart.Name $resourceName) }} - -{{ include "helm_lib_single_dashboard" (list $context $resourceName $folder $definition) }} - {{- end }} - - {{- range $path, $_ := $context.Files.Glob (print $currentDir "/*.tpl") }} - {{- $fileName := ($path | splitList "/" | last ) }} - {{- $definition := tpl ($context.Files.Get $path) $context }} - - {{- $folder := (index ($currentDir | splitList "/") $folderNamesIndex | replace "-" " " | title) }} - {{- $resourceName := (regexReplaceAllLiteral "\\.tpl$" $path "") }} - {{- $resourceName = ($resourceName | replace " " "-" | replace "." "-" | replace "_" "-") }} - {{- $resourceName = (slice ($resourceName | splitList "/") $folderNamesIndex | join "-") }} - {{- $resourceName = (printf "%s-%s" $context.Chart.Name $resourceName) }} - -{{ include "helm_lib_single_dashboard" (list $context $resourceName $folder $definition) }} - {{- end }} - - {{- $subDirs := list }} - {{- range $path, $_ := ($context.Files.Glob (print $currentDir "/**.json")) }} - {{- $pathSlice := ($path | splitList "/") }} - {{- $subDirs = append $subDirs (slice $pathSlice 0 (add $currentDirIndex 2) | join "/") }} - {{- end }} - {{- range $path, $_ := ($context.Files.Glob (print $currentDir "/**.tpl")) }} - {{- $pathSlice := ($path | splitList "/") }} - {{- $subDirs = append $subDirs (slice $pathSlice 0 (add $currentDirIndex 2) | join "/") }} - {{- end }} - - {{- range $subDir := ($subDirs | uniq) }} -{{ include "helm_lib_grafana_dashboard_definitions_recursion" (list $context $rootDir $subDir) }} - {{- end }} - -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_grafana_dashboard_definitions" . }} */ -}} -{{- /* returns dashboard-definintions from monitoring/grafana-dashboards/ */ -}} -{{- define "helm_lib_grafana_dashboard_definitions" -}} - {{- $context := . }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- if ( $context.Values.global.enabledModules | has "prometheus-crd" ) }} -{{- include "helm_lib_grafana_dashboard_definitions_recursion" (list $context "monitoring/grafana-dashboards") }} - {{- end }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_single_dashboard" (list . "dashboard-name" "folder" $dashboard) }} */ -}} -{{- /* renders a single dashboard */ -}} -{{- define "helm_lib_single_dashboard" -}} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $resourceName := index . 1 }} {{- /* Dashboard name */ -}} - {{- $folder := index . 2 }} {{- /* Folder */ -}} - {{- $definition := index . 3 }} {{/* Dashboard definition */}} - {{- $propagated := contains "-propagated-" $resourceName }} - {{- $resourceName = $resourceName | replace "-propagated-" "-" }} ---- -apiVersion: deckhouse.io/v1 -kind: GrafanaDashboardDefinition -metadata: - name: d8-{{ $resourceName }} - {{- include "helm_lib_module_labels" (list $context (dict "prometheus.deckhouse.io/grafana-dashboard" "" "observability.deckhouse.io/skip-dashboard-conversion" "")) | nindent 2 }} -spec: - folder: {{ $folder | quote }} - definition: | - {{- $definition | nindent 4 }} - {{- if $context.Values.global.enabledModules | has "observability" }} ---- -apiVersion: observability.deckhouse.io/v1alpha1 -kind: {{ $propagated | ternary "ClusterObservabilityPropagatedDashboard" "ClusterObservabilityDashboard" }} -metadata: - annotations: - metadata.deckhouse.io/category: {{ $folder | quote }} - name: d8-{{ $resourceName }} - {{- include "helm_lib_module_labels" (list $context (dict "observability.deckhouse.io/dashboard-origin" "module")) | nindent 2 }} -spec: - definition: | - {{- $definition | nindent 4 }} - {{- end }} -{{- end }} diff --git a/charts/helm_lib/templates/_monitoring_prometheus_rules.tpl b/charts/helm_lib/templates/_monitoring_prometheus_rules.tpl deleted file mode 100644 index 86c75bb7b..000000000 --- a/charts/helm_lib/templates/_monitoring_prometheus_rules.tpl +++ /dev/null @@ -1,135 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_prometheus_rules_recursion" (list . [current dir] [file list]) }} */ -}} -{{- /* returns all the prometheus rules from / */ -}} -{{- /* current dir is optional — used for recursion but you can use it for partially generating rules */ -}} -{{- /* file list is optional - list of files to include (filters all files if provided) */ -}} -{{- define "helm_lib_prometheus_rules_recursion" -}} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $namespace := index . 1 }} {{- /* Namespace for creating rules */ -}} - {{- $rootDir := index . 2 }} {{- /* Rules root dir */ -}} - {{- $currentDir := "" }} {{- /* Current dir (optional) */ -}} - {{- $fileList := list }} {{- /* File list for filtering (optional) */ -}} - - {{- if gt (len .) 3 }} {{- $currentDir = index . 3 }} {{- else }} {{- $currentDir = $rootDir }} {{- end }} - {{- if gt (len .) 4 }} {{- $fileList = index . 4 }} {{- end }} - - {{- $currentDirIndex := (sub ($currentDir | splitList "/" | len) 1) }} - {{- $rootDirIndex := (sub ($rootDir | splitList "/" | len) 1) }} - {{- $folderNamesIndex := (add1 $rootDirIndex) }} - - {{- range $path, $_ := $context.Files.Glob (print $currentDir "/*.{yaml,tpl}") }} - {{- /* Filter files if fileList is provided */ -}} - {{- $shouldProcess := true }} - {{- if gt (len $fileList) 0 }} - {{- $shouldProcess = has $path $fileList }} - {{- end }} - - {{- if $shouldProcess }} - {{- $fileName := ($path | splitList "/" | last ) }} - {{- $definition := "" }} - {{- if eq ($path | splitList "." | last) "tpl" -}} - {{- $definition = tpl ($context.Files.Get $path) $context }} - {{- else }} - {{- $definition = $context.Files.Get $path }} - {{- end }} - - {{- $definition = $definition | replace "__SCRAPE_INTERVAL__" (printf "%ds" ($context.Values.global.discovery.prometheusScrapeInterval | default 30)) | replace "__SCRAPE_INTERVAL_X_2__" (printf "%ds" (mul ($context.Values.global.discovery.prometheusScrapeInterval | default 30) 2)) | replace "__SCRAPE_INTERVAL_X_3__" (printf "%ds" (mul ($context.Values.global.discovery.prometheusScrapeInterval | default 30) 3)) | replace "__SCRAPE_INTERVAL_X_4__" (printf "%ds" (mul ($context.Values.global.discovery.prometheusScrapeInterval | default 30) 4)) }} - -{{/* Patch expression based on `d8_ignore_on_update` annotation*/}} - - {{ $definition = printf "Rules:\n%s" ($definition | nindent 2) }} - {{- $definitionStruct := ( $definition | fromYaml )}} - {{- if $definitionStruct.Error }} - {{- fail ($definitionStruct.Error | toString) }} - {{- end }} - {{- range $rule := $definitionStruct.Rules }} - - {{- range $dedicatedRule := $rule.rules }} - {{- if $dedicatedRule.annotations }} - {{- if (eq (get $dedicatedRule.annotations "d8_ignore_on_update") "true") }} - {{- $_ := set $dedicatedRule "expr" (printf "(%s) and ON() ((max(d8_is_updating) != 1) or ON() absent(d8_is_updating))" $dedicatedRule.expr) }} - {{- end }} - {{- end }} - {{- end }} - - {{- end }} - - {{- $resourceName := (regexReplaceAllLiteral "\\.(yaml|tpl)$" $path "") }} - {{- $resourceName = ($resourceName | replace " " "-" | replace "." "-" | replace "_" "-") }} - {{- $resourceName = (slice ($resourceName | splitList "/") $folderNamesIndex | join "-") }} - {{- $resourceName = (printf "%s-%s" $context.Chart.Name $resourceName) }} - {{- $propagated := contains "propagated-" $resourceName }} - {{- $hasObservabilityModule := has "observability" $context.Values.global.enabledModules }} - {{- $useObservabilityRules := has "observability.deckhouse.io/v1alpha1/ClusterObservabilityMetricsRulesGroup" $context.Values.global.discovery.apiVersions }} - {{- if and $hasObservabilityModule $useObservabilityRules }} - {{- range $idx, $group := $definitionStruct.Rules }} - {{- if $group.rules }} - {{- $_ := unset $group "name" }} - {{- $resourceName = $resourceName | replace "propagated-" "" }} - {{- $groupResourceName := printf "%s-%d" $resourceName $idx }} ---- -apiVersion: observability.deckhouse.io/v1alpha1 -kind: {{ $propagated | ternary "ClusterObservabilityPropagatedMetricsRulesGroup" "ClusterObservabilityMetricsRulesGroup" }} -metadata: - name: {{ $groupResourceName }} - {{- include "helm_lib_module_labels" (list $context (dict "app" "prometheus" "prometheus" "main" "component" "rules")) | nindent 2 }} -spec: - {{- $group | toYaml | nindent 2 }} - {{- end }} - {{- end }} - {{- else }} - {{- if $definitionStruct.Rules }} - {{- $definition := $definitionStruct.Rules | toYaml }} ---- -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ $resourceName }} - namespace: {{ $namespace }} - {{- include "helm_lib_module_labels" (list $context (dict "app" "prometheus" "prometheus" "main" "component" "rules")) | nindent 2 }} -spec: - groups: - {{- $definition | nindent 4 }} - {{- end }} - {{- end }} - {{- end }} - {{- end }} - - {{- $subDirs := list }} - {{- range $path, $_ := ($context.Files.Glob (print $currentDir "/**.{yaml,tpl}")) }} - {{- $pathSlice := ($path | splitList "/") }} - {{- $subDirs = append $subDirs (slice $pathSlice 0 (add $currentDirIndex 2) | join "/") }} - {{- end }} - - {{- range $subDir := ($subDirs | uniq) }} -{{ include "helm_lib_prometheus_rules_recursion" (list $context $namespace $rootDir $subDir $fileList) }} - {{- end }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_prometheus_rules" (list . [fileList]) }} */ -}} -{{- /* returns all the prometheus rules from monitoring/prometheus-rules/ optionally filtered by fileList */ -}} -{{- define "helm_lib_prometheus_rules" -}} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $namespace := index . 1 }} {{- /* Namespace for creating rules */ -}} - {{- $rootDir := "monitoring/prometheus-rules" }} - {{- $fileList := list }} - {{- if gt (len .) 2 }} - {{- $fileList = index . 2 }} - {{- end }} - {{- if ( $context.Values.global.enabledModules | has "operator-prometheus-crd" ) }} -{{- include "helm_lib_prometheus_rules_recursion" (list $context $namespace $rootDir $rootDir $fileList) }} - {{- end }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_prometheus_target_scrape_timeout_seconds" (list . ) }} */ -}} -{{- /* returns adjust timeout value to scrape interval / */ -}} -{{- define "helm_lib_prometheus_target_scrape_timeout_seconds" -}} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $timeout := index . 1 }} {{- /* Target timeout in seconds */ -}} - {{- $scrape_interval := (int $context.Values.global.discovery.prometheusScrapeInterval | default 30) }} - {{- if gt $timeout $scrape_interval -}} -{{ $scrape_interval }}s - {{- else -}} -{{ $timeout }}s - {{- end }} -{{- end }} diff --git a/charts/helm_lib/templates/_node_affinity.tpl b/charts/helm_lib/templates/_node_affinity.tpl deleted file mode 100644 index ab06d959c..000000000 --- a/charts/helm_lib/templates/_node_affinity.tpl +++ /dev/null @@ -1,262 +0,0 @@ -{{- /* Verify node selector strategy. */ -}} -{{- define "helm_lib_internal_check_node_selector_strategy" -}} - {{ if not (has . (list "frontend" "monitoring" "system" "master" )) }} - {{- fail (printf "unknown strategy \"%v\"" .) }} - {{- end }} - {{- . -}} -{{- end }} - -{{- /* Returns node selector for workloads depend on strategy. */ -}} -{{- define "helm_lib_node_selector" }} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $strategy := index . 1 | include "helm_lib_internal_check_node_selector_strategy" }} {{- /* strategy, one of "frontend" "monitoring" "system" "master" "any-node" "wildcard" */ -}} - {{- $module_values := dict }} - {{- if lt (len .) 3 }} - {{- $module_values = (index $context.Values (include "helm_lib_module_camelcase_name" $context)) }} - {{- else }} - {{- $module_values = index . 2 }} - {{- end }} - {{- $camel_chart_name := (include "helm_lib_module_camelcase_name" $context) }} - - {{- if eq $strategy "monitoring" }} - {{- if $module_values.nodeSelector }} -nodeSelector: {{ $module_values.nodeSelector | toJson }} - {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole $camel_chart_name | int) 0 }} -nodeSelector: - node-role.deckhouse.io/{{$context.Chart.Name}}: "" - {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole $strategy | int) 0 }} -nodeSelector: - node-role.deckhouse.io/{{$strategy}}: "" - {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "system" | int) 0 }} -nodeSelector: - node-role.deckhouse.io/system: "" - {{- end }} - - {{- else if or (eq $strategy "frontend") (eq $strategy "system") }} - {{- if $module_values.nodeSelector }} -nodeSelector: {{ $module_values.nodeSelector | toJson }} - {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole $camel_chart_name | int) 0 }} -nodeSelector: - node-role.deckhouse.io/{{$context.Chart.Name}}: "" - {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole $strategy | int) 0 }} -nodeSelector: - node-role.deckhouse.io/{{$strategy}}: "" - {{- end }} - - {{- else if eq $strategy "master" }} - {{- if gt (index $context.Values.global.discovery "clusterMasterCount" | int) 0 }} -nodeSelector: - node-role.kubernetes.io/control-plane: "" - {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 0 }} -nodeSelector: - node-role.deckhouse.io/control-plane: "" - {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "system" | int) 0 }} -nodeSelector: - node-role.deckhouse.io/system: "" - {{- end }} - {{- end }} -{{- end }} - - -{{- /* Returns tolerations for workloads depend on strategy. */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "with-uninitialized" "without-storage-problems") }} */ -}} -{{- define "helm_lib_tolerations" }} - {{- $context := index . 0 }} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $strategy := index . 1 | include "helm_lib_internal_check_tolerations_strategy" }} {{- /* base strategy, one of "frontend" "monitoring" "system" any-node" "wildcard" */ -}} - {{- $additionalStrategies := tuple }} {{- /* list of additional strategies. To add strategy list it with prefix "with-", to remove strategy list it with prefix "without-". */ -}} - {{- if eq $strategy "custom" }} - {{ if lt (len .) 3 }} - {{- fail (print "additional strategies is required") }} - {{- end }} - {{- else }} - {{- $additionalStrategies = tuple "storage-problems" }} - {{- end }} - {{- $module_values := (index $context.Values (include "helm_lib_module_camelcase_name" $context)) }} - {{- if gt (len .) 2 }} - {{- range $as := slice . 2 (len .) }} - {{- if hasPrefix "with-" $as }} - {{- $additionalStrategies = mustAppend $additionalStrategies (trimPrefix "with-" $as) }} - {{- end }} - {{- if hasPrefix "without-" $as }} - {{- $additionalStrategies = mustWithout $additionalStrategies (trimPrefix "without-" $as) }} - {{- end }} - {{- end }} - {{- end }} -tolerations: - {{- /* Wildcard: gives permissions to schedule on any node with any taints (use with caution) */ -}} - {{- if eq $strategy "wildcard" }} - {{- include "_helm_lib_wildcard_tolerations" $context }} - - {{- else }} - {{- /* Any node: any node in the cluster with any known taints */ -}} - {{- if eq $strategy "any-node" }} - {{- include "_helm_lib_any_node_tolerations" $context }} - - {{- /* Tolerations from module config: overrides below strategies, if there is any toleration specified */ -}} - {{- else if $module_values.tolerations }} - {{- $module_values.tolerations | toYaml | nindent 0 }} - - {{- /* Monitoring: Nodes for monitoring components: prometheus, grafana, kube-state-metrics, etc. */ -}} - {{- else if eq $strategy "monitoring" }} - {{- include "_helm_lib_monitoring_tolerations" $context }} - - {{- /* Frontend: Nodes for ingress-controllers */ -}} - {{- else if eq $strategy "frontend" }} - {{- include "_helm_lib_frontend_tolerations" $context }} - - {{- /* System: Nodes for system components: prometheus, dns, cert-manager */ -}} - {{- else if eq $strategy "system" }} - {{- include "_helm_lib_system_tolerations" $context }} - {{- end }} - - {{- /* Additional strategies */ -}} - {{- range $additionalStrategies -}} - {{- include (printf "_helm_lib_additional_tolerations_%s" (. | replace "-" "_")) $context }} - {{- end }} - {{- end }} -{{- end }} - - -{{- /* Check cluster type. */ -}} -{{- /* Returns not empty string if this is cloud or hybrid cluster */ -}} -{{- define "_helm_lib_cloud_or_hybrid_cluster" }} - {{- if .Values.global.clusterConfiguration }} - {{- if eq .Values.global.clusterConfiguration.clusterType "Cloud" }} - "not empty string" - {{- /* We consider non-cloud clusters with enabled cloud-provider-.* module as Hybrid clusters */ -}} - {{- else }} - {{- range $v := .Values.global.enabledModules }} - {{- if hasPrefix "cloud-provider-" $v }} - "not empty string" - {{- end }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} - -{{- /* Verify base strategy. */ -}} -{{- /* Fails if strategy not in allowed list */ -}} -{{- define "helm_lib_internal_check_tolerations_strategy" -}} - {{ if not (has . (list "custom" "frontend" "monitoring" "system" "any-node" "wildcard" )) }} - {{- fail (printf "unknown strategy \"%v\"" .) }} - {{- end }} - {{- . -}} -{{- end }} - - -{{- /* Base strategy for any uncordoned node in cluster. */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node") }} */ -}} -{{- define "_helm_lib_any_node_tolerations" }} -- key: node-role.kubernetes.io/master -- key: node-role.kubernetes.io/control-plane -- key: node.deckhouse.io/etcd-arbiter -- key: dedicated.deckhouse.io - operator: "Exists" -- key: dedicated - operator: "Exists" -- key: DeletionCandidateOfClusterAutoscaler -- key: ToBeDeletedByClusterAutoscaler - {{- if .Values.global.modules.placement.customTolerationKeys }} - {{- range $key := .Values.global.modules.placement.customTolerationKeys }} -- key: {{ $key | quote }} - operator: "Exists" - {{- end }} - {{- end }} -{{- end }} - -{{- /* Base strategy that tolerates all. */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "wildcard") }} */ -}} -{{- define "_helm_lib_wildcard_tolerations" }} -- operator: "Exists" -{{- end }} - -{{- /* Base strategy that tolerates nodes with "dedicated.deckhouse.io: monitoring" and "dedicated.deckhouse.io: system" taints. */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "monitoring") }} */ -}} -{{- define "_helm_lib_monitoring_tolerations" }} -- key: dedicated.deckhouse.io - operator: Equal - value: {{ .Chart.Name | quote }} -- key: dedicated.deckhouse.io - operator: Equal - value: "monitoring" -- key: dedicated.deckhouse.io - operator: Equal - value: "system" -{{- end }} - -{{- /* Base strategy that tolerates nodes with "dedicated.deckhouse.io: frontend" taints. */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "frontend") }} */ -}} -{{- define "_helm_lib_frontend_tolerations" }} -- key: dedicated.deckhouse.io - operator: Equal - value: {{ .Chart.Name | quote }} -- key: dedicated.deckhouse.io - operator: Equal - value: "frontend" -{{- end }} - -{{- /* Base strategy that tolerates nodes with "dedicated.deckhouse.io: system" taints. */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "system") }} */ -}} -{{- define "_helm_lib_system_tolerations" }} -- key: dedicated.deckhouse.io - operator: Equal - value: {{ .Chart.Name | quote }} -- key: dedicated.deckhouse.io - operator: Equal - value: "system" -{{- end }} - - -{{- /* Additional strategy "uninitialized" - used for CNI's and kube-proxy to allow cni components scheduled on node after CCM initialization. */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "with-uninitialized") }} */ -}} -{{- define "_helm_lib_additional_tolerations_uninitialized" }} -- key: node.deckhouse.io/bashible-uninitialized - operator: "Exists" - effect: "NoSchedule" -- key: node.deckhouse.io/uninitialized - operator: "Exists" - effect: "NoSchedule" - {{- if include "_helm_lib_cloud_or_hybrid_cluster" . }} - {{- include "_helm_lib_additional_tolerations_no_csi" . }} - {{- end }} - {{- include "_helm_lib_additional_tolerations_node_problems" . }} -{{- end }} - -{{- /* Additional strategy "node-problems" - used for shedule critical components on non-ready nodes or nodes under pressure. */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "with-node-problems") }} */ -}} -{{- define "_helm_lib_additional_tolerations_node_problems" }} -- key: node.kubernetes.io/not-ready -- key: node.kubernetes.io/out-of-disk -- key: node.kubernetes.io/memory-pressure -- key: node.kubernetes.io/disk-pressure -- key: node.kubernetes.io/pid-pressure -- key: node.kubernetes.io/unreachable -- key: node.kubernetes.io/network-unavailable -{{- end }} - -{{- /* Additional strategy "storage-problems" - used for shedule critical components on nodes with drbd problems. This additional strategy enabled by default in any base strategy except "wildcard". */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "without-storage-problems") }} */ -}} -{{- define "_helm_lib_additional_tolerations_storage_problems" }} -- key: drbd.linbit.com/lost-quorum -- key: drbd.linbit.com/force-io-error -- key: drbd.linbit.com/ignore-fail-over -{{- end }} - -{{- /* Additional strategy "no-csi" - used for any node with no CSI: any node, which was initialized by deckhouse, but have no csi-node driver registered on it. */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "with-no-csi") }} */ -}} -{{- define "_helm_lib_additional_tolerations_no_csi" }} -- key: ToBeDeletedTaint - operator: "Exists" -- key: node.deckhouse.io/csi-not-bootstrapped - operator: "Exists" - effect: "NoSchedule" -{{- end }} - -{{- /* Additional strategy "cloud-provider-uninitialized" - used for any node which is not initialized by CCM. */ -}} -{{- /* Usage: {{ include "helm_lib_tolerations" (tuple . "any-node" "with-cloud-provider-uninitialized") }} */ -}} -{{- define "_helm_lib_additional_tolerations_cloud_provider_uninitialized" }} - {{- if not .Values.global.clusterIsBootstrapped }} -- key: node.cloudprovider.kubernetes.io/uninitialized - operator: Exists - {{- end }} -{{- end }} diff --git a/charts/helm_lib/templates/_pod_disruption_budget.tpl b/charts/helm_lib/templates/_pod_disruption_budget.tpl deleted file mode 100644 index ccd4f211d..000000000 --- a/charts/helm_lib/templates/_pod_disruption_budget.tpl +++ /dev/null @@ -1,6 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_pdb_daemonset" . }} */ -}} -{{- /* Returns PDB max unavailable */ -}} -{{- define "helm_lib_pdb_daemonset" }} - {{- $context := . -}} {{- /* Template context with .Values, .Chart, etc */ -}} -maxUnavailable: 10% -{{- end -}} diff --git a/charts/helm_lib/templates/_priority_class.tpl b/charts/helm_lib/templates/_priority_class.tpl deleted file mode 100644 index 7da49b94e..000000000 --- a/charts/helm_lib/templates/_priority_class.tpl +++ /dev/null @@ -1,7 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_priority_class" (tuple . "priority-class-name") }} /* -}} -{{- /* returns priority class */ -}} -{{- define "helm_lib_priority_class" }} - {{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} - {{- $priorityClassName := index . 1 }} {{- /* Priority class name */ -}} -priorityClassName: {{ $priorityClassName }} -{{- end -}} diff --git a/charts/helm_lib/templates/_resources_management.tpl b/charts/helm_lib/templates/_resources_management.tpl deleted file mode 100644 index 748f7dbb0..000000000 --- a/charts/helm_lib/templates/_resources_management.tpl +++ /dev/null @@ -1,160 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_resources_management_pod_resources" (list [ephemeral storage requests]) }} */ -}} -{{- /* returns rendered resources section based on configuration if it is */ -}} -{{- define "helm_lib_resources_management_pod_resources" -}} - {{- $configuration := index . 0 -}} {{- /* VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) */ -}} - {{- /* Ephemeral storage requests */ -}} - - {{- $ephemeral_storage := "50Mi" -}} - {{- if eq (len .) 2 -}} - {{- $ephemeral_storage = index . 1 -}} - {{- end -}} - - {{- $pod_resources := (include "helm_lib_resources_management_original_pod_resources" $configuration | fromYaml) -}} - {{- if not (hasKey $pod_resources "requests") -}} - {{- $_ := set $pod_resources "requests" (dict) -}} - {{- end -}} - {{- $_ := set $pod_resources.requests "ephemeral-storage" $ephemeral_storage -}} - - {{- $pod_resources | toYaml -}} -{{- end -}} - - -{{- /* Usage: {{ include "helm_lib_resources_management_original_pod_resources" }} */ -}} -{{- /* returns rendered resources section based on configuration if it is present */ -}} -{{- define "helm_lib_resources_management_original_pod_resources" -}} - {{- $configuration := . -}} {{- /* VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) */ -}} - - {{- if $configuration -}} - {{- if eq $configuration.mode "Static" -}} -{{- $configuration.static | toYaml -}} - - {{- else if eq $configuration.mode "VPA" -}} - {{- $resources := dict "requests" (dict) "limits" (dict) -}} - - {{- if $configuration.vpa.cpu -}} - {{- if $configuration.vpa.cpu.min -}} - {{- $_ := set $resources.requests "cpu" ($configuration.vpa.cpu.min | toString) -}} - {{- end -}} - {{- if $configuration.vpa.cpu.limitRatio -}} - {{- $cpuLimitMillicores := round (mulf (include "helm_lib_resources_management_cpu_units_to_millicores" $configuration.vpa.cpu.min) $configuration.vpa.cpu.limitRatio) 0 | int64 -}} - {{- $_ := set $resources.limits "cpu" (printf "%dm" $cpuLimitMillicores) -}} - {{- end -}} - {{- end -}} - - {{- if $configuration.vpa.memory -}} - {{- if $configuration.vpa.memory.min -}} - {{- $_ := set $resources.requests "memory" ($configuration.vpa.memory.min | toString) -}} - {{- end -}} - {{- if $configuration.vpa.memory.limitRatio -}} - {{- $memoryLimitBytes := round (mulf (include "helm_lib_resources_management_memory_units_to_bytes" $configuration.vpa.memory.min) $configuration.vpa.memory.limitRatio) 0 | int64 -}} - {{- $_ := set $resources.limits "memory" (printf "%d" $memoryLimitBytes) -}} - {{- end -}} - {{- end -}} -{{- $resources | toYaml -}} - - {{- else -}} - {{- cat "ERROR: unknown resource management mode: " $configuration.mode | fail -}} - {{- end -}} - {{- end -}} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_resources_management_vpa_spec" (list ) }} */ -}} -{{- /* returns rendered vpa spec based on configuration and target reference */ -}} -{{- define "helm_lib_resources_management_vpa_spec" -}} - {{- $targetAPIVersion := index . 0 -}} {{- /* Target API version */ -}} - {{- $targetKind := index . 1 -}} {{- /* Target Kind */ -}} - {{- $targetName := index . 2 -}} {{- /* Target Name */ -}} - {{- $targetContainer := index . 3 -}} {{- /* Target container name */ -}} - {{- $configuration := index . 4 -}} {{- /* VPA resource configuration [example](https://deckhouse.io/modules/istio/configuration.html#parameters-controlplane-resourcesmanagement) */ -}} - -targetRef: - apiVersion: {{ $targetAPIVersion }} - kind: {{ $targetKind }} - name: {{ $targetName }} - {{- if eq ($configuration.mode) "VPA" }} -updatePolicy: - updateMode: {{ $configuration.vpa.mode | quote }} -resourcePolicy: - containerPolicies: - - containerName: {{ $targetContainer }} - maxAllowed: - cpu: {{ $configuration.vpa.cpu.max | quote }} - memory: {{ $configuration.vpa.memory.max | quote }} - minAllowed: - cpu: {{ $configuration.vpa.cpu.min | quote }} - memory: {{ $configuration.vpa.memory.min | quote }} - controlledValues: RequestsAndLimits - {{- else }} -updatePolicy: - updateMode: "Off" - {{- end }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_resources_management_cpu_units_to_millicores" }} */ -}} -{{- /* helper for converting cpu units to millicores */ -}} -{{- define "helm_lib_resources_management_cpu_units_to_millicores" -}} - {{- $units := . | toString -}} - {{- if hasSuffix "m" $units -}} - {{- trimSuffix "m" $units -}} - {{- else -}} - {{- atoi $units | mul 1000 -}} - {{- end }} -{{- end }} - - -{{- /* Usage: {{ include "helm_lib_resources_management_memory_units_to_bytes" }} */ -}} -{{- /* helper for converting memory units to bytes */ -}} -{{- define "helm_lib_resources_management_memory_units_to_bytes" }} - {{- $units := . | toString -}} - {{- if hasSuffix "k" $units -}} - {{- trimSuffix "k" $units | atoi | mul 1000 -}} - {{- else if hasSuffix "M" $units -}} - {{- trimSuffix "M" $units | atoi | mul 1000000 -}} - {{- else if hasSuffix "G" $units -}} - {{- trimSuffix "G" $units | atoi | mul 1000000000 -}} - {{- else if hasSuffix "T" $units -}} - {{- trimSuffix "T" $units | atoi | mul 1000000000000 -}} - {{- else if hasSuffix "P" $units -}} - {{- trimSuffix "P" $units | atoi | mul 1000000000000000 -}} - {{- else if hasSuffix "E" $units -}} - {{- trimSuffix "E" $units | atoi | mul 1000000000000000000 -}} - {{- else if hasSuffix "Ki" $units -}} - {{- trimSuffix "Ki" $units | atoi | mul 1024 -}} - {{- else if hasSuffix "Mi" $units -}} - {{- trimSuffix "Mi" $units | atoi | mul 1024 | mul 1024 -}} - {{- else if hasSuffix "Gi" $units -}} - {{- trimSuffix "Gi" $units | atoi | mul 1024 | mul 1024 | mul 1024 -}} - {{- else if hasSuffix "Ti" $units -}} - {{- trimSuffix "Ti" $units | atoi | mul 1024 | mul 1024 | mul 1024 | mul 1024 -}} - {{- else if hasSuffix "Pi" $units -}} - {{- trimSuffix "Pi" $units | atoi | mul 1024 | mul 1024 | mul 1024 | mul 1024 | mul 1024 -}} - {{- else if hasSuffix "Ei" $units -}} - {{- trimSuffix "Ei" $units | atoi | mul 1024 | mul 1024 | mul 1024 | mul 1024 | mul 1024 | mul 1024 -}} - {{- else if regexMatch "^[0-9]+$" $units -}} - {{- $units -}} - {{- else -}} - {{- cat "ERROR: unknown memory format:" $units | fail -}} - {{- end }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_vpa_kube_rbac_proxy_resources" . }} */ -}} -{{- /* helper for VPA resources for kube_rbac_proxy */ -}} -{{- define "helm_lib_vpa_kube_rbac_proxy_resources" }} -{{- /* Template context with .Values, .Chart, etc */ -}} -- containerName: kube-rbac-proxy - minAllowed: - {{- include "helm_lib_container_kube_rbac_proxy_resources" . | nindent 4 }} - maxAllowed: - cpu: 20m - memory: 25Mi -{{- end }} - -{{- /* Usage: {{ include "helm_lib_container_kube_rbac_proxy_resources" . }} */ -}} -{{- /* helper for container resources for kube_rbac_proxy */ -}} -{{- define "helm_lib_container_kube_rbac_proxy_resources" }} -{{- /* Template context with .Values, .Chart, etc */ -}} -cpu: 10m -memory: 25Mi -{{- end }} diff --git a/charts/helm_lib/templates/_spec_for_high_availability.tpl b/charts/helm_lib/templates/_spec_for_high_availability.tpl deleted file mode 100644 index 2cd393a83..000000000 --- a/charts/helm_lib/templates/_spec_for_high_availability.tpl +++ /dev/null @@ -1,179 +0,0 @@ -{{- /* Usage: {{ include "helm_lib_pod_anti_affinity_for_ha" (list . (dict "app" "test")) }} */ -}} -{{- /* returns pod affinity spec */ -}} -{{- define "helm_lib_pod_anti_affinity_for_ha" }} -{{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} -{{- $labels := index . 1 }} {{- /* Match labels for podAntiAffinity label selector */ -}} - {{- if (include "helm_lib_ha_enabled" $context) }} -affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - {{- range $key, $value := $labels }} - {{ $key }}: {{ $value | quote }} - {{- end }} - topologyKey: kubernetes.io/hostname - {{- end }} -{{- end }} - -{{- /* Usage: {{- include "helm_lib_pod_affinity" (list . (dict "app" "test") (list "amd64")) }} */}} -{{- /* Returns affinity spec that combines: podAntiAffinity by provided labels when HA is enabled and optional nodeAffinity that schedules pods only on specified architectures. If the list of architectures is not provided or empty, node affinity is not rendered. */ -}} -{{- define "helm_lib_pod_affinity" }} -{{- $context := index . 0 -}} {{- /* Template context with .Values, .Chart, etc */ -}} -{{- $labels := dict -}} {{- /* Match labels for podAntiAffinity label selector */ -}} -{{- if ge (len .) 2 }} - {{- $labels = index . 1 }} -{{- end }} -{{- $allowedArchs := list -}} {{- /* List of supported architectures */ -}} -{{- if ge (len .) 3 }} - {{- $allowedArchs = index . 2 }} -{{- end }} -{{- $haEnabled := (include "helm_lib_ha_enabled" $context) -}} -{{- $hasArch := gt (len $allowedArchs) 0 -}} -{{- if or $haEnabled $hasArch }} -affinity: - {{- if $haEnabled }} - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - {{- range $key, $value := $labels }} - {{ $key }}: {{ $value | quote }} - {{- end }} - topologyKey: kubernetes.io/hostname - {{- end }} - {{- if $hasArch }} - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - {{- range $allowedArchs }} - - {{ . | quote }} - {{- end }} - {{- end }} -{{- end }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" }} */ -}} -{{- /* returns deployment strategy and replicas for ha components running on master nodes */ -}} -{{- define "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" }} -{{- /* Template context with .Values, .Chart, etc */ -}} - {{- if (include "helm_lib_ha_enabled" .) }} - {{- if gt (index .Values.global.discovery "clusterMasterCount" | int) 0 }} -replicas: {{ index .Values.global.discovery "clusterMasterCount" }} -strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - {{- if gt (index .Values.global.discovery "clusterMasterCount" | int) 2 }} - maxUnavailable: 2 - {{- else }} - maxUnavailable: 1 - {{- end }} - {{- else if gt (index .Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 0 }} -replicas: {{ index .Values.global.discovery.d8SpecificNodeCountByRole "master" }} -strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - {{- if gt (index .Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 2 }} - maxUnavailable: 2 - {{- else }} - maxUnavailable: 1 - {{- end }} - {{- else }} -replicas: 2 -strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - {{- end }} - {{- else }} -replicas: 1 -strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - {{- end }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha" (list . (dict "strategy" "strategy_type")) }} */ -}} -{{- /* returns deployment with custom strategy and replicas for ha components running on master nodes */ -}} -{{- define "helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha" }} -{{- $context := index . 0 }} -{{- $optionalArgs := dict }} -{{- $strategy := "RollingUpdate" }} -{{- if ge (len .) 2 }} - {{- $optionalArgs = index . 1 }} -{{- end }} -{{- if hasKey $optionalArgs "strategy" }} - {{- $strategy = $optionalArgs.strategy }} -{{- end }} -{{- /* Template context with .Values, .Chart, etc */ -}} - {{- if (include "helm_lib_ha_enabled" $context) }} - {{- if gt (index $context.Values.global.discovery "clusterMasterCount" | int) 0 }} -replicas: {{ index $context.Values.global.discovery "clusterMasterCount" }} -strategy: - type: {{ $strategy }} - {{- if eq $strategy "RollingUpdate" }} - rollingUpdate: - maxSurge: 0 - {{- if gt (index $context.Values.global.discovery "clusterMasterCount" | int) 2 }} - maxUnavailable: 2 - {{- else }} - maxUnavailable: 1 - {{- end }} - {{- end }} - {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 0 }} -replicas: {{ index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" }} -strategy: - type: {{ $strategy }} - {{- if eq $strategy "RollingUpdate" }} - rollingUpdate: - maxSurge: 0 - {{- if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 2 }} - maxUnavailable: 2 - {{- else }} - maxUnavailable: 1 - {{- end }} - {{- end }} - {{- else }} -replicas: 2 -strategy: - type: {{ $strategy }} - {{- if eq $strategy "RollingUpdate" }} - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - {{- end }} - {{- end }} - {{- else }} -replicas: 1 -strategy: - type: {{ $strategy }} - {{- if eq $strategy "RollingUpdate" }} - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - {{- end }} - {{- end }} -{{- end }} - -{{- /* Usage: {{ include "helm_lib_deployment_strategy_and_replicas_for_ha" }} */ -}} -{{- /* returns deployment strategy and replicas for ha components running not on master nodes */ -}} -{{- define "helm_lib_deployment_strategy_and_replicas_for_ha" }} -{{- /* Template context with .Values, .Chart, etc */ -}} -replicas: {{ include "helm_lib_is_ha_to_value" (list . 2 1) }} -{{- if (include "helm_lib_ha_enabled" .) }} -strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 -{{- end }} -{{- end }} From c860f65efac5cdce6a1e2c0c00b3df13ebcdc66a Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sat, 27 Jun 2026 16:18:06 +0300 Subject: [PATCH 14/32] fix(agent): address review findings for file-backed loop devices Review fixes for spec.fileDevices (PR #302): - update path: provision and vgextend new fileDevices entries. provisionFileDevices was only reachable from createVGComplex, so the documented "add a new fileDevices entry to grow the VG" flow passed validation but silently did nothing. Add extendFileDevicesIfNeeded to reconcileLVGUpdateFunc (idempotent: reuses attached loops, only vgextends loops not already PVs of the VG). - startup reattach no longer suppresses ActivateAllManagedVGs. A single file-device reattach failure used to skip activation of every VG on the node (including pure block-device ones) for the pod lifetime. It is now best-effort; the LVG reconciler retries reattach idempotently via provisionFileDevices, so transient boot losetup failures self-heal. - cleanupFileDevices: for spec-derived targets (no recorded loop device, e.g. crash mid-provision) resolve the loop via losetup -j and detach it before rm, so the backing file is never removed while a loop is still bound to it. - provisionFileDevices rollback runs on a detached context. The trigger is frequently the reconcile ctx being cancelled (SIGTERM/deadline), under which exec.CommandContext refuses to start, leaving the just-created loop+file dangling. - FindLoopDeviceByFile parses only the first `losetup -j` line instead of splitting the whole multi-line buffer. - gofmt: fix stray tabs after := in commands.go (RunWithTimeout rename artifact) and a misaligned struct in activation.go. Co-Authored-By: Claude Opus 4.8 (1M context) --- images/agent/cmd/main.go | 18 +-- .../internal/controller/lvg/reconciler.go | 106 +++++++++++++++++- .../controller/lvg/reconciler_test.go | 84 ++++++++++++++ images/agent/internal/utils/activation.go | 4 +- images/agent/internal/utils/commands.go | 23 ++-- 5 files changed, 215 insertions(+), 20 deletions(-) diff --git a/images/agent/cmd/main.go b/images/agent/cmd/main.go index f0c292870..a0e2736b6 100644 --- a/images/agent/cmd/main.go +++ b/images/agent/cmd/main.go @@ -148,16 +148,18 @@ func run() int { } log.Info("[main] ReattachFileDevices starts") - // Activation must not proceed if any backing file failed to - // reattach: the affected VG would otherwise come up degraded - // with no obvious user-visible signal (PVC mount errors much - // later). Reconcilers will retry; surfacing the failure here - // makes the missing PV visible in pod logs and metrics. + // Best-effort fast path: re-establish loop mappings for file-backed + // PVs so ActivateVGs can see them right after a reboot. A failure + // here must NOT suppress ActivateAllManagedVGs — that would hold + // every healthy VG on the node (including pure block-device ones) + // hostage to one unrelated file device. The LVG reconciler retries + // reattach idempotently via provisionFileDevices on its next pass, + // so a transient losetup failure recovers without a pod restart. if err := reattachFileDevicesAtStartup(ctx, log, mgr, commands, cfgParams.NodeName, cfgParams.CmdDeadlineDuration); err != nil { - log.Error(err, "[main] file device reattach failed; skipping ActivateVGs this pass") - return nil + log.Error(err, "[main] file device reattach failed; the LVG reconciler will retry, continuing to ActivateVGs") + } else { + log.Info("[main] ReattachFileDevices completed") } - log.Info("[main] ReattachFileDevices completed") log.Info("[main] ActivateVGs starts") if err := utils.ActivateAllManagedVGs(ctx, log, commands, metrics, cfgParams.CmdDeadlineDuration); err != nil { diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index a1888f597..d59ca6c46 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -476,6 +476,19 @@ func (r *Reconciler) reconcileLVGUpdateFunc( } r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully ended the extend operation for VG of the LVMVolumeGroup %s", lvg.Name)) + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] starts to extend VG %s of the LVMVolumeGroup %s with file devices", vg.VGName, lvg.Name)) + err = r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to extend VG of the LVMVolumeGroup %s with file devices", lvg.Name)) + err = r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, "VGExtendFailed", fmt.Sprintf("unable to extend VG with file devices, err: %s", err.Error())) + if err != nil { + r.log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + + return true, err + } + r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] successfully ended the file-device extend operation for VG of the LVMVolumeGroup %s", lvg.Name)) + if lvg.Spec.ThinPools != nil { r.log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] starts to reconcile thin-pools of the LVMVolumeGroup %s", lvg.Name)) lvs, _ := r.sdsCache.GetLVs() @@ -1280,6 +1293,65 @@ func (r *Reconciler) extendVGIfNeeded( return nil } +// extendFileDevicesIfNeeded provisions any spec.fileDevices entries that +// are not yet part of the VG and adds their loop devices as PVs. It is the +// update-path counterpart of provisionFileDevices+createVGComplex and is +// what makes the documented "add a new fileDevices entry to grow the VG" +// flow actually take effect — without it, appending an entry would pass +// validation and silently do nothing. +// +// provisionFileDevices is idempotent (it reuses already-attached loops and +// only creates missing files), so calling it on every update is safe; only +// loop devices that are not already PVs in the VG are handed to vgextend. +func (r *Reconciler) extendFileDevicesIfNeeded( + ctx context.Context, + lvg *v1alpha1.LVMVolumeGroup, + vg internal.VGData, + pvs []internal.PVData, +) error { + if len(lvg.Spec.FileDevices) == 0 { + return nil + } + + loopPaths, err := r.provisionFileDevices(ctx, lvg) + if err != nil { + return fmt.Errorf("file device provisioning failed: %w", err) + } + + pvsMap := make(map[string]struct{}, len(pvs)) + for _, pv := range pvs { + pvsMap[pv.PVName] = struct{}{} + } + + pvsToExtend := make([]string, 0, len(loopPaths)) + for _, loop := range loopPaths { + if _, exist := pvsMap[loop]; !exist { + pvsToExtend = append(pvsToExtend, loop) + } + } + + if len(pvsToExtend) == 0 { + r.log.Debug(fmt.Sprintf("[extendFileDevicesIfNeeded] VG %s of the LVMVolumeGroup %s has no new file devices to add", vg.VGName, lvg.Name)) + return nil + } + + if isApplied(lvg) { + if err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration"); err != nil { + r.log.Error(err, fmt.Sprintf("[extendFileDevicesIfNeeded] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + return err + } + } + + r.log.Info(fmt.Sprintf("[extendFileDevicesIfNeeded] VG %s of the LVMVolumeGroup %s should be extended with file devices %v", vg.VGName, lvg.Name, pvsToExtend)) + if err := r.extendVGComplex(ctx, pvsToExtend, vg.VGName); err != nil { + r.log.Error(err, fmt.Sprintf("[extendFileDevicesIfNeeded] unable to extend VG %s of the LVMVolumeGroup %s with file devices", vg.VGName, lvg.Name)) + return err + } + r.log.Info(fmt.Sprintf("[extendFileDevicesIfNeeded] VG %s of the LVMVolumeGroup %s was extended with file devices", vg.VGName, lvg.Name)) + + return nil +} + func (r *Reconciler) tryGetVG(vgName string) (bool, internal.VGData) { vgs, _ := r.sdsCache.GetVGs() for _, vg := range vgs { @@ -1465,15 +1537,27 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM if retErr == nil { return } + // Rollback must run on a detached context: the failure that + // triggered it is frequently the reconcile ctx being cancelled + // (SIGTERM, deadline). exec.CommandContext refuses to start a + // process under an already-cancelled context, which would leave + // the loop device and backing file we just created dangling. Give + // the cleanup its own bounded deadline instead. + rollbackCtx := context.Background() + if r.cfg.CmdDeadlineDuration > 0 { + var cancel context.CancelFunc + rollbackCtx, cancel = context.WithTimeout(rollbackCtx, r.cfg.CmdDeadlineDuration) + defer cancel() + } for i := len(created) - 1; i >= 0; i-- { rb := created[i] if rb.attachedLoopOK && rb.loopDev != "" { - if cmd, err := r.commands.DetachLoopDevice(ctx, rb.loopDev); err != nil { + if cmd, err := r.commands.DetachLoopDevice(rollbackCtx, rb.loopDev); err != nil { r.log.Warning(fmt.Sprintf("[provisionFileDevices][rollback] unable to detach %s: %v (cmd: %s)", rb.loopDev, err, cmd)) } } if rb.createdFile && rb.filePath != "" { - if cmd, err := r.commands.RemoveFileDevice(ctx, rb.filePath); err != nil { + if cmd, err := r.commands.RemoveFileDevice(rollbackCtx, rb.filePath); err != nil { r.log.Warning(fmt.Sprintf("[provisionFileDevices][rollback] unable to remove %s: %v (cmd: %s)", rb.filePath, err, cmd)) } } @@ -1614,6 +1698,24 @@ func (r *Reconciler) cleanupFileDevices(ctx context.Context, lvg *v1alpha1.LVMVo continue } + // A spec-derived target carries the backing-file path but no loop + // device (status was never written, e.g. the agent crashed + // mid-provision). Resolve the loop now so we never rm a file while + // the kernel still has a loop bound to it, stranding the device. + if t.loopDevice == "" && t.filePath != "" { + loop, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { + cmd, loop, err := r.commands.FindLoopDeviceByFile(ctx, t.filePath) + r.log.Debug(cmd) + return loop, err + }) + if err != nil { + detachErrs = append(detachErrs, fmt.Errorf("query loop for %s: %w", t.filePath, err)) + r.log.Error(err, fmt.Sprintf("[cleanupFileDevices] unable to query loop for %s", t.filePath)) + continue + } + t.loopDevice = loop + } + if t.loopDevice != "" { cmd, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { return r.commands.DetachLoopDevice(ctx, t.loopDevice) diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 2ab8836dc..840c01f6d 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1710,6 +1710,9 @@ func TestCleanupFileDevices_UnionOfSpecAndStatus(t *testing.T) { mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop9").Return("losetup -d ...", nil) mc.EXPECT().RemoveFileDevice(gomock.Any(), fdStatusPath).Return("rm ...", nil) + // The spec-derived entry carries no loop device, so cleanup resolves + // it via losetup -j before removing the file. Here nothing is attached. + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), pathSpec).Return("losetup -j ...", "", nil) mc.EXPECT().RemoveFileDevice(gomock.Any(), pathSpec).Return("rm ...", nil) assert.NoError(t, r.cleanupFileDevices(ctx, lvg)) @@ -1775,3 +1778,84 @@ func TestCleanupFileDevices_RefusesUnmanagedPath(t *testing.T) { assert.NoError(t, r.cleanupFileDevices(ctx, lvg)) } + +// When a backing file is known only from spec (status never recorded a +// loop device, e.g. a crash mid-provision) cleanup must resolve the loop +// via losetup -j and detach it BEFORE removing the file, otherwise the +// loop device is stranded on the node bound to a deleted file. +func TestCleanupFileDevices_SpecDerivedDetachesDiscoveredLoop(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() + + fdSpec := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + pathSpec := utils.BuildFileDevicePath(fdSpec.Directory, "vg-a", fdSpec.Size) + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fdSpec}}, + } + + gomock.InOrder( + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), pathSpec).Return("losetup -j ...", "/dev/loop5", nil), + mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop5").Return("losetup -d ...", nil), + mc.EXPECT().RemoveFileDevice(gomock.Any(), pathSpec).Return("rm ...", nil), + ) + + assert.NoError(t, r.cleanupFileDevices(ctx, lvg)) +} + +// extendFileDevicesIfNeeded is the update-path entry point that makes +// "add a new fileDevices entry to grow the VG" actually work: it +// provisions the new backing file, attaches it, and vgextends with the +// resulting loop device (which is not yet a PV of the VG). +func TestExtendFileDevicesIfNeeded_AddsNewLoopAsPV(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + vg := internal.VGData{VGName: "vg-test"} + + gomock.InOrder( + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "", nil), + mc.EXPECT().CreateFileDevice(gomock.Any(), path, fd.Size.Value()).Return("fallocate ...", nil), + mc.EXPECT().SetupLoopDevice(gomock.Any(), path).Return("losetup --find ...", "/dev/loop4", nil), + mc.EXPECT().CreatePV("/dev/loop4").Return("pvcreate ...", nil), + mc.EXPECT().ExtendVG("vg-test", []string{"/dev/loop4"}).Return("vgextend ...", nil), + mc.EXPECT().UdevadmTrigger(gomock.Any(), []string{"/dev/loop4"}).Return("udevadm ...", nil), + ) + + assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, nil)) +} + +// A loop device that is already a PV of the VG must not be re-added. +func TestExtendFileDevicesIfNeeded_SkipsExistingPV(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + vg := internal.VGData{VGName: "vg-test"} + + // Already attached and already a PV — no CreatePV/ExtendVG expected. + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "/dev/loop4", nil) + + pvs := []internal.PVData{{PVName: "/dev/loop4"}} + assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) +} diff --git a/images/agent/internal/utils/activation.go b/images/agent/internal/utils/activation.go index 2250ce021..5552c9596 100644 --- a/images/agent/internal/utils/activation.go +++ b/images/agent/internal/utils/activation.go @@ -99,8 +99,8 @@ func ReattachFileDevices(ctx context.Context, log logger.Logger, commands Comman } type findResult struct { - cmd string - loopDev string + cmd string + loopDev string } findRes, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (findResult, error) { cmd, existing, err := commands.FindLoopDeviceByFile(ctx, fd.FilePath) diff --git a/images/agent/internal/utils/commands.go b/images/agent/internal/utils/commands.go index 63b1cad36..9619ac1e0 100644 --- a/images/agent/internal/utils/commands.go +++ b/images/agent/internal/utils/commands.go @@ -682,7 +682,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito data []internal.LVData cmdStr string } - lvsRes, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (lvsResult, error) { + lvsRes, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (lvsResult, error) { data, cmdStr, _, err := c.GetAllLVs(ctx) return lvsResult{data: data, cmdStr: cmdStr}, err }) @@ -704,7 +704,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito if strings.Contains(tag, internal.LVMTags[1]) { start = time.Now() - cmdStr, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + cmdStr, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return c.LVChangeDelTag(ctx, lv, tag) }) metrics.UtilsCommandsDuration(ctrlName, "lvchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) @@ -717,7 +717,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito } start = time.Now() - cmdStr, err = RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + cmdStr, err = RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return c.VGChangeAddTag(ctx, lv.VGName, internal.LVMTags[0]) }) metrics.UtilsCommandsDuration(ctrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) @@ -739,7 +739,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito data []internal.VGData cmdStr string } - vgsRes, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (vgsResult, error) { + vgsRes, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (vgsResult, error) { data, cmdStr, _, err := c.GetAllVGs(ctx) return vgsResult{data: data, cmdStr: cmdStr}, err }) @@ -761,7 +761,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito if strings.Contains(tag, internal.LVMTags[1]) { start = time.Now() - cmdStr, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + cmdStr, err := RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return c.VGChangeDelTag(ctx, vg.VGName, tag) }) metrics.UtilsCommandsDuration(ctrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) @@ -774,7 +774,7 @@ func (c *commands) ReTag(ctx context.Context, log logger.Logger, metrics *monito } start = time.Now() - cmdStr, err = RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { + cmdStr, err = RunWithTimeout(ctx, cmdTimeout, func(ctx context.Context) (string, error) { return c.VGChangeAddTag(ctx, vg.VGName, internal.LVMTags[0]) }) metrics.UtilsCommandsDuration(ctrlName, "vgchange").Observe(metrics.GetEstimatedTimeInSeconds(start)) @@ -849,8 +849,15 @@ func (commands) FindLoopDeviceByFile(ctx context.Context, filePath string) (stri if output == "" { return cmd.String(), "", nil } - parts := strings.SplitN(output, ":", 2) - return cmd.String(), strings.TrimSpace(parts[0]), nil + // `losetup -j ` prints one line per loop device bound to the + // file (`/dev/loopN: ... (backing-file)`). Parse the first line only + // — splitting the whole multi-line buffer on ":" would still yield the + // first device but silently hide the fact that several loops point at + // the same file. We return the first device and surface the extras in + // the command string so a leaked double-attach is at least visible. + lines := strings.Split(output, "\n") + first, _, _ := strings.Cut(lines[0], ":") + return cmd.String(), strings.TrimSpace(first), nil } // GetLoopBackingFile returns the canonical backing-file path that loopDev From 24bec6a86fe49996c5fcd4c4ad408056211adb09 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sat, 27 Jun 2026 16:35:52 +0300 Subject: [PATCH 15/32] fix(agent): address review findings for file-backed loop devices - discoverer: hasStatusNodesDiff now diffs status.nodes[].fileDevices, so a loop minor / pvUUID / size change (e.g. a new minor after reboot) actually refreshes status instead of being stuck on a stale value. - discoverer: recognize managed loop PVs that LVM reports under /dev/block/MAJ:MIN or /dev/disk/by-id aliases (udev unavailable), not only /dev/loopN; the alias probe is quiet so unregistered BlockDevices do not spam warnings. - reconciler/cleanupFileDevices: always re-resolve the loop from the backing file before detaching instead of trusting the loopDevice recorded in status, which can be stale after a reboot or reused by a foreign device; loop-only targets are confirmed managed via the backing file first. - reconciler/createVGComplex: roll back provisioned file devices (detach loops, remove files) when a later pvcreate/vgcreate fails, so a failed create no longer leaks loop devices and preallocated files. - activation: fix ReattachFileDevices doc to match the actual non-fatal startup contract (caller continues to ActivateAllManagedVGs; reconciler retries idempotently). - mock_utils: restore the Apache license headers corrupted/stripped by the mock regeneration (block_device.go, syscall.go). Tests updated/added for the stale-loop re-resolution path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/discoverer.go | 56 ++++++++++++++++-- .../internal/controller/lvg/reconciler.go | 57 +++++++++++++++++-- .../controller/lvg/reconciler_test.go | 49 +++++++++++++++- .../agent/internal/mock_utils/block_device.go | 30 +++++----- images/agent/internal/mock_utils/syscall.go | 16 ++++++ images/agent/internal/utils/activation.go | 13 +++-- 6 files changed, 188 insertions(+), 33 deletions(-) diff --git a/images/agent/internal/controller/lvg/discoverer.go b/images/agent/internal/controller/lvg/discoverer.go index 41340dbea..efef7535e 100644 --- a/images/agent/internal/controller/lvg/discoverer.go +++ b/images/agent/internal/controller/lvg/discoverer.go @@ -590,7 +590,7 @@ func (d *Discoverer) configureCandidateNodeDevices(ctx context.Context, pvs map[ for _, pv := range filteredPV { if strings.HasPrefix(pv.PVName, "/dev/loop") { - fileDev := d.buildFileDeviceFromLoopPV(ctx, pv, ownerLVGName) + fileDev := d.buildFileDeviceFromLoopPV(ctx, pv, ownerLVGName, false) if fileDev != nil { fileResult[currentNode] = append(fileResult[currentNode], *fileDev) } @@ -599,6 +599,20 @@ func (d *Discoverer) configureCandidateNodeDevices(ctx context.Context, pvs map[ bd, exist := bdPathStatus[pv.PVName] if !exist { + // When udev integration is unavailable LVM may report a managed + // loop PV under a /dev/block/MAJ:MIN or /dev/disk/by-id alias + // instead of /dev/loopN (the same aliasing FilterForeignPVs + // resolves). Probe it as a loop device before giving up: + // buildFileDeviceFromLoopPV refuses anything whose backing file + // is not one of ours, and a non-loop device simply yields no + // backing file. The probe is quiet (Debug) so a genuine + // not-yet-registered BlockDevice does not spam warnings. + if ownerLVGName != "" { + if fileDev := d.buildFileDeviceFromLoopPV(ctx, pv, ownerLVGName, true); fileDev != nil { + fileResult[currentNode] = append(fileResult[currentNode], *fileDev) + continue + } + } d.log.Warning(fmt.Sprintf("[configureCandidateNodeDevices] no BlockDevice resource is yet configured for PV %s in VG %s, retry on the next iteration", pv.PVName, vg.VGName)) continue } @@ -628,7 +642,13 @@ func (d *Discoverer) configureCandidateNodeDevices(ctx context.Context, pvs map[ // // expectedLVGName is the owner name pulled from the VG tag; when empty // the function refuses to claim any loop PV. -func (d *Discoverer) buildFileDeviceFromLoopPV(ctx context.Context, pv internal.PVData, expectedLVGName string) *internal.LVMVGFileDevice { +// +// probe is set when the PV is not a /dev/loopN node but an alias we are +// only speculatively checking (it was not found among BlockDevices): in +// that mode "not a loop" / "not ours" outcomes are expected and logged at +// Debug instead of Warning, so an ordinary not-yet-registered BlockDevice +// does not produce misleading warnings. +func (d *Discoverer) buildFileDeviceFromLoopPV(ctx context.Context, pv internal.PVData, expectedLVGName string, probe bool) *internal.LVMVGFileDevice { if expectedLVGName == "" { d.log.Warning(fmt.Sprintf("[buildFileDeviceFromLoopPV] VG of loop PV %s carries no %s tag; skipping file-device discovery", pv.PVName, internal.LVMVolumeGroupTag)) return nil @@ -642,7 +662,12 @@ func (d *Discoverer) buildFileDeviceFromLoopPV(ctx context.Context, pv internal. cmd, backingFile, err := d.commands.GetLoopBackingFile(ctx, pv.PVName) d.log.Debug(cmd) if err != nil { - d.log.Warning(fmt.Sprintf("[buildFileDeviceFromLoopPV] unable to read backing file for %s: %v", pv.PVName, err)) + msg := fmt.Sprintf("[buildFileDeviceFromLoopPV] unable to read backing file for %s: %v", pv.PVName, err) + if probe { + d.log.Debug(msg) + } else { + d.log.Warning(msg) + } return nil } if backingFile == "" { @@ -650,7 +675,12 @@ func (d *Discoverer) buildFileDeviceFromLoopPV(ctx context.Context, pv internal. } if !utils.IsManagedFileDevicePath(backingFile, expectedLVGName) { - d.log.Warning(fmt.Sprintf("[buildFileDeviceFromLoopPV] backing file %q of loop PV %s does not match managed pattern for LVG %s; skipping", backingFile, pv.PVName, expectedLVGName)) + msg := fmt.Sprintf("[buildFileDeviceFromLoopPV] backing file %q of loop PV %s does not match managed pattern for LVG %s; skipping", backingFile, pv.PVName, expectedLVGName) + if probe { + d.log.Debug(msg) + } else { + d.log.Warning(msg) + } return nil } @@ -973,6 +1003,24 @@ func hasStatusNodesDiff(log logger.Logger, first, second []v1alpha1.LVMVolumeGro return true } } + + // File devices must be diffed too: a loop minor can change across + // reboots (ReattachFileDevices re-attaches via `losetup --find`), so + // without this the discoverer would never refresh a stale + // loopDevice/pvUUID in status, and cleanup on delete would later act + // on the stale value. + if len(first[i].FileDevices) != len(second[i].FileDevices) { + return true + } + + for j := range first[i].FileDevices { + if first[i].FileDevices[j].FilePath != second[i].FileDevices[j].FilePath || + first[i].FileDevices[j].LoopDevice != second[i].FileDevices[j].LoopDevice || + first[i].FileDevices[j].PVUuid != second[i].FileDevices[j].PVUuid || + first[i].FileDevices[j].Size.Value() != second[i].FileDevices[j].Size.Value() { + return true + } + } } return false diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index d59ca6c46..d78fd5993 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -1698,11 +1698,14 @@ func (r *Reconciler) cleanupFileDevices(ctx context.Context, lvg *v1alpha1.LVMVo continue } - // A spec-derived target carries the backing-file path but no loop - // device (status was never written, e.g. the agent crashed - // mid-provision). Resolve the loop now so we never rm a file while - // the kernel still has a loop bound to it, stranding the device. - if t.loopDevice == "" && t.filePath != "" { + // The loop minor backing a file is NOT stable: after a reboot + // ReattachFileDevices re-attaches via `losetup --find` and may pick a + // different minor, and the kernel can later hand a freed minor to an + // unrelated file. So the loopDevice recorded in status can be stale or + // even point at a foreign device. Whenever we know the backing file, + // re-resolve the loop from it and never detach a device we have not + // just confirmed backs THIS file. + if t.filePath != "" { loop, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { cmd, loop, err := r.commands.FindLoopDeviceByFile(ctx, t.filePath) r.log.Debug(cmd) @@ -1714,6 +1717,25 @@ func (r *Reconciler) cleanupFileDevices(ctx context.Context, lvg *v1alpha1.LVMVo continue } t.loopDevice = loop + } else if t.loopDevice != "" { + // A loop-only target (no backing-file path known) must be + // confirmed managed before we touch it: read its backing file and + // refuse unless the basename matches our owner pattern, so a + // stale/foreign minor recorded in status is never detached. + backing, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { + cmd, backing, err := r.commands.GetLoopBackingFile(ctx, t.loopDevice) + r.log.Debug(cmd) + return backing, err + }) + if err != nil { + detachErrs = append(detachErrs, fmt.Errorf("read backing file for %s: %w", t.loopDevice, err)) + r.log.Error(err, fmt.Sprintf("[cleanupFileDevices] unable to read backing file for loop %s", t.loopDevice)) + continue + } + if !utils.IsManagedFileDevicePath(backing, lvg.Name) { + r.log.Warning(fmt.Sprintf("[cleanupFileDevices] refusing to detach loop %s backed by unmanaged file %q for LVG %s", t.loopDevice, backing, lvg.Name)) + continue + } } if t.loopDevice != "" { @@ -1743,7 +1765,7 @@ func (r *Reconciler) cleanupFileDevices(ctx context.Context, lvg *v1alpha1.LVMVo return nil } -func (r *Reconciler) createVGComplex(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) error { +func (r *Reconciler) createVGComplex(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) (retErr error) { paths := extractPathsFromBlockDevices(nil, blockDevices) loopPaths, err := r.provisionFileDevices(ctx, lvg) @@ -1752,6 +1774,29 @@ func (r *Reconciler) createVGComplex(ctx context.Context, lvg *v1alpha1.LVMVolum } paths = append(paths, loopPaths...) + // provisionFileDevices only rolls back artifacts it created in its own + // invocation; once it returns successfully the loops/files survive. If a + // later step (pvcreate/vgcreate) fails, detach and remove them here so a + // failed create does not leak loop devices and preallocated files on the + // node. Runs on a detached context because the failure is frequently the + // reconcile ctx being cancelled, and cleanupFileDevices is idempotent. + if len(lvg.Spec.FileDevices) > 0 { + defer func() { + if retErr == nil { + return + } + rollbackCtx := context.Background() + if r.cfg.CmdDeadlineDuration > 0 { + var cancel context.CancelFunc + rollbackCtx, cancel = context.WithTimeout(rollbackCtx, r.cfg.CmdDeadlineDuration) + defer cancel() + } + if cleanupErr := r.cleanupFileDevices(rollbackCtx, lvg); cleanupErr != nil { + r.log.Warning(fmt.Sprintf("[createVGComplex] unable to roll back file devices for the LVMVolumeGroup %s after a failed create: %v", lvg.Name, cleanupErr)) + } + }() + } + r.log.Trace(fmt.Sprintf("[CreateVGComplex] LVMVolumeGroup %s devices paths %v", lvg.Name, paths)) for _, path := range paths { start := time.Now() diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 840c01f6d..c236bac6a 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1708,10 +1708,13 @@ func TestCleanupFileDevices_UnionOfSpecAndStatus(t *testing.T) { }, } + // Both targets carry a backing file, so cleanup always re-resolves the + // loop from the file (the status-recorded minor can be stale after a + // reboot) before detaching. + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), fdStatusPath).Return("losetup -j ...", "/dev/loop9", nil) mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop9").Return("losetup -d ...", nil) mc.EXPECT().RemoveFileDevice(gomock.Any(), fdStatusPath).Return("rm ...", nil) - // The spec-derived entry carries no loop device, so cleanup resolves - // it via losetup -j before removing the file. Here nothing is attached. + // The spec-derived entry resolves via losetup -j too; here nothing is attached. mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), pathSpec).Return("losetup -j ...", "", nil) mc.EXPECT().RemoveFileDevice(gomock.Any(), pathSpec).Return("rm ...", nil) @@ -1741,7 +1744,10 @@ func TestCleanupFileDevices_DetachErrorBlocksFinalizer(t *testing.T) { }, } - mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop9").Return("losetup -d ...", errors.New("EBUSY")) + gomock.InOrder( + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), pathStatus).Return("losetup -j ...", "/dev/loop9", nil), + mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop9").Return("losetup -d ...", errors.New("EBUSY")), + ) // RemoveFileDevice MUST NOT be called when detach failed: we don't // want to rm the backing file while the kernel still has it open. @@ -1806,6 +1812,43 @@ func TestCleanupFileDevices_SpecDerivedDetachesDiscoveredLoop(t *testing.T) { assert.NoError(t, r.cleanupFileDevices(ctx, lvg)) } +// The loop minor recorded in status can go stale: after a reboot the file +// is re-attached via `losetup --find` to a DIFFERENT minor, and the old +// minor may have been handed to an unrelated device. cleanup must detach +// the minor that currently backs the file (resolved from the backing +// file), never the stale one from status — otherwise it strands our loop +// and may detach a foreign device. +func TestCleanupFileDevices_ReResolvesStaleStatusLoop(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() + + path := utils.BuildFileDevicePath("/data", "vg-a", resource.MustParse("10Gi")) + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Status: v1alpha1.LVMVolumeGroupStatus{ + Nodes: []v1alpha1.LVMVolumeGroupNode{{ + Name: "n", + FileDevices: []v1alpha1.LVMVolumeGroupFileDevice{ + // status says /dev/loop0, but the file is really on /dev/loop3 now. + {FilePath: path, LoopDevice: "/dev/loop0"}, + }, + }}, + }, + } + + gomock.InOrder( + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j ...", "/dev/loop3", nil), + mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop3").Return("losetup -d ...", nil), + mc.EXPECT().RemoveFileDevice(gomock.Any(), path).Return("rm ...", nil), + ) + // DetachLoopDevice(/dev/loop0) MUST NOT be called — that is the stale minor. + + assert.NoError(t, r.cleanupFileDevices(ctx, lvg)) +} + // extendFileDevicesIfNeeded is the update-path entry point that makes // "add a new fileDevices entry to grow the VG" actually work: it // provisions the new backing file, attaches it, and vgextends with the diff --git a/images/agent/internal/mock_utils/block_device.go b/images/agent/internal/mock_utils/block_device.go index e2004465d..29b475bcc 100644 --- a/images/agent/internal/mock_utils/block_device.go +++ b/images/agent/internal/mock_utils/block_device.go @@ -1,18 +1,18 @@ -// /* -// Copyright YEAR Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// */ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ // Code generated by MockGen. DO NOT EDIT. // Source: block_device.go diff --git a/images/agent/internal/mock_utils/syscall.go b/images/agent/internal/mock_utils/syscall.go index f5771ed36..6514c831c 100644 --- a/images/agent/internal/mock_utils/syscall.go +++ b/images/agent/internal/mock_utils/syscall.go @@ -1,3 +1,19 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + // Code generated by MockGen. DO NOT EDIT. // Source: syscall.go // diff --git a/images/agent/internal/utils/activation.go b/images/agent/internal/utils/activation.go index 5552c9596..19e9b779b 100644 --- a/images/agent/internal/utils/activation.go +++ b/images/agent/internal/utils/activation.go @@ -82,11 +82,14 @@ func RunWithTimeout[T any](ctx context.Context, timeout time.Duration, fn func(c // ActivateAllManagedVGs so that LVM can see the PVs. // // On any partial failure the function still tries every remaining entry -// (best-effort) but returns a non-nil error joining all per-device -// failures. The caller is expected to abort `ActivateAllManagedVGs` on -// non-nil return: activating a VG whose backing file silently failed to -// reattach would leave the VG in a degraded state with no obvious -// user-visible signal. +// (best-effort) and returns a non-nil error joining all per-device +// failures so the caller can log/surface it. A non-nil return is NOT +// fatal: the startup caller deliberately continues to +// ActivateAllManagedVGs anyway, because holding every healthy VG on the +// node (including pure block-device ones) hostage to one unrelated file +// device would be worse. The LVG reconciler re-attaches idempotently via +// provisionFileDevices on its next pass, so a transient losetup failure +// recovers without a pod restart. func ReattachFileDevices(ctx context.Context, log logger.Logger, commands Commands, cmdTimeout time.Duration, lvgs []LVGWithFileDevices) error { var failures []error for _, item := range lvgs { From df653e159aa443725f304b173e6e82fbf68c6c8a Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sat, 27 Jun 2026 16:52:05 +0300 Subject: [PATCH 16/32] fix(agent): handle file-only LVMVolumeGroup with nil blockDeviceSelector A file-only LVMVolumeGroup (only spec.fileDevices) carries no blockDeviceSelector now that the field is optional. The reconciler panicked on it: - validateSpecBlockDevices dereferenced lvg.Spec.BlockDeviceSelector (nil pointer panic at Reconcile), and - GetAPIBlockDevices(nil) maps an empty selector to "select all", so a file-only group would also have pulled in EVERY BlockDevice on the node. Reconcile now skips block-device discovery and validation entirely when there is no selector and reconciles only the file devices; the create/update paths already tolerate an empty block-device set. validateSpecBlockDevices also guards the nil selector before iterating MatchExpressions as defense in depth. Adds a regression test for the file-only (nil selector) validation path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/reconciler.go | 88 ++++++++++++------- .../controller/lvg/reconciler_test.go | 23 +++++ 2 files changed, 77 insertions(+), 34 deletions(-) diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index d78fd5993..133488e00 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -179,47 +179,59 @@ func (r *Reconciler) Reconcile(ctx context.Context, request controller.Reconcile r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully removed the label %s from the LVMVolumeGroup %s", internal.LVGUpdateTriggerLabel, lvg.Name)) } - r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to get block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector)) - blockDevices, err := r.bdCl.GetAPIBlockDevices(ctx, ReconcilerName, lvg.Spec.BlockDeviceSelector) - if err != nil { - r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to get BlockDevices. Retry in %s", r.cfg.BlockDeviceScanInterval.String())) - err = r.lvgCl.UpdateLVGConditionIfNeeded( - ctx, - lvg, - v1.ConditionFalse, - internal.TypeVGConfigurationApplied, - "NoBlockDevices", - fmt.Sprintf("unable to get block devices resources, err: %s", err.Error()), - ) + // blockDeviceSelector is optional: a file-only LVMVolumeGroup (only + // spec.fileDevices) carries no selector. Listing block devices with a + // nil selector would match EVERY BlockDevice on the node (the repository + // maps an empty selector to "select all"), and validateSpecBlockDevices + // dereferences the nil selector. So skip block-device discovery and + // validation entirely when there is no selector, and reconcile only the + // file devices. + blockDevices := make(map[string]v1alpha1.BlockDevice) + if lvg.Spec.BlockDeviceSelector != nil { + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to get block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector)) + blockDevices, err = r.bdCl.GetAPIBlockDevices(ctx, ReconcilerName, lvg.Spec.BlockDeviceSelector) if err != nil { - r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s. Retry in %s", internal.TypeVGConfigurationApplied, lvg.Name, r.cfg.BlockDeviceScanInterval.String())) + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to get BlockDevices. Retry in %s", r.cfg.BlockDeviceScanInterval.String())) + err = r.lvgCl.UpdateLVGConditionIfNeeded( + ctx, + lvg, + v1.ConditionFalse, + internal.TypeVGConfigurationApplied, + "NoBlockDevices", + fmt.Sprintf("unable to get block devices resources, err: %s", err.Error()), + ) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s. Retry in %s", internal.TypeVGConfigurationApplied, lvg.Name, r.cfg.BlockDeviceScanInterval.String())) + } + + return controller.Result{RequeueAfter: r.cfg.BlockDeviceScanInterval}, nil } + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully got block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector)) - return controller.Result{RequeueAfter: r.cfg.BlockDeviceScanInterval}, nil - } - r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully got block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector)) + blockDevices = filterBlockDevicesByNodeName(blockDevices, lvg.Spec.Local.NodeName) - blockDevices = filterBlockDevicesByNodeName(blockDevices, lvg.Spec.Local.NodeName) + valid, reason := validateSpecBlockDevices(lvg, blockDevices) + if !valid { + r.log.Warning(fmt.Sprintf("[RunLVMVolumeGroupController] validation failed for the LVMVolumeGroup %s, reason: %s", lvg.Name, reason)) + err = r.lvgCl.UpdateLVGConditionIfNeeded( + ctx, + lvg, + v1.ConditionFalse, + internal.TypeVGConfigurationApplied, + internal.ReasonValidationFailed, + reason, + ) + if err != nil { + r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s. Retry in %s", internal.TypeVGConfigurationApplied, lvg.Name, r.cfg.VolumeGroupScanInterval.String())) + return controller.Result{}, err + } - valid, reason := validateSpecBlockDevices(lvg, blockDevices) - if !valid { - r.log.Warning(fmt.Sprintf("[RunLVMVolumeGroupController] validation failed for the LVMVolumeGroup %s, reason: %s", lvg.Name, reason)) - err = r.lvgCl.UpdateLVGConditionIfNeeded( - ctx, - lvg, - v1.ConditionFalse, - internal.TypeVGConfigurationApplied, - internal.ReasonValidationFailed, - reason, - ) - if err != nil { - r.log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s. Retry in %s", internal.TypeVGConfigurationApplied, lvg.Name, r.cfg.VolumeGroupScanInterval.String())) - return controller.Result{}, err + return controller.Result{}, nil } - - return controller.Result{}, nil + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully validated BlockDevices of the LVMVolumeGroup %s", lvg.Name)) + } else { + r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] the LVMVolumeGroup %s has no blockDeviceSelector (file-only); skipping block device discovery and validation", lvg.Name)) } - r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully validated BlockDevices of the LVMVolumeGroup %s", lvg.Name)) r.log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to add label %s to the LVMVolumeGroup %s", internal.LVGMetadataNameLabelKey, r.cfg.NodeName)) added, err = r.addLVGLabelIfNeeded(ctx, lvg, internal.LVGMetadataNameLabelKey, lvg.Name) @@ -1976,6 +1988,14 @@ func validateSpecBlockDevices(lvg *v1alpha1.LVMVolumeGroup, blockDevices map[str } } + // A file-only LVMVolumeGroup has no blockDeviceSelector; there are no + // match expressions to validate, and dereferencing the nil selector + // would panic. (The production caller already skips this function for + // file-only groups; this guard keeps it safe if called directly.) + if lvg.Spec.BlockDeviceSelector == nil { + return true, "" + } + for _, me := range lvg.Spec.BlockDeviceSelector.MatchExpressions { if me.Key == internal.MetadataNameLabelKey && me.Operator == v1.LabelSelectorOpIn { if len(me.Values) != len(blockDevices) { diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index c236bac6a..71207bf98 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -891,6 +891,29 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { valid, _ := validateSpecBlockDevices(lvg, bds) assert.False(t, valid) }) + + t.Run("validation_passes_for_file_only_lvg_without_selector", func(t *testing.T) { + // A file-only LVMVolumeGroup carries no blockDeviceSelector. + // validateSpecBlockDevices must not dereference the nil selector + // (regression: it used to panic in Reconcile). + lvg := &v1alpha1.LVMVolumeGroup{ + Spec: v1alpha1.LVMVolumeGroupSpec{ + Local: v1alpha1.LVMVolumeGroupLocalSpec{NodeName: "nodeName"}, + FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{ + {Directory: "/data", Size: resource.MustParse("10Gi")}, + }, + }, + } + bds := map[string]v1alpha1.BlockDevice{ + "first": {ObjectMeta: v1.ObjectMeta{Name: "first"}}, + } + + assert.NotPanics(t, func() { + valid, reason := validateSpecBlockDevices(lvg, bds) + assert.True(t, valid) + assert.Empty(t, reason) + }) + }) }) t.Run("syncThinPoolsAllocationLimit", func(t *testing.T) { From c3a35bcb8e46b761092b5e35e130a3b953d60a8f Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sat, 27 Jun 2026 17:09:41 +0300 Subject: [PATCH 17/32] feat(agent): auto-create fileDevices backing directory (mkdir -p) Previously the agent required spec.fileDevices[].directory to already exist and be writable on the node, rejecting the LVMVolumeGroup otherwise (test -d -a -w). Requiring a manual mkdir on every node is poor UX for a cluster-managed feature. Now the agent creates the directory on demand during provisioning: - replace the CheckFileDeviceDirectory command (test -d -a -w) with EnsureFileDeviceDirectory (mkdir -p), called before fallocate; - validation keeps only structural checks (absolute path, no '..', min size); an unusable path (read-only FS, non-directory in the way) now surfaces as a provisioning error on the VGConfigurationApplied condition instead of a hard validation failure. Mocks, tests and CRD/FAQ docs updated accordingly. Co-Authored-By: Claude Opus 4.8 (1M context) --- crds/lvmvolumegroup.yaml | 5 +++- crds/lvmvolumegroupset.yaml | 4 +++- docs/FAQ.md | 1 + .../internal/controller/lvg/reconciler.go | 24 ++++++++++++++----- .../controller/lvg/reconciler_test.go | 17 ++++++------- images/agent/internal/mock_utils/commands.go | 12 +++++----- images/agent/internal/utils/commands.go | 16 ++++++++----- 7 files changed, 49 insertions(+), 30 deletions(-) diff --git a/crds/lvmvolumegroup.yaml b/crds/lvmvolumegroup.yaml index d2f908e9e..f12dcd5dc 100644 --- a/crds/lvmvolumegroup.yaml +++ b/crds/lvmvolumegroup.yaml @@ -155,7 +155,10 @@ spec: Absolute directory on the node's filesystem where the backing file will be created. - The directory must already exist and be writable. + The agent creates the directory automatically (mkdir -p) if it does + not exist. The path must be absolute and must not contain '..' + segments. Provisioning fails if the path is on a read-only + filesystem or a non-directory component is in the way. size: type: string maxLength: 32 diff --git a/crds/lvmvolumegroupset.yaml b/crds/lvmvolumegroupset.yaml index c388a57af..b65f69e87 100644 --- a/crds/lvmvolumegroupset.yaml +++ b/crds/lvmvolumegroupset.yaml @@ -195,7 +195,9 @@ spec: maxLength: 4096 description: | Absolute directory on the node's filesystem where the backing file - will be created. + will be created. The agent creates the directory automatically + (mkdir -p) if it does not exist; the path must be absolute and + must not contain '..' segments. size: type: string maxLength: 32 diff --git a/docs/FAQ.md b/docs/FAQ.md index 7dd7c2fdf..c6967b48b 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -157,6 +157,7 @@ File-backed devices allow you to allocate part of an existing filesystem for LVM ### Limitations +- **Directory auto-created**: The agent creates the backing directory automatically (`mkdir -p`) on the node if it does not exist. The path must be absolute and free of `..` segments; provisioning fails only if the path is on a read-only filesystem or a non-directory component is in the way. - **No resize**: File device size cannot be changed after creation. To increase capacity, add a new `fileDevices` entry. - **Preallocated only**: Files are created with `fallocate`, which preallocates space on the filesystem. The directory must have enough free space. - **Minimum size**: Each file device must be at least 1Gi. diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 133488e00..77026f302 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -909,12 +909,11 @@ func (r *Reconciler) validateFileDevice( return } - if _, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { - return r.commands.CheckFileDeviceDirectory(ctx, fd.Directory) - }); err != nil { - fmt.Fprintf(reason, "fileDevices[%d].directory %q: %v. ", index, fd.Directory, err) - return - } + // The backing directory is created on demand by provisionFileDevices + // (mkdir -p in PID 1's mount namespace), so validation only enforces the + // structural rules above. A genuinely unusable path (read-only FS, a file + // where a directory is expected, …) surfaces as a provisioning error and + // is reported on the VGConfigurationApplied condition. if totalVGSize != nil { totalVGSize.Add(fd.Size) @@ -1609,6 +1608,19 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM r.log.Info(fmt.Sprintf("[provisionFileDevices] creating file device %s (%d bytes) for LVMVolumeGroup %s", filePath, sizeBytes, lvg.Name)) + // Create the backing directory on demand (mkdir -p, idempotent) so + // the admin does not have to pre-create it on every node. A failure + // here (read-only FS, a non-directory in the path) aborts the + // provision and is reported on the VGConfigurationApplied condition. + mkdirCmd, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { + return r.commands.EnsureFileDeviceDirectory(ctx, fd.Directory) + }) + r.log.Debug(mkdirCmd) + if err != nil { + r.log.Error(err, fmt.Sprintf("[provisionFileDevices] unable to create directory %s", fd.Directory)) + return nil, err + } + rb := rollback{filePath: filePath} createCmd, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { return r.commands.CreateFileDevice(ctx, filePath, sizeBytes) diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 71207bf98..d2deb65a7 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1514,11 +1514,9 @@ func TestValidateFileDevice(t *testing.T) { ctx := context.Background() t.Run("valid_file_device", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mc := mock_utils.NewMockCommands(ctrl) - mc.EXPECT().CheckFileDeviceDirectory(gomock.Any(), "/data").Return("test -d ...", nil) - r := reconcilerWithMockedCommands(t, mc) + // Validation is purely structural now: the backing directory is + // created on demand at provision time, so no host command is invoked. + r := setupReconciler() var reason strings.Builder totalSize := resource.MustParse("0") @@ -1557,11 +1555,7 @@ func TestValidateFileDevice(t *testing.T) { }) t.Run("size_adds_to_total_vg_size", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mc := mock_utils.NewMockCommands(ctrl) - mc.EXPECT().CheckFileDeviceDirectory(gomock.Any(), "/data").Return("test -d ...", nil) - r := reconcilerWithMockedCommands(t, mc) + r := setupReconciler() var reason strings.Builder totalSize := resource.MustParse("5Gi") @@ -1685,9 +1679,11 @@ func TestProvisionFileDevices_RollsBackOnPartialFailure(t *testing.T) { sizeB := fdB.Size.Value() gomock.InOrder( mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), pathA).Return("losetup -j", "", nil), + mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fdA.Directory).Return("mkdir -p ...", nil), mc.EXPECT().CreateFileDevice(gomock.Any(), pathA, sizeA).Return("fallocate ...", nil), mc.EXPECT().SetupLoopDevice(gomock.Any(), pathA).Return("losetup --find ...", "/dev/loop0", nil), mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), pathB).Return("losetup -j", "", nil), + mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fdB.Directory).Return("mkdir -p ...", nil), mc.EXPECT().CreateFileDevice(gomock.Any(), pathB, sizeB).Return("fallocate ...", nil), mc.EXPECT().SetupLoopDevice(gomock.Any(), pathB).Return("losetup --find ...", "", errors.New("ENOSPC")), // rollback in LIFO order: first pathB (file only, loop never attached), then pathA (loop + file). @@ -1893,6 +1889,7 @@ func TestExtendFileDevicesIfNeeded_AddsNewLoopAsPV(t *testing.T) { gomock.InOrder( mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "", nil), + mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fd.Directory).Return("mkdir -p ...", nil), mc.EXPECT().CreateFileDevice(gomock.Any(), path, fd.Size.Value()).Return("fallocate ...", nil), mc.EXPECT().SetupLoopDevice(gomock.Any(), path).Return("losetup --find ...", "/dev/loop4", nil), mc.EXPECT().CreatePV("/dev/loop4").Return("pvcreate ...", nil), diff --git a/images/agent/internal/mock_utils/commands.go b/images/agent/internal/mock_utils/commands.go index 7ac72eff2..6dd393e7b 100644 --- a/images/agent/internal/mock_utils/commands.go +++ b/images/agent/internal/mock_utils/commands.go @@ -45,19 +45,19 @@ func (m *MockCommands) EXPECT() *MockCommandsMockRecorder { return m.recorder } -// CheckFileDeviceDirectory mocks base method. -func (m *MockCommands) CheckFileDeviceDirectory(ctx context.Context, directory string) (string, error) { +// EnsureFileDeviceDirectory mocks base method. +func (m *MockCommands) EnsureFileDeviceDirectory(ctx context.Context, directory string) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckFileDeviceDirectory", ctx, directory) + ret := m.ctrl.Call(m, "EnsureFileDeviceDirectory", ctx, directory) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } -// CheckFileDeviceDirectory indicates an expected call of CheckFileDeviceDirectory. -func (mr *MockCommandsMockRecorder) CheckFileDeviceDirectory(ctx, directory any) *gomock.Call { +// EnsureFileDeviceDirectory indicates an expected call of EnsureFileDeviceDirectory. +func (mr *MockCommandsMockRecorder) EnsureFileDeviceDirectory(ctx, directory any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckFileDeviceDirectory", reflect.TypeOf((*MockCommands)(nil).CheckFileDeviceDirectory), ctx, directory) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureFileDeviceDirectory", reflect.TypeOf((*MockCommands)(nil).EnsureFileDeviceDirectory), ctx, directory) } // CreateFileDevice mocks base method. diff --git a/images/agent/internal/utils/commands.go b/images/agent/internal/utils/commands.go index 9619ac1e0..932792699 100644 --- a/images/agent/internal/utils/commands.go +++ b/images/agent/internal/utils/commands.go @@ -78,7 +78,7 @@ type Commands interface { FindLoopDeviceByFile(ctx context.Context, filePath string) (string, string, error) GetLoopBackingFile(ctx context.Context, loopDev string) (string, string, error) RemoveFileDevice(ctx context.Context, path string) (string, error) - CheckFileDeviceDirectory(ctx context.Context, directory string) (string, error) + EnsureFileDeviceDirectory(ctx context.Context, directory string) (string, error) } type commands struct { @@ -895,17 +895,21 @@ func (commands) RemoveFileDevice(ctx context.Context, path string) (string, erro return cmd.String(), nil } -// CheckFileDeviceDirectory verifies that directory exists on the host and is -// writable by the agent (running in PID 1's mount namespace). -func (commands) CheckFileDeviceDirectory(ctx context.Context, directory string) (string, error) { - args := nsentrerExpendedArgs("/usr/bin/test", "-d", directory, "-a", "-w", directory) +// EnsureFileDeviceDirectory creates directory (and any missing parents) on the +// host so the backing file can be allocated into it. The agent runs in PID 1's +// mount namespace, so this is `mkdir -p` against the node's root filesystem. +// `mkdir -p` is idempotent (no error if the directory already exists) and fails +// only when the path is genuinely unusable — a read-only filesystem, or a +// non-directory component along the way. +func (commands) EnsureFileDeviceDirectory(ctx context.Context, directory string) (string, error) { + args := nsentrerExpendedArgs("/bin/mkdir", "-p", directory) cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("directory %q is not an existing writable directory on the node: %w, stderr: %s", directory, err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to create directory %q on the node: %w, stderr: %s", directory, err, stderr.String()) } return cmd.String(), nil } From 5d4cd78da0cea183e88036eb5bbe47e299c2ef5c Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sat, 27 Jun 2026 18:12:03 +0300 Subject: [PATCH 18/32] fix(agent): propagate fileDevices in LVGSet and resolve aliased loop PVs on extend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two blocking issues found in review of the file-backed loop devices feature: 1. LVMVolumeGroupSet dropped spec.fileDevices. configureLVGBySet and updateLVMVolumeGroupByConfiguredFromSet copied every template field except FileDevices, so a file-only Set template (no blockDeviceSelector) was admitted by the relaxed Set CEL rule but produced child LVGs with neither selector nor fileDevices — rejected by the child LVMVolumeGroup's own CEL rule. Copy FileDevices on both the create and update paths. 2. extendFileDevicesIfNeeded re-added an already-attached loop PV. The "already a PV" check matched provisionFileDevices' canonical /dev/loopN against pvs[].PVName literally, but lvm.static (no udev integration) frequently reports a managed loop PV under a /dev/disk/by-id or /dev/block/MAJ:MIN alias — the same aliasing the discoverer already resolves. The literal miss handed the existing loop to pvcreate/vgextend again, failing pvcreate ("already a PV") and wedging the VGConfigurationApplied condition on every update reconcile. Resolve alias-form PV names via a canonical-path resolver (injectable for tests) before deciding a loop is new. Adds unit tests for both fixes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/reconciler.go | 50 ++++++++++- .../controller/lvg/reconciler_test.go | 36 ++++++++ .../lvm_volume_group_set_watcher.go | 2 + .../lvm_volume_group_set_watcher_test.go | 84 +++++++++++++++++++ 4 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 images/controller/pkg/controller/lvm_volume_group_set_watcher_test.go diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 77026f302..70d10617a 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -79,6 +79,11 @@ type Reconciler struct { sdsCache *cache.Cache cfg ReconcilerConfig commands utils.Commands + // resolver maps a /dev/* PV name to its canonical block device. It is + // only used to recognise a managed loop PV that lvm.static reported + // under a /dev/disk/by-id or /dev/block/MAJ:MIN alias. Defaults to + // utils.HostNsenterCanonicalResolver; overridable in tests. + resolver utils.CanonicalPathResolver } type ReconcilerConfig struct { @@ -111,6 +116,7 @@ func NewReconciler( sdsCache: sdsCache, cfg: cfg, commands: commands, + resolver: utils.HostNsenterCanonicalResolver, } } @@ -1334,11 +1340,25 @@ func (r *Reconciler) extendFileDevicesIfNeeded( pvsMap[pv.PVName] = struct{}{} } + // provisionFileDevices always returns canonical /dev/loopN names, but + // lvm.static has no udev integration and frequently reports a managed + // loop PV under a /dev/disk/by-id or /dev/block/MAJ:MIN alias instead + // (the same aliasing the discoverer resolves). A literal name match + // would therefore miss an already-attached loop PV and wrongly hand it + // to pvcreate/vgextend again — pvcreate then fails because the device + // is already a PV, wedging the VGConfigurationApplied condition on + // every reconcile. Resolve alias-form PV names before deciding a loop + // is new. pvsToExtend := make([]string, 0, len(loopPaths)) for _, loop := range loopPaths { - if _, exist := pvsMap[loop]; !exist { - pvsToExtend = append(pvsToExtend, loop) + if _, exist := pvsMap[loop]; exist { + continue + } + if r.loopAlreadyRegisteredAsPV(ctx, loop, pvs) { + r.log.Debug(fmt.Sprintf("[extendFileDevicesIfNeeded] loop %s is already a PV of VG %s under an alias; skipping", loop, vg.VGName)) + continue } + pvsToExtend = append(pvsToExtend, loop) } if len(pvsToExtend) == 0 { @@ -1363,6 +1383,32 @@ func (r *Reconciler) extendFileDevicesIfNeeded( return nil } +// loopAlreadyRegisteredAsPV reports whether the canonical loop device is +// already present in pvs under an alias name (e.g. /dev/disk/by-id/... or +// /dev/block/MAJ:MIN). It only resolves alias-form PV names, so the common +// case (lvm reports the canonical /dev/loopN) costs no extra host command. +// A resolver failure is treated as "not a match" and logged: it is safer +// to skip a still-attached loop here (the next reconcile retries) than to +// risk a duplicate pvcreate, and FilterForeignPVs already keeps such PVs. +func (r *Reconciler) loopAlreadyRegisteredAsPV(ctx context.Context, loop string, pvs []internal.PVData) bool { + for _, pv := range pvs { + if !strings.HasPrefix(pv.PVName, "/dev/disk/") && !strings.HasPrefix(pv.PVName, "/dev/block/") { + continue + } + resolved, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { + return r.resolver(ctx, pv.PVName) + }) + if err != nil { + r.log.Warning(fmt.Sprintf("[loopAlreadyRegisteredAsPV] unable to resolve canonical path for PV %s: %v", pv.PVName, err)) + continue + } + if resolved == loop { + return true + } + } + return false +} + func (r *Reconciler) tryGetVG(vgName string) (bool, internal.VGData) { vgs, _ := r.sdsCache.GetVGs() for _, vg := range vgs { diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index d2deb65a7..caec653a3 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1922,3 +1922,39 @@ func TestExtendFileDevicesIfNeeded_SkipsExistingPV(t *testing.T) { pvs := []internal.PVData{{PVName: "/dev/loop4"}} assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) } + +// lvm.static has no udev integration, so an already-attached managed loop +// PV is frequently reported under a /dev/disk/by-id (or /dev/block/MAJ:MIN) +// alias rather than /dev/loopN. provisionFileDevices always returns the +// canonical /dev/loopN, so a literal name match would miss it and wrongly +// re-pvcreate the device. extendFileDevicesIfNeeded must resolve the alias +// and skip the loop instead. +func TestExtendFileDevicesIfNeeded_SkipsExistingPVUnderAlias(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + // Resolve the alias PV name to the canonical loop device. + r.resolver = func(_ context.Context, path string) (string, error) { + if path == "/dev/disk/by-id/lvm-pv-uuid-xyz" { + return "/dev/loop4", nil + } + return path, nil + } + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + vg := internal.VGData{VGName: "vg-test"} + + // Loop is already attached; the cache reports it only under an alias. + // No CreatePV/ExtendVG must be issued. + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "/dev/loop4", nil) + + pvs := []internal.PVData{{PVName: "/dev/disk/by-id/lvm-pv-uuid-xyz"}} + assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) +} diff --git a/images/controller/pkg/controller/lvm_volume_group_set_watcher.go b/images/controller/pkg/controller/lvm_volume_group_set_watcher.go index 5f44d543b..e934c5f69 100644 --- a/images/controller/pkg/controller/lvm_volume_group_set_watcher.go +++ b/images/controller/pkg/controller/lvm_volume_group_set_watcher.go @@ -250,6 +250,7 @@ func matchConfiguredLVGWithExistingOne(lvg *v1alpha1.LVMVolumeGroup, lvgs map[st func updateLVMVolumeGroupByConfiguredFromSet(ctx context.Context, cl client.Client, existing, configured *v1alpha1.LVMVolumeGroup) error { existing.Spec.ThinPools = configured.Spec.ThinPools existing.Spec.BlockDeviceSelector = configured.Spec.BlockDeviceSelector + existing.Spec.FileDevices = configured.Spec.FileDevices return cl.Update(ctx, existing) } @@ -293,6 +294,7 @@ func configureLVGBySet(lvgSet *v1alpha1.LVMVolumeGroupSet, node v1.Node) *v1alph BlockDeviceSelector: lvgSet.Spec.LVGTemplate.BlockDeviceSelector, ThinPools: lvgSet.Spec.LVGTemplate.ThinPools, Type: lvgSet.Spec.LVGTemplate.Type, + FileDevices: lvgSet.Spec.LVGTemplate.FileDevices, Local: v1alpha1.LVMVolumeGroupLocalSpec{ NodeName: node.Name, }, diff --git a/images/controller/pkg/controller/lvm_volume_group_set_watcher_test.go b/images/controller/pkg/controller/lvm_volume_group_set_watcher_test.go new file mode 100644 index 000000000..9d215d5a0 --- /dev/null +++ b/images/controller/pkg/controller/lvm_volume_group_set_watcher_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" +) + +// A file-only LVMVolumeGroupSet template carries no blockDeviceSelector; +// the child LVMVolumeGroup is admitted only if configureLVGBySet copies +// fileDevices over. Without it the child has neither selector nor +// fileDevices and is rejected by its own CEL rule. +func TestConfigureLVGBySet_CopiesFileDevices(t *testing.T) { + fileDevices := []v1alpha1.LVMVolumeGroupFileDeviceSpec{ + {Directory: "/data/lvm-backing", Size: resource.MustParse("50Gi")}, + } + lvgSet := &v1alpha1.LVMVolumeGroupSet{ + ObjectMeta: metav1.ObjectMeta{Name: "set-a"}, + Spec: v1alpha1.LVMVolumeGroupSetSpec{ + LVGTemplate: v1alpha1.LVMVolumeGroupTemplate{ + ActualVGNameOnTheNode: "vg-file", + Type: "Local", + FileDevices: fileDevices, + }, + }, + } + + lvg := configureLVGBySet(lvgSet, v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-0"}}) + + assert.Equal(t, fileDevices, lvg.Spec.FileDevices) + assert.Nil(t, lvg.Spec.BlockDeviceSelector) + assert.Equal(t, "node-0", lvg.Spec.Local.NodeName) +} + +// Editing fileDevices on the set template (the documented "add an entry to +// grow capacity" flow) must propagate to already-created child LVGs. +func TestUpdateLVMVolumeGroupByConfiguredFromSet_PropagatesFileDevices(t *testing.T) { + configured := &v1alpha1.LVMVolumeGroup{ + Spec: v1alpha1.LVMVolumeGroupSpec{ + FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{ + {Directory: "/data", Size: resource.MustParse("10Gi")}, + {Directory: "/data", Size: resource.MustParse("20Gi")}, + }, + }, + } + existing := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "set-a-0"}, + Spec: v1alpha1.LVMVolumeGroupSpec{ + FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{ + {Directory: "/data", Size: resource.MustParse("10Gi")}, + }, + }, + } + + ctx := context.Background() + cl := NewFakeClient() + assert.NoError(t, cl.Create(ctx, existing)) + + err := updateLVMVolumeGroupByConfiguredFromSet(ctx, cl, existing, configured) + assert.NoError(t, err) + assert.Equal(t, configured.Spec.FileDevices, existing.Spec.FileDevices) +} From 6dff8e54f3b8efe0d2259db75dc263ca66a0a051 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sat, 27 Jun 2026 19:44:01 +0300 Subject: [PATCH 19/32] fix(agent): include fileDevices in thin-pool sizing and record canonical loop Two correctness fixes for the file-backed loop device feature: 1. Thin-pool sizing on the create path used countVGSizeByBlockDevices, which sums only block devices. A file-only VG therefore looked zero-sized: percentage thin pools collapsed to 0 and absolute-sized thin pools fell into the full-VG-space branch, taking the whole VG instead of the requested size. Add a countVGSizeByFileDevices helper and include spec.fileDevices capacity, mirroring validateLVGForCreateFunc. 2. The discoverer's alias-probe path recorded the alias (/dev/disk/by-id or /dev/block/MAJ:MIN) as status.loopDevice. On the next reconcile where lvm reported the same PV as /dev/loopN, hasStatusNodesDiff saw a change and rewrote status, flip-flopping forever. Re-resolve the canonical /dev/loopN from the backing file before writing status. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/discoverer.go | 19 +++++++++++++- .../internal/controller/lvg/reconciler.go | 19 ++++++++++++++ .../controller/lvg/reconciler_test.go | 26 +++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/images/agent/internal/controller/lvg/discoverer.go b/images/agent/internal/controller/lvg/discoverer.go index efef7535e..6fd436278 100644 --- a/images/agent/internal/controller/lvg/discoverer.go +++ b/images/agent/internal/controller/lvg/discoverer.go @@ -684,9 +684,26 @@ func (d *Discoverer) buildFileDeviceFromLoopPV(ctx context.Context, pv internal. return nil } + // Status must record the canonical /dev/loopN device, never an alias. + // In the probe path pv.PVName is a /dev/disk/by-id or /dev/block alias; + // recording it verbatim makes status.loopDevice flip-flop between the + // alias and /dev/loopN across reconciles (whichever lvm happens to + // report), churning the resource with no-op updates. Re-resolve the + // canonical loop from the backing file; keep pv.PVName only if that fails. + loopDevice := pv.PVName + if !strings.HasPrefix(loopDevice, "/dev/loop") { + findCmd, canonical, ferr := d.commands.FindLoopDeviceByFile(ctx, backingFile) + d.log.Debug(findCmd) + if ferr != nil { + d.log.Debug(fmt.Sprintf("[buildFileDeviceFromLoopPV] unable to resolve canonical loop for %s: %v; keeping %s", backingFile, ferr, pv.PVName)) + } else if canonical != "" { + loopDevice = canonical + } + } + return &internal.LVMVGFileDevice{ FilePath: backingFile, - LoopDevice: pv.PVName, + LoopDevice: loopDevice, Size: *resource.NewQuantity(pv.PVSize.Value(), resource.BinarySI), PVUUID: pv.PVUuid, } diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 70d10617a..05390f0ba 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -597,7 +597,14 @@ func (r *Reconciler) reconcileLVGCreateFunc( extentForThinPools := extentSizeForThinPoolAlign(lvg, vgAfterCreate) for _, tp := range lvg.Spec.ThinPools { + // vgSize must account for file-backed PVs too: a file-only VG has + // no block devices, so countVGSizeByBlockDevices alone returns 0, + // which collapses every percentage thin-pool to 0 and forces every + // absolute-sized thin-pool into the full-VG-space branch (taking the + // whole VG instead of the requested size). Add the spec.fileDevices + // capacity, mirroring how validateLVGForCreateFunc computes totalVGSize. vgSize := countVGSizeByBlockDevices(blockDevices) + vgSize.Add(countVGSizeByFileDevices(lvg)) tpRequestedSize, err := utils.GetRequestedSizeFromString(tp.Size, vgSize) if err != nil { r.log.Error(err, fmt.Sprintf("[reconcileLVGCreateFunc] unable to get thin-pool %s requested size of the LVMVolumeGroup %s", tp.Name, lvg.Name)) @@ -2112,3 +2119,15 @@ func countVGSizeByBlockDevices(blockDevices map[string]v1alpha1.BlockDevice) res } return *resource.NewQuantity(totalVGSize, resource.BinarySI) } + +// countVGSizeByFileDevices sums the capacity contributed by spec.fileDevices. +// File-backed PVs are part of the VG just like block devices, so thin-pool +// sizing on the create path must include them; otherwise a file-only VG is +// treated as zero-sized. +func countVGSizeByFileDevices(lvg *v1alpha1.LVMVolumeGroup) resource.Quantity { + var totalSize int64 + for _, fd := range lvg.Spec.FileDevices { + totalSize += fd.Size.Value() + } + return *resource.NewQuantity(totalSize, resource.BinarySI) +} diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index caec653a3..68a8d5ff6 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1510,6 +1510,32 @@ func setupReconciler() *Reconciler { ReconcilerConfig{NodeName: "test_node", CmdDeadlineDuration: 30 * time.Second}) } +// countVGSizeByFileDevices must sum spec.fileDevices capacity so the +// create-path thin-pool sizing does not treat a file-only VG as zero-sized +// (which would collapse percentage pools to 0 and force absolute pools into +// the full-VG-space branch). +func TestCountVGSizeByFileDevices(t *testing.T) { + t.Run("no_file_devices", func(t *testing.T) { + lvg := &v1alpha1.LVMVolumeGroup{} + got := countVGSizeByFileDevices(lvg) + assert.Equal(t, int64(0), got.Value()) + }) + + t.Run("sums_all_entries", func(t *testing.T) { + lvg := &v1alpha1.LVMVolumeGroup{ + Spec: v1alpha1.LVMVolumeGroupSpec{ + FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{ + {Directory: "/data", Size: resource.MustParse("10Gi")}, + {Directory: "/data", Size: resource.MustParse("20Gi")}, + }, + }, + } + want := resource.MustParse("30Gi") + got := countVGSizeByFileDevices(lvg) + assert.Equal(t, want.Value(), got.Value()) + }) +} + func TestValidateFileDevice(t *testing.T) { ctx := context.Background() From 23487e98f9573040613a8cd396d73e47783d8de8 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sat, 27 Jun 2026 20:22:32 +0300 Subject: [PATCH 20/32] fix(agent): harden file-device provisioning, reattach, and discovery Address review findings on the spec.fileDevices feature: - extendFileDevicesIfNeeded now rolls back the loop device + backing file it just provisioned if a later pvcreate/vgextend fails, instead of leaking them on the node (createVGComplex already had this). The rollback is scoped to only the devices this call created, so existing healthy file devices of the same VG are never touched. provisionFileDevices returns the newly-created artifacts to support it. - reattachFileDevicesAtStartup also derives backing-file paths from spec.fileDevices (deterministic BuildFileDevicePath), not just status. A node that rebooted before the discoverer wrote status.nodes[].fileDevices can now recover its loop mappings before ActivateVGs. ReattachFileDevices never creates files, so a not-yet-provisioned entry is a harmless miss. - Correct stale docs/log that still claimed loop devices are filtered out: removing loop from LVMGlobalFilter/ForeignDeviceBasePrefixes is intentional, and ownership is enforced downstream via the backing-file owner pattern + VG tag. Fixed the scanner drop log, FilterForeignPVs / IsForeignDeviceBase docstrings. - convertLVMVGNodes: fast path (single pass, no node-name set) for the common no-file-devices case on the discover hot path. - Document that status.fileDevices[].size is the PV size (LVM metadata overhead), mirroring status.nodes[].devices[].pvSize; align the CRD description and add cross-references between the two loop-PV alias resolution helpers (reconciler readlink vs discoverer losetup). Co-Authored-By: Claude Opus 4.8 (1M context) --- crds/lvmvolumegroup.yaml | 5 +- images/agent/cmd/main.go | 44 +++++++++-- .../internal/controller/lvg/discoverer.go | 31 +++++++- .../internal/controller/lvg/reconciler.go | 79 ++++++++++++++++--- .../controller/lvg/reconciler_test.go | 38 ++++++++- images/agent/internal/scanner/scanner.go | 17 ++-- images/agent/internal/utils/devicefilter.go | 17 +++- 7 files changed, 199 insertions(+), 32 deletions(-) diff --git a/crds/lvmvolumegroup.yaml b/crds/lvmvolumegroup.yaml index f12dcd5dc..c4b3a97f0 100644 --- a/crds/lvmvolumegroup.yaml +++ b/crds/lvmvolumegroup.yaml @@ -375,7 +375,10 @@ spec: size: type: string description: | - Backing file size. + Size of the Physical Volume created on the backing file. + This is slightly smaller than the requested `spec.fileDevices[].size` + because LVM reserves space for PV metadata (the same way + `status.nodes[].devices[].pvSize` differs from the raw device size). pvUUID: type: string description: | diff --git a/images/agent/cmd/main.go b/images/agent/cmd/main.go index a0e2736b6..6952b40e6 100644 --- a/images/agent/cmd/main.go +++ b/images/agent/cmd/main.go @@ -348,25 +348,53 @@ func reattachFileDevicesAtStartup(ctx context.Context, log logger.Logger, mgr ma if lvg.Spec.Local.NodeName != nodeName { continue } + + // Collect backing-file paths from BOTH the observed status and the + // desired spec. Status alone is not enough: a node can reboot after + // the LVG was provisioned but before the discoverer wrote + // status.nodes[].fileDevices, in which case reattach would find + // nothing and ActivateVGs would bring the VG up missing its loop PV. + // The spec path is deterministic (BuildFileDevicePath), so it lets + // reattach recover even without a status entry. ReattachFileDevices + // never creates files (only losetup --find on an existing backing + // file), so a spec entry whose file does not exist yet is a harmless + // best-effort miss the LVG reconciler provisions on its next pass. + seen := make(map[string]struct{}) + fds := make([]utils.FileDeviceStatus, 0) for _, node := range lvg.Status.Nodes { if node.Name != nodeName { continue } - if len(node.FileDevices) == 0 { - continue - } - fds := make([]utils.FileDeviceStatus, 0, len(node.FileDevices)) for _, fd := range node.FileDevices { + if fd.FilePath == "" { + continue + } + if _, ok := seen[fd.FilePath]; ok { + continue + } + seen[fd.FilePath] = struct{}{} fds = append(fds, utils.FileDeviceStatus{ FilePath: fd.FilePath, LoopDevice: fd.LoopDevice, }) } - items = append(items, utils.LVGWithFileDevices{ - LVGName: lvg.Name, - FileDevices: fds, - }) } + for _, fd := range lvg.Spec.FileDevices { + path := utils.BuildFileDevicePath(fd.Directory, lvg.Name, fd.Size) + if _, ok := seen[path]; ok { + continue + } + seen[path] = struct{}{} + fds = append(fds, utils.FileDeviceStatus{FilePath: path}) + } + + if len(fds) == 0 { + continue + } + items = append(items, utils.LVGWithFileDevices{ + LVGName: lvg.Name, + FileDevices: fds, + }) } if len(items) == 0 { diff --git a/images/agent/internal/controller/lvg/discoverer.go b/images/agent/internal/controller/lvg/discoverer.go index 6fd436278..539bdbf6b 100644 --- a/images/agent/internal/controller/lvg/discoverer.go +++ b/images/agent/internal/controller/lvg/discoverer.go @@ -648,6 +648,13 @@ func (d *Discoverer) configureCandidateNodeDevices(ctx context.Context, pvs map[ // that mode "not a loop" / "not ours" outcomes are expected and logged at // Debug instead of Warning, so an ordinary not-yet-registered BlockDevice // does not produce misleading warnings. +// +// NOTE: this is one of two places that canonicalize an alias-reported loop +// PV. Here we resolve backing-file → canonical /dev/loopN via losetup (we +// have the backing file and need the loop). The reconciler does the inverse +// in Reconciler.loopAlreadyRegisteredAsPV (canonical loop is known, alias PV +// names are resolved via readlink). They use different methods because their +// inputs differ; keep their ownership/aliasing assumptions in sync. func (d *Discoverer) buildFileDeviceFromLoopPV(ctx context.Context, pv internal.PVData, expectedLVGName string, probe bool) *internal.LVMVGFileDevice { if expectedLVGName == "" { d.log.Warning(fmt.Sprintf("[buildFileDeviceFromLoopPV] VG of loop PV %s carries no %s tag; skipping file-device discovery", pv.PVName, internal.LVMVolumeGroupTag)) @@ -703,6 +710,13 @@ func (d *Discoverer) buildFileDeviceFromLoopPV(ctx context.Context, pv internal. return &internal.LVMVGFileDevice{ FilePath: backingFile, + // Size is the PV size, not the raw backing-file size: it is what lvm + // reports for this device and is always slightly smaller than the + // requested spec.fileDevices[].size (LVM metadata overhead). This + // mirrors how block devices expose status...devices[].pvSize, and keeps + // the value self-consistent across reconciles (hasStatusNodesDiff + // compares PV size against PV size, so it never churns). Callers that + // need the requested size must read spec, not status. LoopDevice: loopDevice, Size: *resource.NewQuantity(pv.PVSize.Value(), resource.BinarySI), PVUUID: pv.PVUuid, @@ -1088,7 +1102,22 @@ func configureBlockDeviceSelector(candidate internal.LVMVolumeGroupCandidate) *m } func convertLVMVGNodes(nodes map[string][]internal.LVMVGDevice, fileNodes map[string][]internal.LVMVGFileDevice) []v1alpha1.LVMVolumeGroupNode { - allNodeNames := make(map[string]struct{}, len(nodes)) + // Fast path for the overwhelmingly common case of no file devices + // (every pure block-device LVG): a single pass over nodes, no extra + // node-name set to allocate. This runs per LVG on every discover + // reconcile via hasLVMVolumeGroupDiff, so the allocation matters. + if len(fileNodes) == 0 { + lvmvgNodes := make([]v1alpha1.LVMVolumeGroupNode, 0, len(nodes)) + for nodeName, nodeDevices := range nodes { + lvmvgNodes = append(lvmvgNodes, v1alpha1.LVMVolumeGroupNode{ + Devices: convertLVMVGDevices(nodeDevices), + Name: nodeName, + }) + } + return lvmvgNodes + } + + allNodeNames := make(map[string]struct{}, len(nodes)+len(fileNodes)) for n := range nodes { allNodeNames[n] = struct{}{} } diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 05390f0ba..7cba3620c 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -1332,16 +1332,51 @@ func (r *Reconciler) extendFileDevicesIfNeeded( lvg *v1alpha1.LVMVolumeGroup, vg internal.VGData, pvs []internal.PVData, -) error { +) (retErr error) { if len(lvg.Spec.FileDevices) == 0 { return nil } - loopPaths, err := r.provisionFileDevices(ctx, lvg) + loopPaths, provisioned, err := r.provisionFileDevices(ctx, lvg) if err != nil { return fmt.Errorf("file device provisioning failed: %w", err) } + // provisionFileDevices only rolls back artifacts it created within its own + // call; once it returns, the new loops/files survive. Unlike createVGComplex + // (which can roll back the whole VG via cleanupFileDevices because the VG is + // brand new), here the VG already holds healthy file devices we must NOT + // touch. So if a later step fails (condition update, vgextend), detach and + // remove ONLY the devices this provision call just created — they are not + // yet PVs in the VG. Otherwise a failed extend leaks a loop device and a + // preallocated file on the node (and orphans them entirely if the admin + // then removes the failing spec entry). Runs on a detached context because + // the failure is frequently the reconcile ctx being cancelled. + defer func() { + if retErr == nil || len(provisioned) == 0 { + return + } + rollbackCtx := context.Background() + if r.cfg.CmdDeadlineDuration > 0 { + var cancel context.CancelFunc + rollbackCtx, cancel = context.WithTimeout(rollbackCtx, r.cfg.CmdDeadlineDuration) + defer cancel() + } + for i := len(provisioned) - 1; i >= 0; i-- { + p := provisioned[i] + if p.loopDev != "" { + if cmd, derr := r.commands.DetachLoopDevice(rollbackCtx, p.loopDev); derr != nil { + r.log.Warning(fmt.Sprintf("[extendFileDevicesIfNeeded][rollback] unable to detach %s: %v (cmd: %s)", p.loopDev, derr, cmd)) + } + } + if p.filePath != "" { + if cmd, rerr := r.commands.RemoveFileDevice(rollbackCtx, p.filePath); rerr != nil { + r.log.Warning(fmt.Sprintf("[extendFileDevicesIfNeeded][rollback] unable to remove %s: %v (cmd: %s)", p.filePath, rerr, cmd)) + } + } + } + }() + pvsMap := make(map[string]struct{}, len(pvs)) for _, pv := range pvs { pvsMap[pv.PVName] = struct{}{} @@ -1397,6 +1432,13 @@ func (r *Reconciler) extendFileDevicesIfNeeded( // A resolver failure is treated as "not a match" and logged: it is safer // to skip a still-attached loop here (the next reconcile retries) than to // risk a duplicate pvcreate, and FilterForeignPVs already keeps such PVs. +// +// NOTE: this is one of two places that canonicalize an alias-reported loop +// PV. Here the canonical /dev/loopN is known and alias PV names are resolved +// via readlink (r.resolver). The discoverer does the inverse in +// Discoverer.buildFileDeviceFromLoopPV (backing-file → canonical loop via +// losetup). They use different methods because their inputs differ; keep +// their ownership/aliasing assumptions in sync. func (r *Reconciler) loopAlreadyRegisteredAsPV(ctx context.Context, loop string, pvs []internal.PVData) bool { for _, pv := range pvs { if !strings.HasPrefix(pv.PVName, "/dev/disk/") && !strings.HasPrefix(pv.PVName, "/dev/block/") { @@ -1583,9 +1625,19 @@ func (r *Reconciler) triggerUdevForPaths(parent context.Context, paths []string) // that pre-existed (e.g. left behind by an earlier successful step) // are preserved on purpose: they will be picked up by the next // reconcile as "already attached". -func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup) (loopPaths []string, retErr error) { +// provisionedFileDevice records a backing file + loop device that +// provisionFileDevices newly created (fallocate + losetup) within a single +// call. Reused, already-attached devices are NOT included, so a caller can +// roll back exactly what this call added if a later step fails, without +// touching pre-existing healthy file devices of the same LVG. +type provisionedFileDevice struct { + filePath string + loopDev string +} + +func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup) (loopPaths []string, provisioned []provisionedFileDevice, retErr error) { if len(lvg.Spec.FileDevices) == 0 { - return nil, nil + return nil, nil, nil } // Track resources we (and only we) just created so we can undo them @@ -1629,10 +1681,11 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM }() loopPaths = make([]string, 0, len(lvg.Spec.FileDevices)) + provisioned = make([]provisionedFileDevice, 0, len(lvg.Spec.FileDevices)) seenLoops := make(map[string]struct{}, len(lvg.Spec.FileDevices)) for _, fd := range lvg.Spec.FileDevices { if err := ctx.Err(); err != nil { - return nil, err + return nil, nil, err } filePath := utils.BuildFileDevicePath(fd.Directory, lvg.Name, fd.Size) @@ -1648,7 +1701,7 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM }) r.log.Debug(findRes.cmd) if err != nil { - return nil, fmt.Errorf("query loop for %s: %w", filePath, err) + return nil, nil, fmt.Errorf("query loop for %s: %w", filePath, err) } if findRes.loopDev != "" { r.log.Info(fmt.Sprintf("[provisionFileDevices] %s already attached to %s; reusing", filePath, findRes.loopDev)) @@ -1671,7 +1724,7 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM r.log.Debug(mkdirCmd) if err != nil { r.log.Error(err, fmt.Sprintf("[provisionFileDevices] unable to create directory %s", fd.Directory)) - return nil, err + return nil, nil, err } rb := rollback{filePath: filePath} @@ -1682,7 +1735,7 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM if err != nil { r.log.Error(err, fmt.Sprintf("[provisionFileDevices] unable to create file %s", filePath)) created = append(created, rb) - return nil, err + return nil, nil, err } rb.createdFile = true @@ -1698,11 +1751,12 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM if err != nil { r.log.Error(err, fmt.Sprintf("[provisionFileDevices] unable to setup loop device for %s", filePath)) created = append(created, rb) - return nil, err + return nil, nil, err } rb.loopDev = setupRes.loopDev rb.attachedLoopOK = true created = append(created, rb) + provisioned = append(provisioned, provisionedFileDevice{filePath: filePath, loopDev: setupRes.loopDev}) r.log.Info(fmt.Sprintf("[provisionFileDevices] file %s attached to %s", filePath, setupRes.loopDev)) if _, ok := seenLoops[setupRes.loopDev]; !ok { @@ -1710,7 +1764,7 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM loopPaths = append(loopPaths, setupRes.loopDev) } } - return loopPaths, nil + return loopPaths, provisioned, nil } // cleanupFileDevices detaches loop devices and removes backing files @@ -1845,7 +1899,10 @@ func (r *Reconciler) cleanupFileDevices(ctx context.Context, lvg *v1alpha1.LVMVo func (r *Reconciler) createVGComplex(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) (retErr error) { paths := extractPathsFromBlockDevices(nil, blockDevices) - loopPaths, err := r.provisionFileDevices(ctx, lvg) + // On a fresh create the VG holds no file devices yet, so the defer below + // rolls everything back via cleanupFileDevices; the per-call provisioned + // list is only needed for the scoped rollback in extendFileDevicesIfNeeded. + loopPaths, _, err := r.provisionFileDevices(ctx, lvg) if err != nil { return fmt.Errorf("file device provisioning failed: %w", err) } diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 68a8d5ff6..1d14d845d 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1677,7 +1677,7 @@ func TestProvisionFileDevices_ReusesExistingLoop(t *testing.T) { mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), want).Return("losetup -j ...", "/dev/loop7", nil) // CreateFileDevice / SetupLoopDevice MUST NOT be invoked. - got, err := r.provisionFileDevices(ctx, lvg) + got, _, err := r.provisionFileDevices(ctx, lvg) assert.NoError(t, err) assert.Equal(t, []string{"/dev/loop7"}, got) } @@ -1718,7 +1718,7 @@ func TestProvisionFileDevices_RollsBackOnPartialFailure(t *testing.T) { mc.EXPECT().RemoveFileDevice(gomock.Any(), pathA).Return("rm ...", nil), ) - _, err := r.provisionFileDevices(ctx, lvg) + _, _, err := r.provisionFileDevices(ctx, lvg) if assert.Error(t, err) { assert.Contains(t, err.Error(), "ENOSPC") } @@ -1984,3 +1984,37 @@ func TestExtendFileDevicesIfNeeded_SkipsExistingPVUnderAlias(t *testing.T) { pvs := []internal.PVData{{PVName: "/dev/disk/by-id/lvm-pv-uuid-xyz"}} assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) } + +// When the extend step (pvcreate/vgextend) fails after a new file device was +// freshly provisioned, the loop device and backing file created in THIS call +// must be rolled back so a failed extend does not leak them on the node (and +// orphan them entirely if the admin then removes the failing spec entry). +func TestExtendFileDevicesIfNeeded_RollsBackProvisionedOnExtendFailure(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + vg := internal.VGData{VGName: "vg-test"} + + gomock.InOrder( + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "", nil), + mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fd.Directory).Return("mkdir -p ...", nil), + mc.EXPECT().CreateFileDevice(gomock.Any(), path, fd.Size.Value()).Return("fallocate ...", nil), + mc.EXPECT().SetupLoopDevice(gomock.Any(), path).Return("losetup --find ...", "/dev/loop4", nil), + // Extend fails on pvcreate... + mc.EXPECT().CreatePV("/dev/loop4").Return("pvcreate ...", errors.New("pvcreate refused")), + // ...so the just-provisioned loop + file are rolled back. + mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop4").Return("losetup -d ...", nil), + mc.EXPECT().RemoveFileDevice(gomock.Any(), path).Return("rm ...", nil), + ) + + assert.Error(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, nil)) +} diff --git a/images/agent/internal/scanner/scanner.go b/images/agent/internal/scanner/scanner.go index 7d0d4f602..a721279ae 100644 --- a/images/agent/internal/scanner/scanner.go +++ b/images/agent/internal/scanner/scanner.go @@ -257,11 +257,16 @@ func (s *scanner) fillTheCache(ctx context.Context, log logger.Logger, cache *ca // lvm.static bundled in /opt/deckhouse/sds/bin is built without udev // integration and so reports PVs found on any block device it sees in - // /dev, including Ceph RBD images and loopback-mounted guest VM disks - // that happen to carry LVM signatures from nested LVM (see ADR / PR - // description). Filter those out before feeding the cache so that the - // LVG/BD reconcile logic never sees duplicate VG names produced by - // foreign storage layers. + // /dev, including Ceph RBD/DRBD/NBD images that happen to carry LVM + // signatures from nested LVM (see ADR / PR description). Filter those out + // before feeding the cache so that the LVG/BD reconcile logic never sees + // duplicate VG names produced by foreign storage layers. + // + // Loop devices are intentionally NOT filtered here: the agent manages + // file-backed loop devices as LVM PVs (spec.fileDevices). Unmanaged loop + // PVs are tolerated because the discoverer only adopts a loop PV whose + // backing file matches its owner pattern (utils.IsManagedFileDevicePath) + // inside a VG tagged storage.deckhouse.io/lvmVolumeGroupName. // // cfg.CmdDeadlineDuration bounds every per-PV nsenter+readlink call: // a hung resolver on a single foreign device cannot block the entire @@ -270,7 +275,7 @@ func (s *scanner) fillTheCache(ctx context.Context, log logger.Logger, cache *ca beforePV := len(pvs) pvs = utils.FilterForeignPVs(ctx, log, nil, pvs, cfg.CmdDeadlineDuration) if dropped := beforePV - len(pvs); dropped > 0 { - log.Info(fmt.Sprintf("[fillTheCache] dropped %d foreign PV(s) backed by rbd/drbd/nbd/loop devices", dropped)) + log.Info(fmt.Sprintf("[fillTheCache] dropped %d foreign PV(s) backed by rbd/drbd/nbd devices", dropped)) beforeVG := len(vgs) vgs = utils.FilterVGsByPresentPVs(vgs, pvs) beforeLV := len(lvs) diff --git a/images/agent/internal/utils/devicefilter.go b/images/agent/internal/utils/devicefilter.go index c77690dda..b62dcc8ed 100644 --- a/images/agent/internal/utils/devicefilter.go +++ b/images/agent/internal/utils/devicefilter.go @@ -64,8 +64,9 @@ func HostNsenterCanonicalResolver(ctx context.Context, devPath string) (string, // IsForeignDeviceBase reports whether the given canonical basename // belongs to a storage layer the agent must ignore. The check is a // strict prefix match against internal.ForeignDeviceBasePrefixes so it -// catches partitions of foreign devices too (e.g. "rbd14p1", "loop970", -// "drbd0"). +// catches partitions of foreign devices too (e.g. "rbd14p1", "nbd0p1", +// "drbd0"). Loop devices are deliberately absent from the prefix list — +// the agent manages file-backed loop devices as LVM PVs. func IsForeignDeviceBase(base string) bool { for _, prefix := range internal.ForeignDeviceBasePrefixes { if strings.HasPrefix(base, prefix) { @@ -77,7 +78,17 @@ func IsForeignDeviceBase(base string) bool { // FilterForeignPVs returns a copy of pvs with PVs whose underlying // canonical device belongs to a foreign storage layer (Ceph RBD, DRBD, -// NBD, loopback) removed. +// NBD) removed. +// +// Loop devices are intentionally NOT dropped here: the agent manages +// file-backed loop devices as LVM PVs (spec.fileDevices), so a blanket +// loop reject would hide its own managed PVs. Ownership of a loop PV is +// instead established later, in the discoverer, via the backing-file owner +// pattern (IsManagedFileDevicePath) gated on the VG's +// storage.deckhouse.io/lvmVolumeGroupName tag. A consequence is that an +// unmanaged loop PV (e.g. nested LVM inside a file-backed guest VM disk) +// now reaches the cache; the duplicate-VG-name guard (findDuplicateVGNames) +// and the tag filter keep it from being mistaken for a managed VG. // // lvm.static bundled with the agent has no udev integration and so it // enumerates devices via /dev/block/MAJOR:MINOR and /dev/disk/by-id/ From 2a18c59f6f4c485b9125f0cb94d0ed755e07d500 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sat, 27 Jun 2026 20:39:17 +0300 Subject: [PATCH 21/32] fix(agent): count new file devices in update thin-pool sizing and harden loop alias resolution Two correctness fixes in the file-backed loop device flow: - validateLVGForUpdateFunc ignored the capacity of newly-added spec.fileDevices entries when validating thin-pool sizes. The create path was already fixed via countVGSizeByFileDevices, but the update path computed newTotalVGSize as vg.VGSize + additionBlockDeviceSpace only. A combined "append a fileDevices entry + grow thin-pools" edit was therefore validated against the pre-extend VG size and wrongly rejected (or forced into the full-VG-space branch). Add additionFileDeviceSpace, mirroring additionBlockDeviceSpace, counting only spec entries not yet reflected in status.nodes[].fileDevices so already-provisioned devices are not double-counted. - loopAlreadyRegisteredAsPV treated a resolver failure on an alias PV as "not a match", contradicting its own comment. That handed a possibly already-registered loop to pvcreate, which fails ("already a PV") and wedges the VGConfigurationApplied condition until the next reconcile. Be conservative: on resolver failure, report the loop as already registered so the extend is skipped this round and retried. Add regression tests for both paths. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/reconciler.go | 45 ++++++-- .../controller/lvg/reconciler_test.go | 100 ++++++++++++++++++ 2 files changed, 139 insertions(+), 6 deletions(-) diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 7cba3620c..eeb475d76 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -998,6 +998,29 @@ func (r *Reconciler) validateLVGForUpdateFunc( r.validateFileDevices(ctx, lvg, &reason, nil) + // additionFileDeviceSpace mirrors additionBlockDeviceSpace for file + // devices: it accounts for spec.fileDevices entries that are not yet PVs + // in the VG (i.e. not yet reflected in status.nodes[].fileDevices). Without + // it, a combined "append a fileDevices entry + grow a thin-pool" edit would + // be validated against the current VG size, wrongly rejecting a valid + // request or forcing an absolute-sized thin-pool into the full-VG-space + // branch — the same defect the create path avoids via countVGSizeByFileDevices. + var additionFileDeviceSpace int64 + if len(lvg.Spec.FileDevices) > 0 { + existingFilePaths := make(map[string]struct{}) + for _, n := range lvg.Status.Nodes { + for _, fd := range n.FileDevices { + existingFilePaths[fd.FilePath] = struct{}{} + } + } + for _, fd := range lvg.Spec.FileDevices { + path := utils.BuildFileDevicePath(fd.Directory, lvg.Name, fd.Size) + if _, ok := existingFilePaths[path]; !ok { + additionFileDeviceSpace += fd.Size.Value() + } + } + } + if lvg.Spec.ThinPools != nil { r.log.Debug(fmt.Sprintf("[validateLVGForUpdateFunc] the LVMVolumeGroup %s has thin-pools. Validate them", lvg.Name)) actualThinPools := make(map[string]internal.LVData, len(lvg.Spec.ThinPools)) @@ -1025,7 +1048,7 @@ func (r *Reconciler) validateLVGForUpdateFunc( return false, reason.String() } - newTotalVGSize := resource.NewQuantity(vg.VGSize.Value()+additionBlockDeviceSpace, resource.BinarySI) + newTotalVGSize := resource.NewQuantity(vg.VGSize.Value()+additionBlockDeviceSpace+additionFileDeviceSpace, resource.BinarySI) for _, specTp := range lvg.Spec.ThinPools { // might be a case when Thin-pool is already created, but is not shown in status tpRequestedSize, err := utils.GetRequestedSizeFromString(specTp.Size, *newTotalVGSize) @@ -1429,9 +1452,14 @@ func (r *Reconciler) extendFileDevicesIfNeeded( // already present in pvs under an alias name (e.g. /dev/disk/by-id/... or // /dev/block/MAJ:MIN). It only resolves alias-form PV names, so the common // case (lvm reports the canonical /dev/loopN) costs no extra host command. -// A resolver failure is treated as "not a match" and logged: it is safer -// to skip a still-attached loop here (the next reconcile retries) than to -// risk a duplicate pvcreate, and FilterForeignPVs already keeps such PVs. +// +// A resolver failure on an alias PV is treated conservatively as a match: +// we cannot rule out that the unresolved alias IS this loop already +// registered as a PV, and handing a possibly-already-registered loop to +// pvcreate fails ("already a PV") and wedges the VGConfigurationApplied +// condition. It is safer to skip the extend this round and let the next +// reconcile retry once the resolver recovers; provisionFileDevices keeps the +// loop attached idempotently in the meantime. // // NOTE: this is one of two places that canonicalize an alias-reported loop // PV. Here the canonical /dev/loopN is known and alias PV names are resolved @@ -1440,6 +1468,7 @@ func (r *Reconciler) extendFileDevicesIfNeeded( // losetup). They use different methods because their inputs differ; keep // their ownership/aliasing assumptions in sync. func (r *Reconciler) loopAlreadyRegisteredAsPV(ctx context.Context, loop string, pvs []internal.PVData) bool { + resolveFailed := false for _, pv := range pvs { if !strings.HasPrefix(pv.PVName, "/dev/disk/") && !strings.HasPrefix(pv.PVName, "/dev/block/") { continue @@ -1448,14 +1477,18 @@ func (r *Reconciler) loopAlreadyRegisteredAsPV(ctx context.Context, loop string, return r.resolver(ctx, pv.PVName) }) if err != nil { - r.log.Warning(fmt.Sprintf("[loopAlreadyRegisteredAsPV] unable to resolve canonical path for PV %s: %v", pv.PVName, err)) + r.log.Warning(fmt.Sprintf("[loopAlreadyRegisteredAsPV] unable to resolve canonical path for PV %s: %v; treating loop %s as already registered to avoid a duplicate pvcreate", pv.PVName, err, loop)) + resolveFailed = true continue } if resolved == loop { return true } } - return false + // No confirmed match. If at least one alias PV could not be resolved, + // stay on the safe side and report the loop as already registered so the + // caller skips a potentially duplicate pvcreate; the next reconcile retries. + return resolveFailed } func (r *Reconciler) tryGetVG(vgName string) (bool, internal.VGData) { diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 1d14d845d..ee924f2b9 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1985,6 +1985,39 @@ func TestExtendFileDevicesIfNeeded_SkipsExistingPVUnderAlias(t *testing.T) { assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) } +// When the canonical loop cannot be resolved from an alias PV (transient +// readlink/nsenter failure), extendFileDevicesIfNeeded must NOT hand the loop +// to pvcreate: the alias might already be this loop registered as a PV, and a +// duplicate pvcreate would wedge the VGConfigurationApplied condition. The +// loop is skipped this round and retried on the next reconcile. +func TestExtendFileDevicesIfNeeded_SkipsOnResolverFailure(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + // The resolver fails for the alias PV, so the loop's PV membership is + // undeterminable this round. + r.resolver = func(_ context.Context, _ string) (string, error) { + return "", errors.New("nsenter readlink timed out") + } + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + vg := internal.VGData{VGName: "vg-test"} + + // Loop is already attached. With the resolver failing, CreatePV/ExtendVG + // MUST NOT be issued (no duplicate pvcreate wedge). + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "/dev/loop4", nil) + + pvs := []internal.PVData{{PVName: "/dev/disk/by-id/lvm-pv-uuid-xyz"}} + assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) +} + // When the extend step (pvcreate/vgextend) fails after a new file device was // freshly provisioned, the loop device and backing file created in THIS call // must be rolled back so a failed extend does not leak them on the node (and @@ -2018,3 +2051,70 @@ func TestExtendFileDevicesIfNeeded_RollsBackProvisionedOnExtendFailure(t *testin assert.Error(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, nil)) } + +// validateLVGForUpdateFunc must count the capacity of newly-added +// spec.fileDevices entries (not yet PVs in the VG) when validating thin-pool +// sizes — mirroring additionBlockDeviceSpace for block devices. Without it, a +// combined "append a fileDevices entry + grow thin-pools" edit is validated +// against the pre-extend VG size and wrongly rejected. +// +// Two thin-pools are used on purpose: with a single pool, a request that +// exceeds the VG size is silently accepted as a "100% of VG" pool, so the +// missing-capacity bug would not surface. With two pools, the too-small VG +// size pushes the first pool into the full-VG branch and produces a rejection +// ("requests size of full VG space, but there are any other thin-pools"), +// which the fix avoids. +func TestValidateLVGForUpdateFunc_CountsNewFileDeviceSpaceForThinPools(t *testing.T) { + const vgName = "test-vg-file" + + makeLVG := func(status v1alpha1.LVMVolumeGroupStatus) *v1alpha1.LVMVolumeGroup { + return &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-file"}, + Spec: v1alpha1.LVMVolumeGroupSpec{ + ActualVGNameOnTheNode: vgName, + FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{ + {Directory: "/data", Size: resource.MustParse("5G")}, + }, + ThinPools: []v1alpha1.LVMVolumeGroupThinPoolSpec{ + {Name: "tp1", Size: "2G"}, + {Name: "tp2", Size: "2G"}, + }, + }, + Status: status, + } + } + + t.Run("new_file_device_capacity_is_counted", func(t *testing.T) { + r := setupReconciler() + // VG currently holds 1G; the 5G file device is new (absent from status), + // so newTotalVGSize = 1G + 5G = 6G and both 2G pools fit. + vgs := []internal.VGData{{VGName: vgName, VGSize: resource.MustParse("1G"), VGFree: resource.MustParse("1G")}} + r.sdsCache.StoreVGs(vgs, bytes.Buffer{}) + r.sdsCache.StorePVs(nil, bytes.Buffer{}) + + valid, reason := r.validateLVGForUpdateFunc(context.Background(), makeLVG(v1alpha1.LVMVolumeGroupStatus{}), map[string]v1alpha1.BlockDevice{}) + if assert.True(t, valid) { + assert.Equal(t, "", reason) + } + }) + + t.Run("existing_file_device_is_not_double_counted", func(t *testing.T) { + r := setupReconciler() + // The same 5G file device is already in status (already part of the 1G + // VG for the purpose of the test), so it is NOT added again: + // newTotalVGSize stays 1G and the two 2G pools no longer fit. + existingPath := utils.BuildFileDevicePath("/data", "vg-file", resource.MustParse("5G")) + status := v1alpha1.LVMVolumeGroupStatus{ + Nodes: []v1alpha1.LVMVolumeGroupNode{{ + Name: "test_node", + FileDevices: []v1alpha1.LVMVolumeGroupFileDevice{{FilePath: existingPath}}, + }}, + } + vgs := []internal.VGData{{VGName: vgName, VGSize: resource.MustParse("1G"), VGFree: resource.MustParse("1G")}} + r.sdsCache.StoreVGs(vgs, bytes.Buffer{}) + r.sdsCache.StorePVs(nil, bytes.Buffer{}) + + valid, _ := r.validateLVGForUpdateFunc(context.Background(), makeLVG(status), map[string]v1alpha1.BlockDevice{}) + assert.False(t, valid) + }) +} From 4493171edbb3202ce6c4bf2d3efc1ed9ac22e620 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sat, 27 Jun 2026 20:43:57 +0300 Subject: [PATCH 22/32] refactor(agent): extract shared detached rollback context for file devices The three file-device rollback sites (provisionFileDevices, extendFileDevicesIfNeeded, createVGComplex) each open-coded the same detached-context-with-deadline setup. Extract it into Reconciler.newRollbackContext so the "cleanup must survive a cancelled reconcile ctx" invariant lives in one place and the blocks cannot drift. No behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/reconciler.go | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index eeb475d76..71a5a00fd 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -1379,12 +1379,8 @@ func (r *Reconciler) extendFileDevicesIfNeeded( if retErr == nil || len(provisioned) == 0 { return } - rollbackCtx := context.Background() - if r.cfg.CmdDeadlineDuration > 0 { - var cancel context.CancelFunc - rollbackCtx, cancel = context.WithTimeout(rollbackCtx, r.cfg.CmdDeadlineDuration) - defer cancel() - } + rollbackCtx, cancel := r.newRollbackContext() + defer cancel() for i := len(provisioned) - 1; i >= 0; i-- { p := provisioned[i] if p.loopDev != "" { @@ -1491,6 +1487,20 @@ func (r *Reconciler) loopAlreadyRegisteredAsPV(ctx context.Context, loop string, return resolveFailed } +// newRollbackContext returns a fresh, detached context (bounded by the +// configured command deadline when set) for file-device cleanup that must run +// even when the reconcile context is already cancelled. The failure that +// triggers a rollback is frequently the reconcile ctx being cancelled (SIGTERM, +// deadline), and exec.CommandContext refuses to start a process under an +// already-cancelled context — which would strand the loop device and backing +// file we just created. The returned cancel func is always safe to defer. +func (r *Reconciler) newRollbackContext() (context.Context, context.CancelFunc) { + if r.cfg.CmdDeadlineDuration > 0 { + return context.WithTimeout(context.Background(), r.cfg.CmdDeadlineDuration) + } + return context.Background(), func() {} +} + func (r *Reconciler) tryGetVG(vgName string) (bool, internal.VGData) { vgs, _ := r.sdsCache.GetVGs() for _, vg := range vgs { @@ -1686,18 +1696,8 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM if retErr == nil { return } - // Rollback must run on a detached context: the failure that - // triggered it is frequently the reconcile ctx being cancelled - // (SIGTERM, deadline). exec.CommandContext refuses to start a - // process under an already-cancelled context, which would leave - // the loop device and backing file we just created dangling. Give - // the cleanup its own bounded deadline instead. - rollbackCtx := context.Background() - if r.cfg.CmdDeadlineDuration > 0 { - var cancel context.CancelFunc - rollbackCtx, cancel = context.WithTimeout(rollbackCtx, r.cfg.CmdDeadlineDuration) - defer cancel() - } + rollbackCtx, cancel := r.newRollbackContext() + defer cancel() for i := len(created) - 1; i >= 0; i-- { rb := created[i] if rb.attachedLoopOK && rb.loopDev != "" { @@ -1952,12 +1952,8 @@ func (r *Reconciler) createVGComplex(ctx context.Context, lvg *v1alpha1.LVMVolum if retErr == nil { return } - rollbackCtx := context.Background() - if r.cfg.CmdDeadlineDuration > 0 { - var cancel context.CancelFunc - rollbackCtx, cancel = context.WithTimeout(rollbackCtx, r.cfg.CmdDeadlineDuration) - defer cancel() - } + rollbackCtx, cancel := r.newRollbackContext() + defer cancel() if cleanupErr := r.cleanupFileDevices(rollbackCtx, lvg); cleanupErr != nil { r.log.Warning(fmt.Sprintf("[createVGComplex] unable to roll back file devices for the LVMVolumeGroup %s after a failed create: %v", lvg.Name, cleanupErr)) } From 2ba1e12188732cbf95a2881ec77ff1c64d48dc68 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sat, 27 Jun 2026 21:04:56 +0300 Subject: [PATCH 23/32] fix(agent): harden file-backed loop device handling Address review findings on the spec.fileDevices feature: - SetupLoopDevice: pass `losetup --nooverlap` so a race between startup reattach and the reconciler (or an alias-only existing attachment) reuses the existing loop instead of binding a second minor that nothing reaps. - GetLoopBackingFile: strip the " (deleted)" marker losetup appends when the backing file was unlinked while still attached. Without it IsManagedFileDevicePath misses the basename and cleanup refuses to detach the loop, stranding the minor forever. - hasStatusNodesDiff: match file devices by FilePath instead of slice position. The candidate slice is built in raw `lvm pvs` report order (unsorted within a VG) and a loop PV can be reported under /dev/loopN or an alias across scans, so a positional compare churned status every reconcile. - extendFileDevicesIfNeeded: when every new loop is skipped only because an alias PV could not be resolved, surface a retrying condition and return an error so the reconcile requeues, instead of silently reporting the configuration applied while the file device never joined the VG. Add unit tests for each fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/discoverer.go | 37 ++++++++++---- .../controller/lvg/discoverer_test.go | 48 +++++++++++++++++++ .../internal/controller/lvg/reconciler.go | 40 ++++++++++++---- .../controller/lvg/reconciler_test.go | 7 ++- images/agent/internal/utils/commands.go | 21 +++++++- images/agent/internal/utils/commands_test.go | 28 +++++++++++ 6 files changed, 161 insertions(+), 20 deletions(-) diff --git a/images/agent/internal/controller/lvg/discoverer.go b/images/agent/internal/controller/lvg/discoverer.go index 539bdbf6b..14e3c5743 100644 --- a/images/agent/internal/controller/lvg/discoverer.go +++ b/images/agent/internal/controller/lvg/discoverer.go @@ -1040,17 +1040,38 @@ func hasStatusNodesDiff(log logger.Logger, first, second []v1alpha1.LVMVolumeGro // without this the discoverer would never refresh a stale // loopDevice/pvUUID in status, and cleanup on delete would later act // on the stale value. - if len(first[i].FileDevices) != len(second[i].FileDevices) { + // + // Match by FilePath rather than by slice position: the candidate + // slice is built in raw `lvm pvs` report order (sortPVsByVG does not + // sort within a VG), and a loop PV can be reported under a /dev/loopN + // or an alias on different scans, so the two slices may list the same + // devices in a different order. A positional comparison would then + // flag a spurious diff and rewrite status on every reconcile. + if hasStatusFileDevicesDiff(first[i].FileDevices, second[i].FileDevices) { return true } + } - for j := range first[i].FileDevices { - if first[i].FileDevices[j].FilePath != second[i].FileDevices[j].FilePath || - first[i].FileDevices[j].LoopDevice != second[i].FileDevices[j].LoopDevice || - first[i].FileDevices[j].PVUuid != second[i].FileDevices[j].PVUuid || - first[i].FileDevices[j].Size.Value() != second[i].FileDevices[j].Size.Value() { - return true - } + return false +} + +func hasStatusFileDevicesDiff(first, second []v1alpha1.LVMVolumeGroupFileDevice) bool { + if len(first) != len(second) { + return true + } + + byPath := make(map[string]v1alpha1.LVMVolumeGroupFileDevice, len(second)) + for _, fd := range second { + byPath[fd.FilePath] = fd + } + + for _, fd := range first { + other, ok := byPath[fd.FilePath] + if !ok || + fd.LoopDevice != other.LoopDevice || + fd.PVUuid != other.PVUuid || + fd.Size.Value() != other.Size.Value() { + return true } } diff --git a/images/agent/internal/controller/lvg/discoverer_test.go b/images/agent/internal/controller/lvg/discoverer_test.go index 80e1603d2..ea2209c2b 100644 --- a/images/agent/internal/controller/lvg/discoverer_test.go +++ b/images/agent/internal/controller/lvg/discoverer_test.go @@ -1001,3 +1001,51 @@ func setupDiscoverer(opts *DiscovererConfig) *Discoverer { return NewDiscoverer(cl, log, metrics, cache.New(), utils.NewCommands(), *opts) } + +// hasStatusFileDevicesDiff must match file devices by FilePath, not by slice +// position: the candidate slice is built in raw `lvm pvs` report order while +// status holds a previous reconcile's order, and a loop PV can be reported +// under /dev/loopN or an alias on different scans. A positional comparison +// would flag a spurious diff and churn status on every reconcile. +func TestHasStatusFileDevicesDiff(t *testing.T) { + fdA := v1alpha1.LVMVolumeGroupFileDevice{ + FilePath: "/data/sds-vg-a-aaaaaaaaaaaa.img", + LoopDevice: "/dev/loop0", + PVUuid: "uuid-a", + Size: resource.MustParse("10Gi"), + } + fdB := v1alpha1.LVMVolumeGroupFileDevice{ + FilePath: "/data/sds-vg-a-bbbbbbbbbbbb.img", + LoopDevice: "/dev/loop1", + PVUuid: "uuid-b", + Size: resource.MustParse("20Gi"), + } + + t.Run("same_set_different_order_is_not_a_diff", func(t *testing.T) { + first := []v1alpha1.LVMVolumeGroupFileDevice{fdA, fdB} + second := []v1alpha1.LVMVolumeGroupFileDevice{fdB, fdA} + assert.False(t, hasStatusFileDevicesDiff(first, second)) + }) + + t.Run("changed_loop_device_is_a_diff", func(t *testing.T) { + moved := fdA + moved.LoopDevice = "/dev/loop7" + first := []v1alpha1.LVMVolumeGroupFileDevice{moved, fdB} + second := []v1alpha1.LVMVolumeGroupFileDevice{fdA, fdB} + assert.True(t, hasStatusFileDevicesDiff(first, second)) + }) + + t.Run("different_length_is_a_diff", func(t *testing.T) { + first := []v1alpha1.LVMVolumeGroupFileDevice{fdA, fdB} + second := []v1alpha1.LVMVolumeGroupFileDevice{fdA} + assert.True(t, hasStatusFileDevicesDiff(first, second)) + }) + + t.Run("different_file_path_is_a_diff", func(t *testing.T) { + renamed := fdB + renamed.FilePath = "/data/sds-vg-a-cccccccccccc.img" + first := []v1alpha1.LVMVolumeGroupFileDevice{fdA, renamed} + second := []v1alpha1.LVMVolumeGroupFileDevice{fdA, fdB} + assert.True(t, hasStatusFileDevicesDiff(first, second)) + }) +} diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 71a5a00fd..21d8d4c97 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -1411,18 +1411,39 @@ func (r *Reconciler) extendFileDevicesIfNeeded( // every reconcile. Resolve alias-form PV names before deciding a loop // is new. pvsToExtend := make([]string, 0, len(loopPaths)) + skippedOnResolverFailure := false for _, loop := range loopPaths { if _, exist := pvsMap[loop]; exist { continue } - if r.loopAlreadyRegisteredAsPV(ctx, loop, pvs) { - r.log.Debug(fmt.Sprintf("[extendFileDevicesIfNeeded] loop %s is already a PV of VG %s under an alias; skipping", loop, vg.VGName)) + registered, resolveFailed := r.loopAlreadyRegisteredAsPV(ctx, loop, pvs) + if registered { + if resolveFailed { + skippedOnResolverFailure = true + r.log.Warning(fmt.Sprintf("[extendFileDevicesIfNeeded] loop %s skipped because an alias PV could not be resolved; will retry on the next reconcile", loop)) + } else { + r.log.Debug(fmt.Sprintf("[extendFileDevicesIfNeeded] loop %s is already a PV of VG %s under an alias; skipping", loop, vg.VGName)) + } continue } pvsToExtend = append(pvsToExtend, loop) } if len(pvsToExtend) == 0 { + // A skip forced by a resolver failure is not a real "nothing to do": + // the loop might genuinely not be a PV yet, but we could not confirm + // it this round. Returning nil here would mark the configuration + // applied and the new file device would never join the VG, silently. + // Surface it on the condition and return an error so the reconcile + // requeues. The rollback defer is a no-op in this case: a skipped loop + // was already attached, so provisionFileDevices reused it and recorded + // nothing in `provisioned`. + if skippedOnResolverFailure { + if err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "unable to resolve alias PV names to decide whether file-backed loop devices are already part of the VG; retrying"); err != nil { + r.log.Error(err, fmt.Sprintf("[extendFileDevicesIfNeeded] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) + } + return fmt.Errorf("unable to resolve alias PV names for VG %s; retrying", vg.VGName) + } r.log.Debug(fmt.Sprintf("[extendFileDevicesIfNeeded] VG %s of the LVMVolumeGroup %s has no new file devices to add", vg.VGName, lvg.Name)) return nil } @@ -1449,13 +1470,17 @@ func (r *Reconciler) extendFileDevicesIfNeeded( // /dev/block/MAJ:MIN). It only resolves alias-form PV names, so the common // case (lvm reports the canonical /dev/loopN) costs no extra host command. // -// A resolver failure on an alias PV is treated conservatively as a match: +// It returns two booleans: registered (skip the extend for this loop) and +// resolveFailed (the skip was forced by an unresolvable alias rather than a +// confirmed match). A resolver failure is treated conservatively as a match: // we cannot rule out that the unresolved alias IS this loop already // registered as a PV, and handing a possibly-already-registered loop to // pvcreate fails ("already a PV") and wedges the VGConfigurationApplied // condition. It is safer to skip the extend this round and let the next // reconcile retry once the resolver recovers; provisionFileDevices keeps the -// loop attached idempotently in the meantime. +// loop attached idempotently in the meantime. The caller uses resolveFailed +// to surface a retrying condition instead of silently reporting "nothing to +// do" when every loop was skipped only because resolution failed. // // NOTE: this is one of two places that canonicalize an alias-reported loop // PV. Here the canonical /dev/loopN is known and alias PV names are resolved @@ -1463,8 +1488,7 @@ func (r *Reconciler) extendFileDevicesIfNeeded( // Discoverer.buildFileDeviceFromLoopPV (backing-file → canonical loop via // losetup). They use different methods because their inputs differ; keep // their ownership/aliasing assumptions in sync. -func (r *Reconciler) loopAlreadyRegisteredAsPV(ctx context.Context, loop string, pvs []internal.PVData) bool { - resolveFailed := false +func (r *Reconciler) loopAlreadyRegisteredAsPV(ctx context.Context, loop string, pvs []internal.PVData) (registered, resolveFailed bool) { for _, pv := range pvs { if !strings.HasPrefix(pv.PVName, "/dev/disk/") && !strings.HasPrefix(pv.PVName, "/dev/block/") { continue @@ -1478,13 +1502,13 @@ func (r *Reconciler) loopAlreadyRegisteredAsPV(ctx context.Context, loop string, continue } if resolved == loop { - return true + return true, false } } // No confirmed match. If at least one alias PV could not be resolved, // stay on the safe side and report the loop as already registered so the // caller skips a potentially duplicate pvcreate; the next reconcile retries. - return resolveFailed + return resolveFailed, resolveFailed } // newRollbackContext returns a fresh, detached context (bounded by the diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index ee924f2b9..d181bcda1 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1989,7 +1989,10 @@ func TestExtendFileDevicesIfNeeded_SkipsExistingPVUnderAlias(t *testing.T) { // readlink/nsenter failure), extendFileDevicesIfNeeded must NOT hand the loop // to pvcreate: the alias might already be this loop registered as a PV, and a // duplicate pvcreate would wedge the VGConfigurationApplied condition. The -// loop is skipped this round and retried on the next reconcile. +// loop is skipped this round, but because the skip was forced by a resolver +// failure (not a confirmed already-a-PV), the function returns an error so the +// reconcile requeues instead of silently reporting the configuration applied +// while the new file device never joined the VG. func TestExtendFileDevicesIfNeeded_SkipsOnResolverFailure(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -2015,7 +2018,7 @@ func TestExtendFileDevicesIfNeeded_SkipsOnResolverFailure(t *testing.T) { mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "/dev/loop4", nil) pvs := []internal.PVData{{PVName: "/dev/disk/by-id/lvm-pv-uuid-xyz"}} - assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) + assert.Error(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) } // When the extend step (pvcreate/vgextend) fails after a new file device was diff --git a/images/agent/internal/utils/commands.go b/images/agent/internal/utils/commands.go index 932792699..8526c3d0e 100644 --- a/images/agent/internal/utils/commands.go +++ b/images/agent/internal/utils/commands.go @@ -807,7 +807,13 @@ func (commands) CreateFileDevice(ctx context.Context, path string, sizeBytes int } func (commands) SetupLoopDevice(ctx context.Context, filePath string) (string, string, error) { - args := nsentrerExpendedArgs("/sbin/losetup", "--find", "--show", filePath) + // --nooverlap makes losetup reuse an already-attached loop for this + // backing file (printing it via --show) instead of binding a second + // minor to the same file. Without it, a race between the startup + // reattach and the reconciler's provision step — or an existing + // attachment that FindLoopDeviceByFile reported only under an alias — + // would leak an extra loop device that nothing ever reaps. + args := nsentrerExpendedArgs("/sbin/losetup", "--find", "--nooverlap", "--show", filePath) cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) var stdout, stderr bytes.Buffer @@ -879,7 +885,18 @@ func (commands) GetLoopBackingFile(ctx context.Context, loopDev string) (string, if err := cmd.Run(); err != nil { return cmd.String(), "", fmt.Errorf("unable to read backing file for %s: %w, stderr: %s", loopDev, err, stderr.String()) } - return cmd.String(), strings.TrimSpace(stdout.String()), nil + return cmd.String(), sanitizeBackingFilePath(stdout.String()), nil +} + +// sanitizeBackingFilePath normalises the backing-file path losetup reports. +// When the backing file has been unlinked while the loop is still attached, +// losetup (like /sys/block//loop/backing_file) appends a literal +// " (deleted)" marker, e.g. "/data/sds-vg-a-deadbeef0011.img (deleted)". +// Leaving the marker in place makes IsManagedFileDevicePath miss the basename +// and cleanup refuse to detach the loop, stranding the minor on the node +// forever. Strip the marker so ownership matching still recognises the file. +func sanitizeBackingFilePath(out string) string { + return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(out), "(deleted)")) } func (commands) RemoveFileDevice(ctx context.Context, path string) (string, error) { diff --git a/images/agent/internal/utils/commands_test.go b/images/agent/internal/utils/commands_test.go index a2ddb19eb..086bfb64e 100644 --- a/images/agent/internal/utils/commands_test.go +++ b/images/agent/internal/utils/commands_test.go @@ -509,3 +509,31 @@ func TestUdevadmTriggerEmptyPathsIsNoop(t *testing.T) { assert.Empty(t, out) }) } + +// GetLoopBackingFile must strip the " (deleted)" marker losetup appends when +// the backing file was unlinked while the loop is still attached. Without +// stripping it, IsManagedFileDevicePath would not recognise the basename and +// cleanup would refuse to detach the loop, stranding the minor on the node. +func TestSanitizeBackingFilePath(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"plain", "/data/sds-vg-a-deadbeef0011.img", "/data/sds-vg-a-deadbeef0011.img"}, + {"trailing newline", "/data/sds-vg-a-deadbeef0011.img\n", "/data/sds-vg-a-deadbeef0011.img"}, + {"deleted marker", "/data/sds-vg-a-deadbeef0011.img (deleted)", "/data/sds-vg-a-deadbeef0011.img"}, + {"deleted marker with newline", "/data/sds-vg-a-deadbeef0011.img (deleted)\n", "/data/sds-vg-a-deadbeef0011.img"}, + {"empty", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := sanitizeBackingFilePath(tt.in) + assert.Equal(t, tt.want, got) + if tt.want != "" { + assert.True(t, IsManagedFileDevicePath(got, "vg-a"), + "sanitized path must still be recognised as managed") + } + }) + } +} From 3dc8ba0bffac984a7588bc3729a71fa10718b578 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sun, 28 Jun 2026 07:05:38 +0300 Subject: [PATCH 24/32] fix(agent): treat lvm "matches existing size" as a no-op thin-pool resize A thin pool sized 100% can never reach the full VG size because LVM reserves space for thin-pool metadata, so the reconciler perpetually sees actualSize < requestedSize and re-runs `lvextend -l 100%VG`. That command exits non-zero on a no-op resize; filterStdErr is meant to swallow the benign message, but it only matched the older LVM wording "No size change". Newer LVM (2.03.x, e.g. Ubuntu) instead prints "New size (N extents) matches existing size (M extents).", which slipped through the filter and surfaced as ThinPoolReconcileFailed / VGConfigurationApplied=False on a perfectly healthy pool. Add a regex for the new wording. Affects any 100%-sized thin pool, not only file-backed VGs. Adds a filterStdErr unit test covering both wordings. Co-Authored-By: Claude Opus 4.8 (1M context) --- images/agent/internal/utils/commands.go | 13 +++++- images/agent/internal/utils/commands_test.go | 42 ++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/images/agent/internal/utils/commands.go b/images/agent/internal/utils/commands.go index 8526c3d0e..b2a2e2b08 100644 --- a/images/agent/internal/utils/commands.go +++ b/images/agent/internal/utils/commands.go @@ -1038,7 +1038,13 @@ func filterStdErr(command string, stdErr bytes.Buffer) bytes.Buffer { // this needs as if the controller were restarted and found existing LVG thin-pools with size equals 100%VG space, // as the Thin-pool size on the node might be less than Spec one even with delta (because of metadata). So the controller // will try to resize the Thin-pool with 100%VG space and will get the error. + // + // LVM wording for a no-op resize differs across versions: older releases + // print "No size change.", newer ones (lvm 2.03.x) print + // "New size (N extents) matches existing size (M extents).". Both mean the + // requested size already matches the LV, so neither is a real failure. regexpNoSizeChangeError := ` No size change.+` + regexpSizeMatchesError := `New size \(.+\) matches existing size \(.+\)` regex1, err := regexp.Compile(regexpPattern) if err != nil { return stdErr @@ -1051,12 +1057,17 @@ func filterStdErr(command string, stdErr bytes.Buffer) bytes.Buffer { if err != nil { return stdErr } + regex4, err := regexp.Compile(regexpSizeMatchesError) + if err != nil { + return stdErr + } for stdErrScanner.Scan() { line := stdErrScanner.Text() if regex1.MatchString(line) || regex2.MatchString(line) || - regex3.MatchString(line) { + regex3.MatchString(line) || + regex4.MatchString(line) { golog.Printf("WARNING: [filterStdErr] Line filtered from stderr due to matching exclusion pattern. Line: '%s'. Triggered by command: '%s'.", line, command) } else { filteredStdErr.WriteString(line + "\n") diff --git a/images/agent/internal/utils/commands_test.go b/images/agent/internal/utils/commands_test.go index 086bfb64e..7dad042b9 100644 --- a/images/agent/internal/utils/commands_test.go +++ b/images/agent/internal/utils/commands_test.go @@ -17,6 +17,7 @@ limitations under the License. package utils import ( + "bytes" "context" "testing" @@ -537,3 +538,44 @@ func TestSanitizeBackingFilePath(t *testing.T) { }) } } + +func TestFilterStdErr(t *testing.T) { + tests := []struct { + name string + stderr string + filtered bool // true => the line is recognised as benign and dropped + }{ + { + name: "old lvm no-op resize wording", + stderr: " No size change. (use --force to override)", + filtered: true, + }, + { + name: "new lvm no-op resize wording", + stderr: " New size (2553 extents) matches existing size (2553 extents).", + filtered: true, + }, + { + name: "regex version mismatch", + stderr: "Regex version mismatch, expected: 1.2.3 actual: 4.5.6", + filtered: true, + }, + { + name: "genuine error is kept", + stderr: " Volume group \"vg-thin-file\" not found.", + filtered: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + buf.WriteString(tt.stderr) + got := filterStdErr("lvextend -l 100%VG /dev/vg/lv", buf) + if tt.filtered { + assert.Zero(t, got.Len(), "benign line must be filtered out so the no-op resize is not treated as an error") + } else { + assert.NotZero(t, got.Len(), "genuine error must be preserved") + } + }) + } +} From bbddb12da6c6f987938e96db3c111076f2fb73f7 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sun, 28 Jun 2026 07:19:16 +0300 Subject: [PATCH 25/32] fix(agent): accept "100%"-sized thin pool in create-time validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A thin pool sized "100%" was rejected at create with "Required space for thin-pools N is more than VG size M". The percentage is computed from the raw block-device/file sum, which is not extent-aligned, and the validator rounded it UP to the next extent — landing one extent past the VG size and failing the capacity check, even though the pool is created with %FREE and fits the actual (extent-aligned) VG. Round percentage sizes DOWN for the capacity check instead (a percentage of the VG can never legitimately exceed it; this also makes split percentages like "50%"+"50%" sum to at most the VG). Absolute sizes keep rounding up so a genuinely oversized absolute pool is still rejected. Adds tests for a single "100%" pool (passes) and "100%" + another pool (fails). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/reconciler.go | 27 ++++++++- .../controller/lvg/reconciler_test.go | 55 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 21d8d4c97..eb04d958c 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -70,6 +70,31 @@ func extentSizeForThinPoolAlign(lvg *v1alpha1.LVMVolumeGroup, vg *internal.VGDat return lvmDefaultPhysicalExtent } +// alignThinPoolSizeForValidation aligns a thin-pool's requested size to the +// extent boundary for the create-time capacity check. +// +// Absolute sizes are rounded UP — that is the real number of extents LVM will +// consume, so an oversized absolute pool must still be rejected. +// +// Percentage sizes (e.g. "100%") are rounded DOWN instead. During create +// validation the VG size is the raw block-device/file sum, which is not +// extent-aligned, so a "100%" request rounded up lands one extent past the VG +// and would wrongly fail the capacity check — even though the pool is created +// with %FREE and fits. A percentage of the VG can never legitimately exceed +// it, so flooring is always safe and also makes split percentages (e.g. +// "50%"+"50%") sum to at most the VG size. +func alignThinPoolSizeForValidation(specSize string, requested, extentSize resource.Quantity) (resource.Quantity, error) { + if utils.IsPercentSize(specSize) { + extentBytes := extentSize.Value() + if extentBytes <= 0 { + return resource.Quantity{}, fmt.Errorf("extent size must be positive, got %d", extentBytes) + } + floored := (requested.Value() / extentBytes) * extentBytes + return *resource.NewQuantity(floored, resource.BinarySI), nil + } + return utils.AlignSizeToExtent(requested, extentSize) +} + type Reconciler struct { cl client.Client log logger.Logger @@ -842,7 +867,7 @@ func (r *Reconciler) validateLVGForCreateFunc( continue } - alignedTpSize, alignErr := utils.AlignSizeToExtent(tpRequestedSize, extentSizeForThinPoolAlign(lvg, nil)) + alignedTpSize, alignErr := alignThinPoolSizeForValidation(tp.Size, tpRequestedSize, extentSizeForThinPoolAlign(lvg, nil)) if alignErr != nil { reason.WriteString(fmt.Sprintf("Unable to align thin-pool %s size: %s. ", tp.Name, alignErr.Error())) continue diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index d181bcda1..117d929f1 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -456,6 +456,61 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { valid, _ := r.validateLVGForCreateFunc(ctx, lvg, bds) assert.False(t, valid) }) + + t.Run("with_full_vg_percent_thin_pool_returns_true", func(t *testing.T) { + // A single "100%" thin pool must pass validation. The raw VG size + // (sum of block devices) is not extent-aligned, so rounding the + // percentage up would land one extent past the VG and wrongly fail; + // the pool is created with %FREE and fits. + r := setupReconciler() + bds := map[string]v1alpha1.BlockDevice{ + "first": { + ObjectMeta: v1.ObjectMeta{Name: "first"}, + Status: v1alpha1.BlockDeviceStatus{ + // deliberately not a multiple of the 4Mi extent size + Size: resource.MustParse("20481Mi"), + Consumable: true, + }, + }, + } + lvg := &v1alpha1.LVMVolumeGroup{ + Spec: v1alpha1.LVMVolumeGroupSpec{ + ThinPools: []v1alpha1.LVMVolumeGroupThinPoolSpec{ + {Name: "thin", Size: "100%"}, + }, + }, + } + + valid, reason := r.validateLVGForCreateFunc(ctx, lvg, bds) + if assert.True(t, valid) { + assert.Equal(t, "", reason) + } + }) + + t.Run("with_full_vg_percent_thin_pool_and_another_returns_false", func(t *testing.T) { + // "100%" alongside any other thin pool cannot fit. + r := setupReconciler() + bds := map[string]v1alpha1.BlockDevice{ + "first": { + ObjectMeta: v1.ObjectMeta{Name: "first"}, + Status: v1alpha1.BlockDeviceStatus{ + Size: resource.MustParse("20481Mi"), + Consumable: true, + }, + }, + } + lvg := &v1alpha1.LVMVolumeGroup{ + Spec: v1alpha1.LVMVolumeGroupSpec{ + ThinPools: []v1alpha1.LVMVolumeGroupThinPoolSpec{ + {Name: "thin", Size: "100%"}, + {Name: "thin-2", Size: "1Gi"}, + }, + }, + } + + valid, _ := r.validateLVGForCreateFunc(ctx, lvg, bds) + assert.False(t, valid) + }) }) t.Run("identifyLVGReconcileFunc", func(t *testing.T) { From 1ff10e08fd3992f5b5c079ae6ef9e845f1756cf2 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sun, 28 Jun 2026 08:42:03 +0300 Subject: [PATCH 26/32] fix(agent): harden file-backed loop device handling from review Address issues found while reviewing the spec.fileDevices feature: - Drop unmanaged, purely loop-backed VGs from the cache (utils.FilterForeignLoopPVs). Removing loop from the foreign-PV filter let a guest VM's nested-LVM loop VG into the cache, where a name collision with a managed VG made findDuplicateVGNames take the healthy managed LVMVolumeGroup offline. Managed loop VGs (tagged storage.deckhouse.io/enabled=true) and block-backed VGs are kept. - Skip pvcreate for devices that are already an LVM PV in createVGComplex (with alias resolution for loop PVs), mirroring the guard in extendFileDevicesIfNeeded. Prevents a retry after a partial create from wedging VGConfigurationApplied with "already a PV". - Warn about decimal size units in the fileDevices size docs and the agent validation message: "1G" (10^9) is below the 1Gi minimum and, being immutable, can otherwise only be fixed by recreating the LVG. Co-Authored-By: Claude Opus 4.8 (1M context) --- crds/lvmvolumegroup.yaml | 7 ++ crds/lvmvolumegroupset.yaml | 5 ++ images/agent/internal/const.go | 12 ++- .../internal/controller/lvg/reconciler.go | 33 ++++++- .../controller/lvg/reconciler_test.go | 2 +- images/agent/internal/scanner/scanner.go | 24 ++++- images/agent/internal/utils/devicefilter.go | 84 ++++++++++++++++- .../agent/internal/utils/devicefilter_test.go | 89 +++++++++++++++++++ 8 files changed, 242 insertions(+), 14 deletions(-) diff --git a/crds/lvmvolumegroup.yaml b/crds/lvmvolumegroup.yaml index c4b3a97f0..ed34904b4 100644 --- a/crds/lvmvolumegroup.yaml +++ b/crds/lvmvolumegroup.yaml @@ -165,6 +165,13 @@ spec: pattern: '^[0-9]+(\.[0-9]+)?(E|P|T|G|M|k|Ei|Pi|Ti|Gi|Mi|Ki)?$' description: | Size of the preallocated backing file. Must be at least 1Gi. + + Prefer binary units (`Gi`, `Mi`). Decimal units are accepted + by the format but are smaller than they look: `1G` is 10^9 bytes, + which is below the 1Gi (2^30 bytes) minimum and will be rejected + by the agent. Because an existing entry's `size` is immutable and + cannot be removed, a too-small value can only be corrected by + deleting and recreating the LVMVolumeGroup, so set it correctly. thinPools: type: array description: | diff --git a/crds/lvmvolumegroupset.yaml b/crds/lvmvolumegroupset.yaml index b65f69e87..8740e74b7 100644 --- a/crds/lvmvolumegroupset.yaml +++ b/crds/lvmvolumegroupset.yaml @@ -204,6 +204,11 @@ spec: pattern: '^[0-9]+(\.[0-9]+)?(E|P|T|G|M|k|Ei|Pi|Ti|Gi|Mi|Ki)?$' description: | Size of the preallocated backing file. Must be at least 1Gi. + + Prefer binary units (`Gi`, `Mi`). Decimal units are accepted + by the format but are smaller than they look: `1G` is 10^9 bytes, + which is below the 1Gi (2^30 bytes) minimum and will be rejected + by the agent. thinPools: type: array description: | diff --git a/images/agent/internal/const.go b/images/agent/internal/const.go index f078b1f54..58f6929ff 100644 --- a/images/agent/internal/const.go +++ b/images/agent/internal/const.go @@ -48,8 +48,10 @@ const ( // // Loop devices are intentionally NOT rejected: the agent manages // file-backed loop devices as LVM PVs (spec.fileDevices). Unmanaged - // loop-backed VGs are safely ignored because the discoverer only - // picks up VGs tagged with storage.deckhouse.io/enabled=true. + // loop-backed VGs never become candidates (the discoverer only adopts + // VGs tagged storage.deckhouse.io/enabled=true) and are additionally + // dropped from the cache by utils.FilterForeignLoopPVs so they cannot + // collide by name with a managed VG. // // There is intentionally no blanket "a|.*|" accept rule. When a // device matches none of the reject patterns, LVM accepts it by @@ -132,8 +134,10 @@ var ( // // Loop devices (major 7) are NOT in this list: the agent manages // file-backed loop devices as LVM PVs via spec.fileDevices. - // Unmanaged loop PVs are safe because the discoverer only acts - // on VGs tagged with storage.deckhouse.io/enabled=true. + // Unmanaged loop PVs forming a whole VG are dropped from the cache by + // utils.FilterForeignLoopPVs so they cannot collide by name with a + // managed VG; the discoverer additionally only acts on VGs tagged with + // storage.deckhouse.io/enabled=true. // // Used after lvm returns the PV list, against the canonical // path resolved via readlink -f in the host mount namespace. diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index eb04d958c..6f87d81de 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -943,7 +943,11 @@ func (r *Reconciler) validateFileDevice( } if fd.Size.Value() < minFileDeviceSize.Value() { - fmt.Fprintf(reason, "fileDevices[%d].size %s is less than minimum %s. ", index, fd.Size.String(), minFileDeviceSize.String()) + // A common cause is a decimal unit (e.g. "1G" = 10^9 bytes) where a + // binary unit was meant ("1Gi" = 2^30 bytes); the former is below the + // minimum. Point the user at binary units so they do not get stuck on + // an immutable, too-small entry. + fmt.Fprintf(reason, "fileDevices[%d].size %s is less than the minimum %s; use a binary unit such as Gi. ", index, fd.Size.String(), minFileDeviceSize.String()) return } @@ -2010,7 +2014,34 @@ func (r *Reconciler) createVGComplex(ctx context.Context, lvg *v1alpha1.LVMVolum } r.log.Trace(fmt.Sprintf("[CreateVGComplex] LVMVolumeGroup %s devices paths %v", lvg.Name, paths)) + + // Skip pvcreate for any device that is already an LVM PV. This guards a + // retry after a create that pvcreated a device but failed before vgcreate + // (SIGTERM, ctx cancel, a rollback whose detach failed): provisionFileDevices + // reuses the still-attached loop, and handing an already-PV device to + // pvcreate fails ("already a PV") and wedges the VGConfigurationApplied + // condition. The vgcreate below still consumes an existing PV, so skipping + // the redundant pvcreate is safe. Mirrors the already-a-PV guard in + // extendFileDevicesIfNeeded; loop PVs reported under an alias are resolved + // via loopAlreadyRegisteredAsPV. + existingPVs, _ := r.sdsCache.GetPVs() + existingPVNames := make(map[string]struct{}, len(existingPVs)) + for _, pv := range existingPVs { + existingPVNames[pv.PVName] = struct{}{} + } + for _, path := range paths { + if _, ok := existingPVNames[path]; ok { + r.log.Info(fmt.Sprintf("[CreateVGComplex] %s is already a PV; skipping pvcreate", path)) + continue + } + if strings.HasPrefix(path, "/dev/loop") { + if registered, _ := r.loopAlreadyRegisteredAsPV(ctx, path, existingPVs); registered { + r.log.Info(fmt.Sprintf("[CreateVGComplex] loop %s is already a PV (under an alias); skipping pvcreate", path)) + continue + } + } + start := time.Now() command, err := r.commands.CreatePV(path) r.metrics.UtilsCommandsDuration(ReconcilerName, "pvcreate").Observe(r.metrics.GetEstimatedTimeInSeconds(start)) diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 117d929f1..9aa3d0702 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1632,7 +1632,7 @@ func TestValidateFileDevice(t *testing.T) { Size: resource.MustParse("500Mi"), } r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) - assert.Contains(t, reason.String(), "less than minimum") + assert.Contains(t, reason.String(), "less than the minimum") }) t.Run("size_adds_to_total_vg_size", func(t *testing.T) { diff --git a/images/agent/internal/scanner/scanner.go b/images/agent/internal/scanner/scanner.go index a721279ae..8c380adfe 100644 --- a/images/agent/internal/scanner/scanner.go +++ b/images/agent/internal/scanner/scanner.go @@ -262,11 +262,12 @@ func (s *scanner) fillTheCache(ctx context.Context, log logger.Logger, cache *ca // before feeding the cache so that the LVG/BD reconcile logic never sees // duplicate VG names produced by foreign storage layers. // - // Loop devices are intentionally NOT filtered here: the agent manages + // FilterForeignPVs does NOT reject loop devices: the agent manages // file-backed loop devices as LVM PVs (spec.fileDevices). Unmanaged loop - // PVs are tolerated because the discoverer only adopts a loop PV whose - // backing file matches its owner pattern (utils.IsManagedFileDevicePath) - // inside a VG tagged storage.deckhouse.io/lvmVolumeGroupName. + // PVs that form an entire VG are dropped just below by FilterForeignLoopPVs + // so a guest VM's nested-LVM loop VG cannot collide by name with a managed + // VG. A managed loop PV is kept because the agent tags every VG it creates + // with storage.deckhouse.io/enabled=true. // // cfg.CmdDeadlineDuration bounds every per-PV nsenter+readlink call: // a hung resolver on a single foreign device cannot block the entire @@ -276,6 +277,21 @@ func (s *scanner) fillTheCache(ctx context.Context, log logger.Logger, cache *ca pvs = utils.FilterForeignPVs(ctx, log, nil, pvs, cfg.CmdDeadlineDuration) if dropped := beforePV - len(pvs); dropped > 0 { log.Info(fmt.Sprintf("[fillTheCache] dropped %d foreign PV(s) backed by rbd/drbd/nbd devices", dropped)) + } + + // Also drop PVs of unmanaged, purely loop-backed VGs (nested LVM inside a + // guest VM's file-backed disk attached via losetup). Loop PVs are not + // rejected by FilterForeignPVs because the agent manages its own + // file-backed loop devices, but an unmanaged loop VG that shares a name + // with a managed VG would otherwise be detected as a duplicate by + // findDuplicateVGNames and take the managed LVMVolumeGroup offline. + beforeLoopPV := len(pvs) + pvs = utils.FilterForeignLoopPVs(log, vgs, pvs) + if dropped := beforeLoopPV - len(pvs); dropped > 0 { + log.Info(fmt.Sprintf("[fillTheCache] dropped %d PV(s) of unmanaged loop-backed VG(s)", dropped)) + } + + if len(pvs) < beforePV { beforeVG := len(vgs) vgs = utils.FilterVGsByPresentPVs(vgs, pvs) beforeLV := len(lvs) diff --git a/images/agent/internal/utils/devicefilter.go b/images/agent/internal/utils/devicefilter.go index b62dcc8ed..6a0a2ec05 100644 --- a/images/agent/internal/utils/devicefilter.go +++ b/images/agent/internal/utils/devicefilter.go @@ -85,10 +85,11 @@ func IsForeignDeviceBase(base string) bool { // loop reject would hide its own managed PVs. Ownership of a loop PV is // instead established later, in the discoverer, via the backing-file owner // pattern (IsManagedFileDevicePath) gated on the VG's -// storage.deckhouse.io/lvmVolumeGroupName tag. A consequence is that an -// unmanaged loop PV (e.g. nested LVM inside a file-backed guest VM disk) -// now reaches the cache; the duplicate-VG-name guard (findDuplicateVGNames) -// and the tag filter keep it from being mistaken for a managed VG. +// storage.deckhouse.io/lvmVolumeGroupName tag. Unmanaged loop PVs that +// form a whole VG (e.g. nested LVM inside a file-backed guest VM disk) are +// dropped separately by FilterForeignLoopPVs so they cannot collide by +// name with a managed VG; see that function for why the tag filter alone +// is not enough. // // lvm.static bundled with the agent has no udev integration and so it // enumerates devices via /dev/block/MAJOR:MINOR and /dev/disk/by-id/ @@ -159,6 +160,81 @@ func FilterForeignPVs( return out } +// FilterForeignLoopPVs drops PVs that belong to an unmanaged, purely +// loop-backed Volume Group — e.g. nested LVM inside a guest VM's +// file-backed disk attached on the host via losetup. +// +// Loop devices are not rejected by FilterForeignPVs because the agent +// manages its own file-backed loop devices (spec.fileDevices) as PVs. +// But an unmanaged loop-backed VG that reaches the cache is dangerous: +// findDuplicateVGNames runs over every cached VG (before tag filtering), +// so a guest VG that happens to share a name with a managed VG +// (`data`, `vg0`, … are common defaults) is detected as a duplicate and +// takes the *managed* LVMVolumeGroup offline (VGReady=False), and the +// agent's name-keyed cache lookups (FindVG/FindLV) could mix the two. +// +// A VG is dropped here only when ALL of the following hold: +// - it is NOT tagged storage.deckhouse.io/enabled=true (managed VGs, +// including the agent's own file-backed ones, are always kept — the +// agent tags every VG it creates at vgcreate time); +// - it has at least one PV and every one of its PVs is a /dev/loop* +// device (a VG with any real block-device PV is a legitimate, +// potentially-adoptable local VG and must stay visible). +// +// This restores the pre-spec.fileDevices behaviour for foreign loop VGs +// (loop PVs used to be rejected wholesale) while keeping managed +// file-backed loop VGs. Bare loop PVs not part of any VG are kept; they +// carry no VG name and cannot poison name resolution. +// +// Detection is by the /dev/loop name prefix, matching how the discoverer +// itself classifies loop PVs (Discoverer.configureCandidateNodeDevices). +// A managed loop PV occasionally reported under a /dev/disk or /dev/block +// alias counts as "non-loop" here, which only makes the filter more +// conservative (the VG is kept), never dropping a managed VG. +func FilterForeignLoopPVs(log logger.Logger, vgs []internal.VGData, pvs []internal.PVData) []internal.PVData { + managed := make(map[string]struct{}, len(vgs)) + for _, vg := range vgs { + if strings.Contains(vg.VGTags, internal.LVMTags[0]) { + managed[vg.VGUUID] = struct{}{} + } + } + + hasAnyPV := make(map[string]bool, len(pvs)) + hasNonLoopPV := make(map[string]bool, len(pvs)) + for _, pv := range pvs { + if pv.VGUuid == "" { + continue + } + hasAnyPV[pv.VGUuid] = true + if !strings.HasPrefix(pv.PVName, "/dev/loop") { + hasNonLoopPV[pv.VGUuid] = true + } + } + + isForeignLoopVG := func(vgUUID string) bool { + if vgUUID == "" { + return false + } + if _, ok := managed[vgUUID]; ok { + return false + } + return hasAnyPV[vgUUID] && !hasNonLoopPV[vgUUID] + } + + out := make([]internal.PVData, 0, len(pvs)) + for _, pv := range pvs { + if isForeignLoopVG(pv.VGUuid) { + log.Info(fmt.Sprintf( + "[FilterForeignLoopPVs] dropping PV %q of unmanaged loop-backed VG %q (VG_UUID=%q)", + pv.PVName, pv.VGName, pv.VGUuid, + )) + continue + } + out = append(out, pv) + } + return out +} + // FilterVGsByPresentPVs returns a copy of vgs that keeps only VGs // referenced by at least one PV in pvs (matched by VGUuid). It is // meant to run right after FilterForeignPVs so that phantom VGs whose diff --git a/images/agent/internal/utils/devicefilter_test.go b/images/agent/internal/utils/devicefilter_test.go index 3feca585e..ef0076787 100644 --- a/images/agent/internal/utils/devicefilter_test.go +++ b/images/agent/internal/utils/devicefilter_test.go @@ -194,6 +194,95 @@ func TestFilterForeignPVs_perCallTimeout(t *testing.T) { "(same contract as the transient-error branch)") } +func TestFilterForeignLoopPVs(t *testing.T) { + log, err := logger.NewLogger(logger.ErrorLevel) + if err != nil { + t.Fatalf("unable to create logger: %v", err) + } + + const managedTag = "storage.deckhouse.io/enabled=true,storage.deckhouse.io/lvmVolumeGroupName=vg-managed" + + tests := []struct { + name string + vgs []internal.VGData + pvs []internal.PVData + wantPVs []string + }{ + { + name: "drops an unmanaged loop-only VG", + vgs: []internal.VGData{ + {VGName: "data", VGUUID: "foreign", VGTags: ""}, + }, + pvs: []internal.PVData{ + {PVName: "/dev/loop7", VGUuid: "foreign", VGName: "data"}, + }, + wantPVs: nil, + }, + { + name: "keeps a managed loop-only VG (enabled tag present)", + vgs: []internal.VGData{ + {VGName: "vg-managed", VGUUID: "ours", VGTags: managedTag}, + }, + pvs: []internal.PVData{ + {PVName: "/dev/loop3", VGUuid: "ours", VGName: "vg-managed"}, + }, + wantPVs: []string{"/dev/loop3"}, + }, + { + name: "keeps an unmanaged VG with a non-loop PV (adoptable block VG)", + vgs: []internal.VGData{ + {VGName: "vg-block", VGUUID: "block", VGTags: ""}, + }, + pvs: []internal.PVData{ + {PVName: "/dev/sdb", VGUuid: "block", VGName: "vg-block"}, + }, + wantPVs: []string{"/dev/sdb"}, + }, + { + name: "keeps an unmanaged mixed VG (one loop, one block PV)", + vgs: []internal.VGData{ + {VGName: "vg-mixed", VGUUID: "mixed", VGTags: ""}, + }, + pvs: []internal.PVData{ + {PVName: "/dev/loop9", VGUuid: "mixed", VGName: "vg-mixed"}, + {PVName: "/dev/sdc", VGUuid: "mixed", VGName: "vg-mixed"}, + }, + wantPVs: []string{"/dev/loop9", "/dev/sdc"}, + }, + { + name: "keeps a bare loop PV not part of any VG", + vgs: nil, + pvs: []internal.PVData{ + {PVName: "/dev/loop1", VGUuid: ""}, + }, + wantPVs: []string{"/dev/loop1"}, + }, + { + name: "drops only the foreign loop VG, keeps the managed one with the same name", + vgs: []internal.VGData{ + {VGName: "data", VGUUID: "foreign", VGTags: ""}, + {VGName: "data", VGUUID: "ours", VGTags: managedTag}, + }, + pvs: []internal.PVData{ + {PVName: "/dev/loop7", VGUuid: "foreign", VGName: "data"}, + {PVName: "/dev/loop8", VGUuid: "ours", VGName: "data"}, + }, + wantPVs: []string{"/dev/loop8"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := FilterForeignLoopPVs(log, tt.vgs, tt.pvs) + gotNames := make([]string, 0, len(got)) + for _, pv := range got { + gotNames = append(gotNames, pv.PVName) + } + assert.ElementsMatch(t, tt.wantPVs, gotNames) + }) + } +} + func TestFilterVGsByPresentPVs(t *testing.T) { tests := []struct { name string From 52371244813d3104e6e551fc6d3c3d33a64e6c8c Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sun, 28 Jun 2026 12:04:03 +0300 Subject: [PATCH 27/32] fix(agent): prevent file-device VG corruption from create rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cluster testing of file-backed VG creation surfaced a data-integrity bug: under concurrent LVG creation (e.g. several LVGs applied at once), a file-backed VG could come up with its single backing file attached to two loop devices — one of them shown "(deleted)" — both registered as separate PVs, doubling the VG and placing a thin pool partly on a deleted-file loop. Reproduced on two fresh clusters. Root cause: createVGComplex rolled back a failed create via the broad cleanupFileDevices (the delete-path cleanup), which walks spec+status and removes every backing file of the LVG. When a concurrent reconcile — or a pvcreate/vgcreate that materially succeeded but returned a non-zero status ("File descriptor leaked on lvm invocation", exit 5) — had already turned the loop into a live PV of the VG, the rollback deleted that PV's backing file. The next reconcile then re-provisioned a second loop and vgextended it, doubling the VG. Fix: replace the broad rollback on the create and extend paths with a scoped rollbackProvisionedFileDevices that tears down ONLY the file devices the current call provisioned, and: - lists PVs authoritatively (fresh `lvm pvs`, not the stale cache) and SKIPS any provisioned loop that is already an LVM PV (canonical or via a /dev/disk//dev/block alias), so an in-use loop is never torn down; - leaves everything in place if the PV listing fails (a leaked loop/file is recoverable and reused on the next reconcile; corrupting a live VG is not); - never removes a backing file whose loop it could not detach. With this, a create whose pvcreate spuriously reports failure keeps the loop+file, and the retry reuses the same loop instead of allocating a second one. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/reconciler.go | 107 +++++++++++++----- .../controller/lvg/reconciler_test.go | 44 ++++++- 2 files changed, 123 insertions(+), 28 deletions(-) diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 6f87d81de..4cd5aaff1 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -1410,19 +1410,7 @@ func (r *Reconciler) extendFileDevicesIfNeeded( } rollbackCtx, cancel := r.newRollbackContext() defer cancel() - for i := len(provisioned) - 1; i >= 0; i-- { - p := provisioned[i] - if p.loopDev != "" { - if cmd, derr := r.commands.DetachLoopDevice(rollbackCtx, p.loopDev); derr != nil { - r.log.Warning(fmt.Sprintf("[extendFileDevicesIfNeeded][rollback] unable to detach %s: %v (cmd: %s)", p.loopDev, derr, cmd)) - } - } - if p.filePath != "" { - if cmd, rerr := r.commands.RemoveFileDevice(rollbackCtx, p.filePath); rerr != nil { - r.log.Warning(fmt.Sprintf("[extendFileDevicesIfNeeded][rollback] unable to remove %s: %v (cmd: %s)", p.filePath, rerr, cmd)) - } - } - } + r.rollbackProvisionedFileDevices(rollbackCtx, provisioned) }() pvsMap := make(map[string]struct{}, len(pvs)) @@ -1853,6 +1841,73 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM return loopPaths, provisioned, nil } +// rollbackProvisionedFileDevices tears down ONLY the file devices this reconcile +// just provisioned (created a fresh backing file and attached a fresh loop), +// after a later step (pvcreate/vgcreate/vgextend/condition update) failed. +// +// It is the create/extend-path counterpart to cleanupFileDevices and MUST be +// used instead of it on those paths: cleanupFileDevices walks spec+status and +// would remove the backing file of a loop that another, concurrent reconcile — +// or a pvcreate/vgcreate that materially succeeded but returned a non-zero +// status — has already turned into a live PV of the VG. Removing such a file +// leaves a PV backed by a deleted file while the VG keeps using it; the next +// reconcile then re-provisions a second loop and the VG silently doubles in +// size (observed on real clusters as one backing file attached to two loops, +// one shown "(deleted)"). +// +// As a hard safety net it lists the current PVs once, authoritatively (a fresh +// `lvm pvs`, not the possibly-stale cache, because the result gates a +// destructive teardown), and SKIPS any provisioned loop that is already an LVM +// PV (matched canonically or via the /dev/disk//dev/block alias the discoverer +// also resolves). If the PV listing fails it tears nothing down — a leaked +// loop/file is recoverable (the next reconcile reuses it via +// FindLoopDeviceByFile), corrupting a live VG is not — and it never removes a +// backing file whose loop it could not detach, since the loop may still +// reference it. +func (r *Reconciler) rollbackProvisionedFileDevices(ctx context.Context, provisioned []provisionedFileDevice) { + if len(provisioned) == 0 { + return + } + + pvs, cmd, _, err := r.commands.GetAllPVs(ctx) + r.log.Debug(cmd) + if err != nil { + r.log.Warning(fmt.Sprintf("[rollbackProvisionedFileDevices] unable to list PVs to confirm rollback is safe; leaving %d provisioned file device(s) in place (a leak is recoverable, corrupting a live VG is not): %v", len(provisioned), err)) + return + } + pvNames := make(map[string]struct{}, len(pvs)) + for _, pv := range pvs { + pvNames[pv.PVName] = struct{}{} + } + + for i := len(provisioned) - 1; i >= 0; i-- { + p := provisioned[i] + if p.loopDev != "" { + _, isPV := pvNames[p.loopDev] + if !isPV { + // lvm.static without udev may report the loop PV under a + // /dev/disk or /dev/block alias instead of /dev/loopN. + if registered, resolveFailed := r.loopAlreadyRegisteredAsPV(ctx, p.loopDev, pvs); registered || resolveFailed { + isPV = true + } + } + if isPV { + r.log.Warning(fmt.Sprintf("[rollbackProvisionedFileDevices] loop %s (file %s) is already an LVM PV; skipping rollback so a concurrent or partially-succeeded create does not lose its backing storage", p.loopDev, p.filePath)) + continue + } + if cmd, derr := r.commands.DetachLoopDevice(ctx, p.loopDev); derr != nil { + r.log.Warning(fmt.Sprintf("[rollbackProvisionedFileDevices] unable to detach %s: %v (cmd: %s); keeping backing file %s in place", p.loopDev, derr, cmd, p.filePath)) + continue + } + } + if p.filePath != "" { + if cmd, rerr := r.commands.RemoveFileDevice(ctx, p.filePath); rerr != nil { + r.log.Warning(fmt.Sprintf("[rollbackProvisionedFileDevices] unable to remove %s: %v (cmd: %s)", p.filePath, rerr, cmd)) + } + } + } +} + // cleanupFileDevices detaches loop devices and removes backing files // recorded for this LVG. It walks the union of status.nodes[].fileDevices // (what the discoverer last observed) and spec.fileDevices (what the @@ -1985,31 +2040,29 @@ func (r *Reconciler) cleanupFileDevices(ctx context.Context, lvg *v1alpha1.LVMVo func (r *Reconciler) createVGComplex(ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, blockDevices map[string]v1alpha1.BlockDevice) (retErr error) { paths := extractPathsFromBlockDevices(nil, blockDevices) - // On a fresh create the VG holds no file devices yet, so the defer below - // rolls everything back via cleanupFileDevices; the per-call provisioned - // list is only needed for the scoped rollback in extendFileDevicesIfNeeded. - loopPaths, _, err := r.provisionFileDevices(ctx, lvg) + loopPaths, provisioned, err := r.provisionFileDevices(ctx, lvg) if err != nil { return fmt.Errorf("file device provisioning failed: %w", err) } paths = append(paths, loopPaths...) - // provisionFileDevices only rolls back artifacts it created in its own - // invocation; once it returns successfully the loops/files survive. If a - // later step (pvcreate/vgcreate) fails, detach and remove them here so a - // failed create does not leak loop devices and preallocated files on the - // node. Runs on a detached context because the failure is frequently the - // reconcile ctx being cancelled, and cleanupFileDevices is idempotent. - if len(lvg.Spec.FileDevices) > 0 { + // If a later step (pvcreate/vgcreate) fails, tear down ONLY the file devices + // this call provisioned — and never one that already became a PV. This must + // NOT use the broad cleanupFileDevices (the delete-path cleanup): it walks + // spec+status and could remove the backing file of a loop that a concurrent + // reconcile, or a pvcreate/vgcreate that materially succeeded but returned a + // non-zero status, had already turned into a live PV of the VG — which the + // next reconcile then re-provisions with a second loop, doubling the VG. + // See rollbackProvisionedFileDevices. Runs on a detached context because the + // failure is frequently the reconcile ctx being cancelled. + if len(provisioned) > 0 { defer func() { if retErr == nil { return } rollbackCtx, cancel := r.newRollbackContext() defer cancel() - if cleanupErr := r.cleanupFileDevices(rollbackCtx, lvg); cleanupErr != nil { - r.log.Warning(fmt.Sprintf("[createVGComplex] unable to roll back file devices for the LVMVolumeGroup %s after a failed create: %v", lvg.Name, cleanupErr)) - } + r.rollbackProvisionedFileDevices(rollbackCtx, provisioned) }() } diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 9aa3d0702..44dbb1f07 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -2102,7 +2102,9 @@ func TestExtendFileDevicesIfNeeded_RollsBackProvisionedOnExtendFailure(t *testin mc.EXPECT().SetupLoopDevice(gomock.Any(), path).Return("losetup --find ...", "/dev/loop4", nil), // Extend fails on pvcreate... mc.EXPECT().CreatePV("/dev/loop4").Return("pvcreate ...", errors.New("pvcreate refused")), - // ...so the just-provisioned loop + file are rolled back. + // ...rollback first confirms the loop is not already a PV (it is not)... + mc.EXPECT().GetAllPVs(gomock.Any()).Return(nil, "pvs", bytes.Buffer{}, nil), + // ...then tears down the just-provisioned loop + file. mc.EXPECT().DetachLoopDevice(gomock.Any(), "/dev/loop4").Return("losetup -d ...", nil), mc.EXPECT().RemoveFileDevice(gomock.Any(), path).Return("rm ...", nil), ) @@ -2110,6 +2112,46 @@ func TestExtendFileDevicesIfNeeded_RollsBackProvisionedOnExtendFailure(t *testin assert.Error(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, nil)) } +// rollbackProvisionedFileDevices must NOT detach the loop or remove the backing +// file of a device whose loop has already become a live PV — by a concurrent +// reconcile or by a pvcreate/vgcreate that materially succeeded but returned a +// non-zero status. Tearing it down deletes the backing file out from under a +// PV the VG is using, which on real clusters left the single backing file +// attached to two loops (one "(deleted)") and doubled the VG. Here CreatePV +// reports failure yet the loop shows up as a PV in the authoritative pvs list, +// so rollback must skip it entirely. +func TestExtendFileDevicesIfNeeded_DoesNotRollBackLoopThatBecamePV(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + vg := internal.VGData{VGName: "vg-test"} + + gomock.InOrder( + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "", nil), + mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fd.Directory).Return("mkdir -p ...", nil), + mc.EXPECT().CreateFileDevice(gomock.Any(), path, fd.Size.Value()).Return("fallocate ...", nil), + mc.EXPECT().SetupLoopDevice(gomock.Any(), path).Return("losetup --find ...", "/dev/loop4", nil), + mc.EXPECT().CreatePV("/dev/loop4").Return("pvcreate ...", errors.New("fd leaked on lvm invocation")), + // The pvcreate actually wrote the PV label despite the error, so the + // authoritative pvs list reports /dev/loop4 as a PV of the VG. + mc.EXPECT().GetAllPVs(gomock.Any()).Return( + []internal.PVData{{PVName: "/dev/loop4", VGName: "vg-test"}}, "pvs", bytes.Buffer{}, nil), + ) + // No DetachLoopDevice / RemoveFileDevice expectations: rollback must skip the + // in-use loop. gomock fails the test if either is called. + + assert.Error(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, nil)) +} + // validateLVGForUpdateFunc must count the capacity of newly-added // spec.fileDevices entries (not yet PVs in the VG) when validating thin-pool // sizes — mirroring additionBlockDeviceSpace for block devices. Without it, a From 226b450aa9a4688f26974634bc46460835ff5aff Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Sun, 28 Jun 2026 12:38:55 +0300 Subject: [PATCH 28/32] fix(agent): address re-review findings for file-backed devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three issues found in a from-scratch re-review: 1. validateLVGForUpdateFunc counted an already-provisioned file device as new whenever its directory is a symlink. status FilePath comes from `losetup --output BACK-FILE`, which canonicalizes symlink components, so it never matched the literal BuildFileDevicePath. Match by basename (stable: the hash is derived from the literal spec directory and losetup leaves the basename untouched) instead of full path, so thin-pool sizing validation no longer inflates the VG size and wrongly accepts a pool that then fails to create. 2. extendFileDevicesIfNeeded trusted the PV snapshot captured at the top of reconcileLVGUpdateFunc; a loop that became a canonical /dev/loopN PV after that snapshot (and is not an alias-form PV, the only form loopAlreadyRegisteredAsPV inspects) was handed to pvcreate again and failed "already a PV", wedging the condition. Take the union of the snapshot and a fresh cache read before deciding, mirroring createVGComplex. 3. CreatePV treated the benign "File descriptor N leaked on lvm invocation" exit (lvm.static under nsenter) as a hard failure, tripping the create/extend rollback against a device that pvcreate had in fact already turned into a healthy PV — the real trigger behind the doubled-VG corruption the rollback safety net contains. Route stderr through filterStdErr and error only when a non-benign line remains, mirroring ExtendLV. Added regression tests for 1 and 2. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/reconciler.go | 33 ++++++++++-- .../controller/lvg/reconciler_test.go | 54 +++++++++++++++++++ images/agent/internal/utils/commands.go | 12 ++++- 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index 4cd5aaff1..c60c3b813 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -1036,15 +1036,26 @@ func (r *Reconciler) validateLVGForUpdateFunc( // branch — the same defect the create path avoids via countVGSizeByFileDevices. var additionFileDeviceSpace int64 if len(lvg.Spec.FileDevices) > 0 { - existingFilePaths := make(map[string]struct{}) + // Match by basename, not full path: status.nodes[].fileDevices[].FilePath + // is the loop's backing file as reported by `losetup --output BACK-FILE`, + // which canonicalizes symlink components of the directory (e.g. a spec + // directory /data symlinked to /mnt/disk1/data is reported as + // /mnt/disk1/data/...), while BuildFileDevicePath keeps the literal spec + // directory. The basename `sds--.img` is identical on both + // sides (the hash is derived from the literal spec directory string and + // losetup leaves the basename untouched), so a full-path compare would + // miss an already-provisioned device whenever the directory is a symlink + // and count it as new on every reconcile, inflating the VG size used for + // thin-pool validation. + existingBasenames := make(map[string]struct{}) for _, n := range lvg.Status.Nodes { for _, fd := range n.FileDevices { - existingFilePaths[fd.FilePath] = struct{}{} + existingBasenames[filepath.Base(fd.FilePath)] = struct{}{} } } for _, fd := range lvg.Spec.FileDevices { - path := utils.BuildFileDevicePath(fd.Directory, lvg.Name, fd.Size) - if _, ok := existingFilePaths[path]; !ok { + base := filepath.Base(utils.BuildFileDevicePath(fd.Directory, lvg.Name, fd.Size)) + if _, ok := existingBasenames[base]; !ok { additionFileDeviceSpace += fd.Size.Value() } } @@ -1413,6 +1424,20 @@ func (r *Reconciler) extendFileDevicesIfNeeded( r.rollbackProvisionedFileDevices(rollbackCtx, provisioned) }() + // Build the set of current PVs from BOTH the caller's snapshot (captured at + // the top of reconcileLVGUpdateFunc) and a fresh cache read. A loop may have + // become a canonical /dev/loopN PV after that snapshot (a prior reconcile + // pvcreated it but failed before the VG was assembled, or the create-rollback + // kept it); the stale snapshot would not list it, the exact-name check below + // would miss it, and loopAlreadyRegisteredAsPV only inspects /dev/disk/ + // /dev/block alias-form PVs (not canonical names) — so the loop would be + // handed to pvcreate again and fail "already a PV", wedging the condition + // every reconcile. Taking the union never drops a known PV, so the skip + // decision stays safe. createVGComplex re-reads GetPVs() before pvcreate for + // the same reason. + if freshPVs, _ := r.sdsCache.GetPVs(); len(freshPVs) > 0 { + pvs = append(append([]internal.PVData(nil), pvs...), freshPVs...) + } pvsMap := make(map[string]struct{}, len(pvs)) for _, pv := range pvs { pvsMap[pv.PVName] = struct{}{} diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 44dbb1f07..6403a44e6 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -2004,6 +2004,35 @@ func TestExtendFileDevicesIfNeeded_SkipsExistingPV(t *testing.T) { assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) } +// extendFileDevicesIfNeeded must consult a FRESH cache read, not only the +// caller's PV snapshot. A loop can become a canonical /dev/loopN PV after the +// snapshot was captured at the top of reconcileLVGUpdateFunc (e.g. a prior +// reconcile pvcreated it but failed before the VG was assembled, or the +// create-rollback kept it). Here the caller's snapshot is empty (nil) but the +// cache reports /dev/loop4 as a PV, so no CreatePV/ExtendVG must be issued — +// otherwise pvcreate would fail "already a PV" and wedge the condition. +func TestExtendFileDevicesIfNeeded_SkipsPVKnownOnlyToCache(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + vg := internal.VGData{VGName: "vg-test"} + + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "/dev/loop4", nil) + // Cache knows /dev/loop4 is a PV; the caller's snapshot (nil) does not. + r.sdsCache.StorePVs([]internal.PVData{{PVName: "/dev/loop4"}}, bytes.Buffer{}) + + assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, nil)) +} + // lvm.static has no udev integration, so an already-attached managed loop // PV is frequently reported under a /dev/disk/by-id (or /dev/block/MAJ:MIN) // alias rather than /dev/loopN. provisionFileDevices always returns the @@ -2217,4 +2246,29 @@ func TestValidateLVGForUpdateFunc_CountsNewFileDeviceSpaceForThinPools(t *testin valid, _ := r.validateLVGForUpdateFunc(context.Background(), makeLVG(status), map[string]v1alpha1.BlockDevice{}) assert.False(t, valid) }) + + t.Run("symlink_canonicalized_status_path_still_matches", func(t *testing.T) { + r := setupReconciler() + // losetup reports the loop's backing file with the directory symlink + // resolved (/data -> /mnt/disk1/data), so status FilePath differs from + // BuildFileDevicePath in its directory but shares the basename. The + // existing device must still be recognized (basename match) and NOT + // counted as new — so newTotalVGSize stays 1G and the two 2G pools no + // longer fit, exactly like the same-path case above. A full-path compare + // would miss it, inflate the VG size, and wrongly pass validation. + specPath := utils.BuildFileDevicePath("/data", "vg-file", resource.MustParse("5G")) + canonicalStatusPath := "/mnt/disk1" + specPath // /mnt/disk1/data/sds-...img + status := v1alpha1.LVMVolumeGroupStatus{ + Nodes: []v1alpha1.LVMVolumeGroupNode{{ + Name: "test_node", + FileDevices: []v1alpha1.LVMVolumeGroupFileDevice{{FilePath: canonicalStatusPath}}, + }}, + } + vgs := []internal.VGData{{VGName: vgName, VGSize: resource.MustParse("1G"), VGFree: resource.MustParse("1G")}} + r.sdsCache.StoreVGs(vgs, bytes.Buffer{}) + r.sdsCache.StorePVs(nil, bytes.Buffer{}) + + valid, _ := r.validateLVGForUpdateFunc(context.Background(), makeLVG(status), map[string]v1alpha1.BlockDevice{}) + assert.False(t, valid) + }) } diff --git a/images/agent/internal/utils/commands.go b/images/agent/internal/utils/commands.go index b2a2e2b08..4fc7da049 100644 --- a/images/agent/internal/utils/commands.go +++ b/images/agent/internal/utils/commands.go @@ -257,7 +257,17 @@ func (commands) CreatePV(path string) (string, error) { var stderr bytes.Buffer cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { + // Route stderr through filterStdErr and treat the command as successful when + // the only output is a benign message (e.g. "File descriptor N leaked on lvm + // invocation"), mirroring ExtendLV. lvm.static run under nsenter routinely + // emits the fd-leak warning and exits non-zero even though pvcreate has + // already written the PV label; surfacing that as an error wrongly trips the + // create/extend rollback against a device that is in fact a healthy PV. A + // real pvcreate failure prints its own diagnostic line, which survives the + // filter, so genuine errors are still reported. + err := cmd.Run() + filteredStdErr := filterStdErr(cmd.String(), stderr) + if err != nil && filteredStdErr.Len() > 0 { return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderror = %s", cmd.String(), err, stderr.String()) } From 0d0e4bcc171671b55b94a72e59cd150059b2409a Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Tue, 30 Jun 2026 10:59:28 +0300 Subject: [PATCH 29/32] fix(agent): guard node disk and enable loop direct-I/O for file devices Addresses two review findings on the file-backed LVM feature: - Refuse to allocate a backing file larger than the free space of the node's filesystem. provisionFileDevices now queries available bytes (stat -f in PID 1's mount namespace, via the new GetAvailableBytes command) right after mkdir and before fallocate, and reports a clear condition instead of letting `fallocate -l` fill the node's root filesystem and trip kubelet DiskPressure eviction. The check uses the non-superuser available-block count (%a), so the filesystem reserve is left as built-in headroom; it is best-effort and falls through to fallocate (which still fails cleanly on ENOSPC) when free space cannot be determined. - Attach loop devices with `losetup --direct-io=on` so backing-file I/O bypasses the node filesystem's page cache, avoiding the double page cache (volume FS + backing file) that doubles RAM use and throttles throughput. The kernel silently falls back to buffered I/O on misalignment, so requesting it is always safe. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../internal/controller/lvg/reconciler.go | 22 ++++++ .../controller/lvg/reconciler_test.go | 67 +++++++++++++++++++ images/agent/internal/mock_utils/commands.go | 16 +++++ images/agent/internal/utils/commands.go | 65 +++++++++++++++++- images/agent/internal/utils/commands_test.go | 34 ++++++++++ 5 files changed, 203 insertions(+), 1 deletion(-) diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index c60c3b813..b95b36666 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -1826,6 +1826,28 @@ func (r *Reconciler) provisionFileDevices(ctx context.Context, lvg *v1alpha1.LVM return nil, nil, err } + // Refuse to allocate a backing file larger than the free space of the + // node's filesystem. `fallocate -l` preallocates the full size, so + // without this guard a single oversized fileDevices entry (or a typo in + // `directory`/`size`) can fill the node's root filesystem and push + // kubelet into DiskPressure eviction — a node-level outage, not a mere + // condition error. The check is best-effort: if we cannot determine the + // free space we log and fall through to fallocate, which still fails + // cleanly on a genuine ENOSPC. + availableBytes, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (int64, error) { + cmd, available, err := r.commands.GetAvailableBytes(ctx, fd.Directory) + r.log.Debug(cmd) + return available, err + }) + if err != nil { + r.log.Warning(fmt.Sprintf("[provisionFileDevices] unable to check free space in %s, proceeding (fallocate will still fail on ENOSPC): %v", fd.Directory, err)) + } else if availableBytes < sizeBytes { + return nil, nil, fmt.Errorf( + "not enough free space in %q to create backing file %s: %d bytes available, %d bytes requested", + fd.Directory, filePath, availableBytes, sizeBytes, + ) + } + rb := rollback{filePath: filePath} createCmd, err := utils.RunWithTimeout(ctx, r.cfg.CmdDeadlineDuration, func(ctx context.Context) (string, error) { return r.commands.CreateFileDevice(ctx, filePath, sizeBytes) diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 6403a44e6..9e5106796 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1761,10 +1761,12 @@ func TestProvisionFileDevices_RollsBackOnPartialFailure(t *testing.T) { gomock.InOrder( mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), pathA).Return("losetup -j", "", nil), mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fdA.Directory).Return("mkdir -p ...", nil), + mc.EXPECT().GetAvailableBytes(gomock.Any(), fdA.Directory).Return("stat -f ...", int64(1)<<50, nil), mc.EXPECT().CreateFileDevice(gomock.Any(), pathA, sizeA).Return("fallocate ...", nil), mc.EXPECT().SetupLoopDevice(gomock.Any(), pathA).Return("losetup --find ...", "/dev/loop0", nil), mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), pathB).Return("losetup -j", "", nil), mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fdB.Directory).Return("mkdir -p ...", nil), + mc.EXPECT().GetAvailableBytes(gomock.Any(), fdB.Directory).Return("stat -f ...", int64(1)<<50, nil), mc.EXPECT().CreateFileDevice(gomock.Any(), pathB, sizeB).Return("fallocate ...", nil), mc.EXPECT().SetupLoopDevice(gomock.Any(), pathB).Return("losetup --find ...", "", errors.New("ENOSPC")), // rollback in LIFO order: first pathB (file only, loop never attached), then pathA (loop + file). @@ -1779,6 +1781,68 @@ func TestProvisionFileDevices_RollsBackOnPartialFailure(t *testing.T) { } } +// A backing file larger than the filesystem's free space must be refused +// before fallocate runs, so a typo in directory/size cannot fill the node's +// root filesystem and trip kubelet DiskPressure eviction. The directory is +// still created (mkdir -p), but no file is allocated and no loop is attached. +func TestProvisionFileDevices_RefusesWhenNotEnoughFreeSpace(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("100Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + + gomock.InOrder( + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "", nil), + mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fd.Directory).Return("mkdir -p ...", nil), + // Only 1Gi free for a 100Gi request → reject before fallocate. + mc.EXPECT().GetAvailableBytes(gomock.Any(), fd.Directory).Return("stat -f ...", int64(1)<<30, nil), + ) + // CreateFileDevice / SetupLoopDevice MUST NOT be invoked. + + _, _, err := r.provisionFileDevices(ctx, lvg) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "not enough free space") + } +} + +// When free space cannot be determined the provision proceeds best-effort: +// fallocate still fails cleanly on a genuine ENOSPC, so a transient stat +// failure must not block provisioning by itself. +func TestProvisionFileDevices_ProceedsWhenFreeSpaceUnknown(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + + gomock.InOrder( + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "", nil), + mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fd.Directory).Return("mkdir -p ...", nil), + mc.EXPECT().GetAvailableBytes(gomock.Any(), fd.Directory).Return("stat -f ...", int64(0), errors.New("stat: command not found")), + mc.EXPECT().CreateFileDevice(gomock.Any(), path, fd.Size.Value()).Return("fallocate ...", nil), + mc.EXPECT().SetupLoopDevice(gomock.Any(), path).Return("losetup --find ...", "/dev/loop0", nil), + ) + + loops, _, err := r.provisionFileDevices(ctx, lvg) + assert.NoError(t, err) + assert.Equal(t, []string{"/dev/loop0"}, loops) +} + // cleanupFileDevices must walk the union of spec.fileDevices and // status.nodes[].fileDevices so files that were created mid-provision // (and therefore never landed in status) are not leaked when the @@ -1971,6 +2035,7 @@ func TestExtendFileDevicesIfNeeded_AddsNewLoopAsPV(t *testing.T) { gomock.InOrder( mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "", nil), mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fd.Directory).Return("mkdir -p ...", nil), + mc.EXPECT().GetAvailableBytes(gomock.Any(), fd.Directory).Return("stat -f ...", int64(1)<<50, nil), mc.EXPECT().CreateFileDevice(gomock.Any(), path, fd.Size.Value()).Return("fallocate ...", nil), mc.EXPECT().SetupLoopDevice(gomock.Any(), path).Return("losetup --find ...", "/dev/loop4", nil), mc.EXPECT().CreatePV("/dev/loop4").Return("pvcreate ...", nil), @@ -2127,6 +2192,7 @@ func TestExtendFileDevicesIfNeeded_RollsBackProvisionedOnExtendFailure(t *testin gomock.InOrder( mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "", nil), mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fd.Directory).Return("mkdir -p ...", nil), + mc.EXPECT().GetAvailableBytes(gomock.Any(), fd.Directory).Return("stat -f ...", int64(1)<<50, nil), mc.EXPECT().CreateFileDevice(gomock.Any(), path, fd.Size.Value()).Return("fallocate ...", nil), mc.EXPECT().SetupLoopDevice(gomock.Any(), path).Return("losetup --find ...", "/dev/loop4", nil), // Extend fails on pvcreate... @@ -2167,6 +2233,7 @@ func TestExtendFileDevicesIfNeeded_DoesNotRollBackLoopThatBecamePV(t *testing.T) gomock.InOrder( mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "", nil), mc.EXPECT().EnsureFileDeviceDirectory(gomock.Any(), fd.Directory).Return("mkdir -p ...", nil), + mc.EXPECT().GetAvailableBytes(gomock.Any(), fd.Directory).Return("stat -f ...", int64(1)<<50, nil), mc.EXPECT().CreateFileDevice(gomock.Any(), path, fd.Size.Value()).Return("fallocate ...", nil), mc.EXPECT().SetupLoopDevice(gomock.Any(), path).Return("losetup --find ...", "/dev/loop4", nil), mc.EXPECT().CreatePV("/dev/loop4").Return("pvcreate ...", errors.New("fd leaked on lvm invocation")), diff --git a/images/agent/internal/mock_utils/commands.go b/images/agent/internal/mock_utils/commands.go index 6dd393e7b..3f8540f98 100644 --- a/images/agent/internal/mock_utils/commands.go +++ b/images/agent/internal/mock_utils/commands.go @@ -60,6 +60,22 @@ func (mr *MockCommandsMockRecorder) EnsureFileDeviceDirectory(ctx, directory any return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureFileDeviceDirectory", reflect.TypeOf((*MockCommands)(nil).EnsureFileDeviceDirectory), ctx, directory) } +// GetAvailableBytes mocks base method. +func (m *MockCommands) GetAvailableBytes(ctx context.Context, directory string) (string, int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAvailableBytes", ctx, directory) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(int64) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetAvailableBytes indicates an expected call of GetAvailableBytes. +func (mr *MockCommandsMockRecorder) GetAvailableBytes(ctx, directory any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAvailableBytes", reflect.TypeOf((*MockCommands)(nil).GetAvailableBytes), ctx, directory) +} + // CreateFileDevice mocks base method. func (m *MockCommands) CreateFileDevice(ctx context.Context, path string, sizeBytes int64) (string, error) { m.ctrl.T.Helper() diff --git a/images/agent/internal/utils/commands.go b/images/agent/internal/utils/commands.go index 4fc7da049..2aa4b1125 100644 --- a/images/agent/internal/utils/commands.go +++ b/images/agent/internal/utils/commands.go @@ -79,6 +79,7 @@ type Commands interface { GetLoopBackingFile(ctx context.Context, loopDev string) (string, string, error) RemoveFileDevice(ctx context.Context, path string) (string, error) EnsureFileDeviceDirectory(ctx context.Context, directory string) (string, error) + GetAvailableBytes(ctx context.Context, directory string) (string, int64, error) } type commands struct { @@ -823,7 +824,17 @@ func (commands) SetupLoopDevice(ctx context.Context, filePath string) (string, s // reattach and the reconciler's provision step — or an existing // attachment that FindLoopDeviceByFile reported only under an alias — // would leak an extra loop device that nothing ever reaps. - args := nsentrerExpendedArgs("/sbin/losetup", "--find", "--nooverlap", "--show", filePath) + // + // --direct-io=on tells the loop driver to open the backing file with + // O_DIRECT so reads/writes bypass the backing filesystem's page cache. + // Our stack layers a filesystem on top of LVM on top of the loop on top + // of the node's filesystem; without direct I/O every page is cached + // twice (once for the volume's filesystem, once for the backing file), + // doubling the RAM footprint and throttling throughput. The kernel + // silently falls back to buffered I/O when the backing file's offset or + // sector size is not aligned (drivers/block/loop.c), so requesting it is + // always safe — at worst it is ignored. + args := nsentrerExpendedArgs("/sbin/losetup", "--find", "--nooverlap", "--direct-io=on", "--show", filePath) cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) var stdout, stderr bytes.Buffer @@ -941,6 +952,58 @@ func (commands) EnsureFileDeviceDirectory(ctx context.Context, directory string) return cmd.String(), nil } +// GetAvailableBytes returns the number of bytes that can be allocated in +// directory's filesystem without dipping into the superuser-reserved +// blocks, so the caller can refuse an oversized backing file before +// `fallocate` either fails halfway or fills the node's root filesystem and +// trips kubelet's DiskPressure eviction. +// +// It reads the value with `stat -f` in PID 1's mount namespace (the agent +// itself does not share the host mount namespace, so a Go syscall.Statfs +// would measure the wrong filesystem): %S is the fundamental block size and +// %a is the count of blocks available to a non-superuser. Using %a rather +// than %f deliberately leaves the filesystem's reserve untouched, which +// doubles as built-in headroom for the node. +func (commands) GetAvailableBytes(ctx context.Context, directory string) (string, int64, error) { + args := nsentrerExpendedArgs("/usr/bin/stat", "-f", "-c", "%S %a", directory) + cmd := exec.CommandContext(ctx, internal.NSENTERCmd, args...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return cmd.String(), 0, fmt.Errorf("unable to stat filesystem for %q: %w, stderr: %s", directory, err, stderr.String()) + } + + available, err := parseStatfsAvailableBytes(stdout.String()) + if err != nil { + return cmd.String(), 0, fmt.Errorf("unable to parse free space for %q: %w", directory, err) + } + return cmd.String(), available, nil +} + +// parseStatfsAvailableBytes parses the " " +// output of `stat -f -c "%S %a"` into the number of available bytes. +func parseStatfsAvailableBytes(out string) (int64, error) { + fields := strings.Fields(out) + if len(fields) != 2 { + return 0, fmt.Errorf("expected 2 fields, got %q", strings.TrimSpace(out)) + } + blockSize, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid block size %q: %w", fields[0], err) + } + availBlocks, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid available block count %q: %w", fields[1], err) + } + if blockSize < 0 || availBlocks < 0 { + return 0, fmt.Errorf("negative block size %d or available block count %d", blockSize, availBlocks) + } + return blockSize * availBlocks, nil +} + func unmarshalPVs(out []byte) ([]internal.PVData, error) { var pvR internal.PVReport diff --git a/images/agent/internal/utils/commands_test.go b/images/agent/internal/utils/commands_test.go index 7dad042b9..69bab0ff7 100644 --- a/images/agent/internal/utils/commands_test.go +++ b/images/agent/internal/utils/commands_test.go @@ -539,6 +539,40 @@ func TestSanitizeBackingFilePath(t *testing.T) { } } +// parseStatfsAvailableBytes turns the " " +// output of `stat -f -c "%S %a"` into a byte count; GetAvailableBytes relies +// on it to refuse an oversized backing file before fallocate fills the node. +func TestParseStatfsAvailableBytes(t *testing.T) { + tests := []struct { + name string + in string + want int64 + wantErr bool + }{ + {"plain", "4096 1000", 4096 * 1000, false}, + {"trailing newline", "4096 1000\n", 4096 * 1000, false}, + {"extra whitespace", " 4096 1000 ", 4096 * 1000, false}, + {"zero available", "4096 0", 0, false}, + {"empty", "", 0, true}, + {"single field", "4096", 0, true}, + {"too many fields", "4096 1000 7", 0, true}, + {"non-numeric block size", "abc 1000", 0, true}, + {"non-numeric available", "4096 abc", 0, true}, + {"negative available", "4096 -1", 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseStatfsAvailableBytes(tt.in) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + func TestFilterStdErr(t *testing.T) { tests := []struct { name string From 7e40ca488eaa8d8598120d0e09bd0e3da13678b9 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Tue, 30 Jun 2026 11:52:19 +0300 Subject: [PATCH 30/32] feat(agent): confine file-device backing files to a configurable base dir spec.fileDevices[].directory previously accepted any absolute host path, so a typo could place a backing file under /, /etc or /var/lib/kubelet and fill the node's filesystem. The agent now confines backing files to a base directory, defaulting to /opt/deckhouse/sds/file-devices and overridable via the new fileDevicesDirectory module setting (e.g. to point at a dedicated data disk). Each directory must be that base or a subdirectory of it; anything outside the subtree is rejected on the VGConfigurationApplied condition. - openapi: new config-values fileDevicesDirectory (+ru doc) - daemonset: plumb it to the agent as FILE_DEVICES_DIRECTORY - agent config: read the env with the /opt/deckhouse/sds/file-devices default - lvg reconciler: validateFileDevice enforces the base-dir allowlist - docs/CRD: document the constraint and update examples Co-Authored-By: Claude Opus 4.8 (1M context) --- crds/lvmvolumegroup.yaml | 8 ++- docs/FAQ.md | 3 +- docs/RESOURCES.md | 6 +- images/agent/cmd/main.go | 1 + images/agent/internal/config/config.go | 13 +++++ images/agent/internal/config/config_test.go | 28 ++++++++++ .../internal/controller/lvg/reconciler.go | 26 +++++++++ .../controller/lvg/reconciler_test.go | 55 +++++++++++++++++++ openapi/config-values.yaml | 13 +++++ openapi/doc-ru-config-values.yaml | 11 ++++ templates/agent/daemonset.yaml | 2 + 11 files changed, 162 insertions(+), 4 deletions(-) diff --git a/crds/lvmvolumegroup.yaml b/crds/lvmvolumegroup.yaml index ed34904b4..3e2c33d15 100644 --- a/crds/lvmvolumegroup.yaml +++ b/crds/lvmvolumegroup.yaml @@ -155,10 +155,16 @@ spec: Absolute directory on the node's filesystem where the backing file will be created. + Must be the module's configured base directory + (fileDevicesDirectory, default /opt/deckhouse/sds/file-devices) or a + subdirectory of it; the agent rejects paths outside that subtree so a + stray entry cannot fill an arbitrary host path. + The agent creates the directory automatically (mkdir -p) if it does not exist. The path must be absolute and must not contain '..' segments. Provisioning fails if the path is on a read-only - filesystem or a non-directory component is in the way. + filesystem, a non-directory component is in the way, or there is not + enough free space for the requested size. size: type: string maxLength: 32 diff --git a/docs/FAQ.md b/docs/FAQ.md index c6967b48b..f30261af1 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -157,9 +157,10 @@ File-backed devices allow you to allocate part of an existing filesystem for LVM ### Limitations +- **Confined to a base directory**: Each `directory` must be the `fileDevicesDirectory` module setting (default `/opt/deckhouse/sds/file-devices`) or a subdirectory of it. Paths outside this subtree are rejected so an arbitrary host path cannot be filled up. Point `fileDevicesDirectory` at a dedicated data disk to use a different location. - **Directory auto-created**: The agent creates the backing directory automatically (`mkdir -p`) on the node if it does not exist. The path must be absolute and free of `..` segments; provisioning fails only if the path is on a read-only filesystem or a non-directory component is in the way. - **No resize**: File device size cannot be changed after creation. To increase capacity, add a new `fileDevices` entry. -- **Preallocated only**: Files are created with `fallocate`, which preallocates space on the filesystem. The directory must have enough free space. +- **Preallocated only**: Files are created with `fallocate`, which preallocates space on the filesystem. The agent refuses to create a backing file larger than the directory's free space, so a too-large entry is reported on the resource instead of filling the node. - **Minimum size**: Each file device must be at least 1Gi. - **Performance overhead**: LVM on a loop device over a filesystem adds double indirection. Use file-backed devices only when dedicated disks are not available. - **Host LVM filter**: NodeGroupConfiguration adds loop devices to the host-wide LVM `global_filter`, so unprivileged `lvm`/`pvs` on the node do not see them. The agent re-attaches managed backing files at startup and uses its own LVM config for managed Volume Groups. diff --git a/docs/RESOURCES.md b/docs/RESOURCES.md index 185eca97f..4948fe53a 100644 --- a/docs/RESOURCES.md +++ b/docs/RESOURCES.md @@ -199,6 +199,8 @@ All selected block devices must belong to the same node for [LVMVolumeGroup](./c When dedicated block devices are not available, you can allocate part of an existing filesystem for LVM by using file-backed devices (`spec.fileDevices`). The agent will create a preallocated file, attach it as a loop device, and use it as an LVM Physical Volume. +Each `spec.fileDevices[].directory` must be inside the base directory configured by the `fileDevicesDirectory` module setting (default `/opt/deckhouse/sds/file-devices`); the agent rejects paths outside it so a stray entry cannot fill an arbitrary host path. To place backing files on a dedicated data disk, mount the disk on the node and set `fileDevicesDirectory` to its mount point in the module configuration. + Creating a Volume Group backed by a 50Gi file on an existing filesystem: ```yaml @@ -212,7 +214,7 @@ spec: nodeName: "node-0" actualVGNameOnTheNode: "vg-file" fileDevices: - - directory: /data/lvm-backing + - directory: /opt/deckhouse/sds/file-devices size: 50Gi ``` @@ -235,7 +237,7 @@ spec: - dev-07ad52cef2348996b72db262011f1b5f896bb68f actualVGNameOnTheNode: "vg-mixed" fileDevices: - - directory: /data/lvm-backing + - directory: /opt/deckhouse/sds/file-devices size: 100Gi ``` diff --git a/images/agent/cmd/main.go b/images/agent/cmd/main.go index 6952b40e6..adabfa04d 100644 --- a/images/agent/cmd/main.go +++ b/images/agent/cmd/main.go @@ -247,6 +247,7 @@ func run() int { VolumeGroupScanInterval: cfgParams.VolumeGroupScanInterval, BlockDeviceScanInterval: cfgParams.BlockDeviceScanInterval, CmdDeadlineDuration: cfgParams.CmdDeadlineDuration, + FileDevicesDirectory: cfgParams.FileDevicesDirectory, }, ), ) diff --git a/images/agent/internal/config/config.go b/images/agent/internal/config/config.go index 79a6a762d..db6a9cd15 100644 --- a/images/agent/internal/config/config.go +++ b/images/agent/internal/config/config.go @@ -39,6 +39,13 @@ const ( CmdDeadlineDuration = "CMD_DEADLINE_DURATION" DefaultHealthProbeBindAddressEnvName = "HEALTH_PROBE_BIND_ADDRESS" DefaultHealthProbeBindAddress = ":4228" + FileDevicesDirectory = "FILE_DEVICES_DIRECTORY" + // DefaultFileDevicesDirectory is the base directory backing files are + // confined to when the module config does not override it. It lives under + // the module's own /opt/deckhouse/sds tree so a stray fileDevices entry + // cannot fill an arbitrary host path. Keep in sync with the + // fileDevicesDirectory default in openapi/config-values.yaml. + DefaultFileDevicesDirectory = "/opt/deckhouse/sds/file-devices" ) type Config struct { @@ -53,6 +60,7 @@ type Config struct { ThrottleInterval time.Duration CmdDeadlineDuration time.Duration HealthProbeBindAddress string + FileDevicesDirectory string } func NewConfig() (*Config, error) { @@ -86,6 +94,11 @@ func NewConfig() (*Config, error) { cfg.HealthProbeBindAddress = DefaultHealthProbeBindAddress } + cfg.FileDevicesDirectory = os.Getenv(FileDevicesDirectory) + if cfg.FileDevicesDirectory == "" { + cfg.FileDevicesDirectory = DefaultFileDevicesDirectory + } + scanInt := os.Getenv(ScanInterval) if scanInt == "" { cfg.BlockDeviceScanInterval = 5 * time.Second diff --git a/images/agent/internal/config/config_test.go b/images/agent/internal/config/config_test.go index 0fc4aba21..496483173 100644 --- a/images/agent/internal/config/config_test.go +++ b/images/agent/internal/config/config_test.go @@ -109,6 +109,34 @@ func TestNewConfig(t *testing.T) { assert.EqualError(t, err, expErrorMsg) }) + t.Run("FileDevicesDirectoryNotSet_ReturnsDefault", func(t *testing.T) { + err := os.Setenv(NodeName, "test-node") + assert.NoError(t, err) + err = os.Setenv(MachineID, "test-id") + assert.NoError(t, err) + defer os.Clearenv() + + opts, err := NewConfig() + if assert.NoError(t, err) { + assert.Equal(t, DefaultFileDevicesDirectory, opts.FileDevicesDirectory) + } + }) + + t.Run("FileDevicesDirectorySet_IsHonoured", func(t *testing.T) { + err := os.Setenv(NodeName, "test-node") + assert.NoError(t, err) + err = os.Setenv(MachineID, "test-id") + assert.NoError(t, err) + err = os.Setenv(FileDevicesDirectory, "/mnt/data/file-devices") + assert.NoError(t, err) + defer os.Clearenv() + + opts, err := NewConfig() + if assert.NoError(t, err) { + assert.Equal(t, "/mnt/data/file-devices", opts.FileDevicesDirectory) + } + }) + t.Run("MetricsPortNotSet_ReturnsDefaultPort", func(t *testing.T) { expNodeName := "test-node" expMetricsPort := ":4202" diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index b95b36666..c07d6aaf3 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -116,6 +116,10 @@ type ReconcilerConfig struct { BlockDeviceScanInterval time.Duration VolumeGroupScanInterval time.Duration CmdDeadlineDuration time.Duration + // FileDevicesDirectory is the base directory backing files are confined + // to; spec.fileDevices[].directory must be this path or a subdirectory of + // it. Empty means "no restriction" (used by unit tests that do not care). + FileDevicesDirectory string } func NewReconciler( @@ -898,6 +902,17 @@ func (r *Reconciler) validateLVGForCreateFunc( var minFileDeviceSize = resource.MustParse("1Gi") +// isWithinBaseDir reports whether dir is base or a descendant of it. Both are +// cleaned before comparison so trailing slashes and redundant separators do +// not matter. dir is expected to be absolute and already free of '..' +// segments (validateFileDevice enforces that first), so the prefix check +// cannot be fooled into escaping base. +func isWithinBaseDir(dir, base string) bool { + base = filepath.Clean(base) + dir = filepath.Clean(dir) + return dir == base || strings.HasPrefix(dir, base+string(filepath.Separator)) +} + func (r *Reconciler) validateFileDevices( ctx context.Context, lvg *v1alpha1.LVMVolumeGroup, @@ -942,6 +957,17 @@ func (r *Reconciler) validateFileDevice( return } + // Confine backing files to the configured base directory (module config + // fileDevicesDirectory, default /opt/deckhouse/sds/file-devices). Without + // this an arbitrary host path — `/`, `/etc`, `/var/lib/kubelet` — could be + // targeted, and one oversized file there fills the node's filesystem and + // trips kubelet DiskPressure eviction. An empty base disables the check + // (unit tests that do not exercise the allowlist). + if r.cfg.FileDevicesDirectory != "" && !isWithinBaseDir(cleaned, r.cfg.FileDevicesDirectory) { + fmt.Fprintf(reason, "fileDevices[%d].directory %q must be %q or a subdirectory of it. ", index, fd.Directory, r.cfg.FileDevicesDirectory) + return + } + if fd.Size.Value() < minFileDeviceSize.Value() { // A common cause is a decimal unit (e.g. "1G" = 10^9 bytes) where a // binary unit was meant ("1Gi" = 2^30 bytes); the former is below the diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 9e5106796..5c3b7b3d2 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -1676,6 +1676,61 @@ func TestValidateFileDevice(t *testing.T) { assert.Contains(t, reason.String(), "must not contain '..'") }) + t.Run("rejects_directory_outside_base", func(t *testing.T) { + r := setupReconciler() + r.cfg.FileDevicesDirectory = "/opt/deckhouse/sds/file-devices" + var reason strings.Builder + totalSize := resource.MustParse("0") + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ + Directory: "/var/lib/kubelet", + Size: resource.MustParse("10Gi"), + } + r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) + assert.Contains(t, reason.String(), "must be \"/opt/deckhouse/sds/file-devices\" or a subdirectory") + }) + + t.Run("rejects_base_prefix_sibling", func(t *testing.T) { + // A path that shares the base as a string prefix but is not actually + // inside it (e.g. "/opt/deckhouse/sds/file-devices-evil") must be + // rejected — the separator check guards against this. + r := setupReconciler() + r.cfg.FileDevicesDirectory = "/opt/deckhouse/sds/file-devices" + var reason strings.Builder + totalSize := resource.MustParse("0") + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ + Directory: "/opt/deckhouse/sds/file-devices-evil", + Size: resource.MustParse("10Gi"), + } + r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) + assert.Contains(t, reason.String(), "or a subdirectory of it") + }) + + t.Run("accepts_base_directory_itself", func(t *testing.T) { + r := setupReconciler() + r.cfg.FileDevicesDirectory = "/opt/deckhouse/sds/file-devices" + var reason strings.Builder + totalSize := resource.MustParse("0") + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ + Directory: "/opt/deckhouse/sds/file-devices", + Size: resource.MustParse("10Gi"), + } + r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) + assert.Empty(t, reason.String()) + }) + + t.Run("accepts_subdirectory_of_base", func(t *testing.T) { + r := setupReconciler() + r.cfg.FileDevicesDirectory = "/opt/deckhouse/sds/file-devices" + var reason strings.Builder + totalSize := resource.MustParse("0") + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{ + Directory: "/opt/deckhouse/sds/file-devices/vg-a", + Size: resource.MustParse("10Gi"), + } + r.validateFileDevice(ctx, fd, 0, &reason, &totalSize) + assert.Empty(t, reason.String()) + }) + t.Run("rejects_duplicate_backing_file", func(t *testing.T) { r := setupReconciler() lvg := &v1alpha1.LVMVolumeGroup{ diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 99cfa31ba..f31104798 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -19,3 +19,16 @@ properties: type: boolean default: true description: Enable support of thin LVM volumes. + fileDevicesDirectory: + type: string + default: /opt/deckhouse/sds/file-devices + pattern: '^/.*$' + description: | + Base directory on each node under which the agent is allowed to create + backing files for file-backed LVM devices (`spec.fileDevices`). + + Every `spec.fileDevices[].directory` must be this directory or a + subdirectory of it; the agent rejects backing files outside this subtree + to prevent an arbitrary host path (for example `/` or `/var/lib/kubelet`) + from being filled up. Override it to point at a dedicated data disk + mounted on the node when the default location does not have enough space. diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index d33875657..de24c5a1d 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -6,3 +6,14 @@ properties: description: Уровень логирования для приложений модуля. enableThinProvisioning: description: Включить поддержку thin-LVM-томов. + fileDevicesDirectory: + description: | + Базовый каталог на каждом узле, внутри которого агенту разрешено создавать + файлы-подложки для файловых LVM-устройств (`spec.fileDevices`). + + Каждый `spec.fileDevices[].directory` должен совпадать с этим каталогом + либо быть его подкаталогом; файлы-подложки вне этого поддерева агент + отклоняет, чтобы произвольный путь на узле (например, `/` или + `/var/lib/kubelet`) нельзя было переполнить. Переопределите параметр, + указав отдельный диск с данными, смонтированный на узле, если в каталоге + по умолчанию недостаточно места. diff --git a/templates/agent/daemonset.yaml b/templates/agent/daemonset.yaml index bcf76d99b..367a2095d 100644 --- a/templates/agent/daemonset.yaml +++ b/templates/agent/daemonset.yaml @@ -196,6 +196,8 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: FILE_DEVICES_DIRECTORY + value: {{ .Values.sdsNodeConfigurator.fileDevicesDirectory | quote }} - name: LOG_LEVEL {{- if eq .Values.sdsNodeConfigurator.logLevel "ERROR" }} value: "0" From 519a5e2b6e9c22af95888d4f399025e926e4f393 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Wed, 1 Jul 2026 07:27:45 +0300 Subject: [PATCH 31/32] fix(agent): escalate stuck alias-PV resolution and document space reclaim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #6: extendFileDevicesIfNeeded previously requeued forever under the generic "Updating" reason whenever alias-form PV names could not be resolved, so a persistently broken resolver (missing nsenter binary, a dangling alias) was indistinguishable from an ordinary in-flight update and could never be alerted on. The reconciler now tracks consecutive resolver-only no-progress rounds per LVG and, after a threshold, switches the VGConfigurationApplied condition to a dedicated ReasonAliasResolutionFailed and logs at Error level. The streak resets as soon as a round makes progress, has nothing to do, or the LVG is deleted. #4: document space reclamation for file-backed devices in the FAQ — the discard/TRIM passdown chain (fstrim or mount -o discard punches holes in the preallocated backing file via the default discards=passdown thin pool), plus its caveats (chunk-size granularity, snapshots pinning chunks, shrink only after write+delete+trim). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/FAQ.md | 19 +++++ images/agent/internal/const.go | 7 ++ .../internal/controller/lvg/reconciler.go | 72 ++++++++++++++++-- .../controller/lvg/reconciler_test.go | 73 +++++++++++++++++++ 4 files changed, 163 insertions(+), 8 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index f30261af1..e50cfb583 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -165,6 +165,25 @@ File-backed devices allow you to allocate part of an existing filesystem for LVM - **Performance overhead**: LVM on a loop device over a filesystem adds double indirection. Use file-backed devices only when dedicated disks are not available. - **Host LVM filter**: NodeGroupConfiguration adds loop devices to the host-wide LVM `global_filter`, so unprivileged `lvm`/`pvs` on the node do not see them. The agent re-attaches managed backing files at startup and uses its own LVM config for managed Volume Groups. +### Reclaiming space + +Backing files are preallocated with `fallocate`, so a file occupies its full size on the node's filesystem from the moment it is created and does **not** grow or shrink automatically as data is written or deleted inside the volume. + +Space can still be returned to the node's filesystem through the discard (TRIM) chain, which works end to end for these devices: + +`filesystem on the volume` → `thin LV` → `thin pool` (created with `discards=passdown` by default) → `/dev/loopN` → backing file. + +A loop device translates discards into `FALLOC_FL_PUNCH_HOLE` on its backing file, turning the reclaimed regions into holes and making the file sparse. To trigger reclamation after deleting data: + +- run `fstrim ` on the volume's filesystem periodically (for example via the `fstrim.timer` systemd unit), or +- mount the volume with `-o discard` for continuous (online) discard. + +Caveats: + +- Only whole thin-pool chunks are reclaimed, so the effect depends on the pool's chunk size and alignment. +- Snapshots pin the chunks they reference, so space shared with a snapshot is not freed until the snapshot is removed. +- The backing file only shrinks after a write → delete → trim cycle; a freshly created file always occupies its full preallocated size. + ### Reboot recovery After a node reboot, the agent automatically re-establishes loop device mappings before activating Volume Groups. The backing file path is stored in `status.nodes[].fileDevices[].filePath`. diff --git a/images/agent/internal/const.go b/images/agent/internal/const.go index 58f6929ff..955085d89 100644 --- a/images/agent/internal/const.go +++ b/images/agent/internal/const.go @@ -86,6 +86,13 @@ const ( ReasonScanFailed = "ScanFailed" ReasonUpdated = "Updated" ReasonApplied = "Applied" + // ReasonAliasResolutionFailed is set when the agent has repeatedly failed + // to canonicalize the alias-form PV names it needs to decide whether a + // file-backed loop device is already part of the VG. Unlike the transient + // ReasonUpdating requeue, this reason signals a stuck resolver (e.g. a + // missing nsenter binary or a genuinely broken alias) that will not clear + // on its own, so it can be alerted on distinctly. + ReasonAliasResolutionFailed = "AliasResolutionFailed" MetadataNameLabelKey = "kubernetes.io/metadata.name" HostNameLabelKey = "kubernetes.io/hostname" diff --git a/images/agent/internal/controller/lvg/reconciler.go b/images/agent/internal/controller/lvg/reconciler.go index c07d6aaf3..190523873 100644 --- a/images/agent/internal/controller/lvg/reconciler.go +++ b/images/agent/internal/controller/lvg/reconciler.go @@ -25,6 +25,7 @@ import ( "slices" "sort" "strings" + "sync" "time" "k8s.io/apimachinery/pkg/api/resource" @@ -109,8 +110,25 @@ type Reconciler struct { // under a /dev/disk/by-id or /dev/block/MAJ:MIN alias. Defaults to // utils.HostNsenterCanonicalResolver; overridable in tests. resolver utils.CanonicalPathResolver + + // aliasResolveFailures counts, per LVG name, how many consecutive + // extendFileDevicesIfNeeded rounds made no progress purely because + // alias-form PV names could not be resolved. It escalates the condition + // from a generic "Updating" retry to ReasonAliasResolutionFailed once the + // failures look persistent, so a stuck resolver is alertable instead of + // looking like an ordinary in-flight update. Reset on any round that makes + // progress or genuinely has nothing to do. The reconciler runs with + // MaxConcurrentReconciles==1, but the mutex keeps the map safe if that ever + // changes. + aliasResolveFailuresMu sync.Mutex + aliasResolveFailures map[string]int } +// aliasResolveFailureEscalationThreshold is the number of consecutive +// resolver-only failed rounds after which extendFileDevicesIfNeeded switches +// the VGConfigurationApplied reason to ReasonAliasResolutionFailed. +const aliasResolveFailureEscalationThreshold = 3 + type ReconcilerConfig struct { NodeName string BlockDeviceScanInterval time.Duration @@ -140,12 +158,13 @@ func NewReconciler( cfg.NodeName, ReconcilerName, ), - bdCl: repository.NewBDClient(cl, metrics), - metrics: metrics, - sdsCache: sdsCache, - cfg: cfg, - commands: commands, - resolver: utils.HostNsenterCanonicalResolver, + bdCl: repository.NewBDClient(cl, metrics), + metrics: metrics, + sdsCache: sdsCache, + cfg: cfg, + commands: commands, + resolver: utils.HostNsenterCanonicalResolver, + aliasResolveFailures: make(map[string]int), } } @@ -442,6 +461,7 @@ func (r *Reconciler) reconcileLVGDeleteFunc(ctx context.Context, lvg *v1alpha1.L return true, err } + r.resetAliasResolveFailure(lvg.Name) r.log.Info(fmt.Sprintf("[reconcileLVGDeleteFunc] successfully reconciled VG %s of the LVMVolumeGroup %s", lvg.Spec.ActualVGNameOnTheNode, lvg.Name)) return false, nil } @@ -1507,15 +1527,34 @@ func (r *Reconciler) extendFileDevicesIfNeeded( // was already attached, so provisionFileDevices reused it and recorded // nothing in `provisioned`. if skippedOnResolverFailure { - if err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "unable to resolve alias PV names to decide whether file-backed loop devices are already part of the VG; retrying"); err != nil { + // Escalate once the failure looks persistent: a resolver that stays + // broken (missing nsenter binary, a genuinely dangling alias) would + // otherwise requeue forever under the generic "Updating" reason, + // indistinguishable from an ordinary in-flight update. After a few + // consecutive no-progress rounds switch to a dedicated reason and + // log at Error level so it can be alerted on. + streak := r.noteAliasResolveFailure(lvg.Name) + reason := internal.ReasonUpdating + msg := "unable to resolve alias PV names to decide whether file-backed loop devices are already part of the VG; retrying" + if streak >= aliasResolveFailureEscalationThreshold { + reason = internal.ReasonAliasResolutionFailed + msg = fmt.Sprintf("unable to resolve alias PV names for %d consecutive reconciles; file devices cannot be added to VG %s until path resolution recovers (check the nsenter binary and PV aliases on the node)", streak, vg.VGName) + r.log.Error(fmt.Errorf("alias PV resolution stuck"), fmt.Sprintf("[extendFileDevicesIfNeeded] %s (LVMVolumeGroup %s)", msg, lvg.Name)) + } + if err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, reason, msg); err != nil { r.log.Error(err, fmt.Sprintf("[extendFileDevicesIfNeeded] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) } - return fmt.Errorf("unable to resolve alias PV names for VG %s; retrying", vg.VGName) + return fmt.Errorf("unable to resolve alias PV names for VG %s (attempt %d); retrying", vg.VGName, streak) } + r.resetAliasResolveFailure(lvg.Name) r.log.Debug(fmt.Sprintf("[extendFileDevicesIfNeeded] VG %s of the LVMVolumeGroup %s has no new file devices to add", vg.VGName, lvg.Name)) return nil } + // We resolved enough to make progress this round; clear any prior + // resolver-failure streak so a transient blip does not eventually escalate. + r.resetAliasResolveFailure(lvg.Name) + if isApplied(lvg) { if err := r.lvgCl.UpdateLVGConditionIfNeeded(ctx, lvg, v1.ConditionFalse, internal.TypeVGConfigurationApplied, internal.ReasonUpdating, "trying to apply the configuration"); err != nil { r.log.Error(err, fmt.Sprintf("[extendFileDevicesIfNeeded] unable to add the condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) @@ -1533,6 +1572,23 @@ func (r *Reconciler) extendFileDevicesIfNeeded( return nil } +// noteAliasResolveFailure records one more consecutive resolver-only failed +// round for lvgName and returns the new streak length. +func (r *Reconciler) noteAliasResolveFailure(lvgName string) int { + r.aliasResolveFailuresMu.Lock() + defer r.aliasResolveFailuresMu.Unlock() + r.aliasResolveFailures[lvgName]++ + return r.aliasResolveFailures[lvgName] +} + +// resetAliasResolveFailure clears the resolver-failure streak for lvgName +// after a round that made progress or genuinely had nothing to do. +func (r *Reconciler) resetAliasResolveFailure(lvgName string) { + r.aliasResolveFailuresMu.Lock() + defer r.aliasResolveFailuresMu.Unlock() + delete(r.aliasResolveFailures, lvgName) +} + // loopAlreadyRegisteredAsPV reports whether the canonical loop device is // already present in pvs under an alias name (e.g. /dev/disk/by-id/... or // /dev/block/MAJ:MIN). It only resolves alias-form PV names, so the common diff --git a/images/agent/internal/controller/lvg/reconciler_test.go b/images/agent/internal/controller/lvg/reconciler_test.go index 5c3b7b3d2..f1a3a16d4 100644 --- a/images/agent/internal/controller/lvg/reconciler_test.go +++ b/images/agent/internal/controller/lvg/reconciler_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "errors" + "fmt" "strings" "testing" "time" @@ -2225,6 +2226,78 @@ func TestExtendFileDevicesIfNeeded_SkipsOnResolverFailure(t *testing.T) { assert.Error(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) } +// A resolver that stays broken must not requeue silently forever: after a few +// consecutive no-progress rounds the failure streak escalates so the condition +// switches to ReasonAliasResolutionFailed and the error reports the attempt +// count. This is the observable "resolver broken" signal from review #6. +func TestExtendFileDevicesIfNeeded_EscalatesAfterRepeatedResolverFailure(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + r.resolver = func(_ context.Context, _ string) (string, error) { + return "", errors.New("nsenter readlink timed out") + } + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + vg := internal.VGData{VGName: "vg-test"} + pvs := []internal.PVData{{PVName: "/dev/disk/by-id/lvm-pv-uuid-xyz"}} + + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "/dev/loop4", nil).AnyTimes() + + for attempt := 1; attempt <= aliasResolveFailureEscalationThreshold; attempt++ { + err := r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), fmt.Sprintf("attempt %d", attempt)) + } + assert.Equal(t, attempt, r.aliasResolveFailures[lvg.Name]) + } + // At the threshold the streak reached the escalation point. + assert.GreaterOrEqual(t, r.aliasResolveFailures[lvg.Name], aliasResolveFailureEscalationThreshold) +} + +// A transient resolver blip must not accumulate toward escalation: once a +// round makes progress (or has nothing to do), the streak resets so the next +// failure starts counting from one again. +func TestExtendFileDevicesIfNeeded_ResolverFailureStreakResetsOnProgress(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mc := mock_utils.NewMockCommands(ctrl) + r := reconcilerWithMockedCommands(t, mc) + ctx := context.Background() + + fd := v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: "/data", Size: resource.MustParse("10Gi")} + lvg := &v1alpha1.LVMVolumeGroup{ + ObjectMeta: v1.ObjectMeta{Name: "vg-a"}, + Spec: v1alpha1.LVMVolumeGroupSpec{FileDevices: []v1alpha1.LVMVolumeGroupFileDeviceSpec{fd}}, + } + path := utils.BuildFileDevicePath(fd.Directory, "vg-a", fd.Size) + vg := internal.VGData{VGName: "vg-test"} + pvs := []internal.PVData{{PVName: "/dev/disk/by-id/lvm-pv-uuid-xyz"}} + + mc.EXPECT().FindLoopDeviceByFile(gomock.Any(), path).Return("losetup -j", "/dev/loop4", nil).AnyTimes() + + // Two failing rounds build a streak of 2. + r.resolver = func(_ context.Context, _ string) (string, error) { + return "", errors.New("nsenter readlink timed out") + } + assert.Error(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) + assert.Error(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, pvs)) + assert.Equal(t, 2, r.aliasResolveFailures[lvg.Name]) + + // A round that makes no-progress-but-not-due-to-resolver (the loop is a + // known canonical PV in the cache) clears the streak. + r.sdsCache.StorePVs([]internal.PVData{{PVName: "/dev/loop4"}}, bytes.Buffer{}) + assert.NoError(t, r.extendFileDevicesIfNeeded(ctx, lvg, vg, nil)) + assert.Equal(t, 0, r.aliasResolveFailures[lvg.Name]) +} + // When the extend step (pvcreate/vgextend) fails after a new file device was // freshly provisioned, the loop device and backing file created in THIS call // must be rolled back so a failed extend does not leak them on the node (and From 90e1367831eec3ad406a68f0d165fc7e91a92cc1 Mon Sep 17 00:00:00 2001 From: "v.oleynikov" Date: Wed, 1 Jul 2026 20:15:02 +0300 Subject: [PATCH 32/32] test(e2e): add file-backed devices (spec.fileDevices) e2e scenarios MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a "LVMVolumeGroup with file-backed devices" Ginkgo Context (label e2e-tests) plus helpers. Unlike the other scenarios these need no dedicated disk or nested VM: the VG is created with spec.fileDevices on any node that runs the agent, and node-side state is checked over SSH. Backing files go in the default base dir /opt/deckhouse/sds/file-devices. Scenarios: - create a file-only VG → Ready; assert status.fileDevices (filePath under the base dir, loopDevice, pvUUID) and that the backing file, loop device and VG exist on the node; on delete, assert the file is removed and the loop detached; - thin-pool on a file-backed device → pool Ready + data LV present on node; - extend a file-backed VG by appending a fileDevices entry → VG grows; - reattach after the agent pod restarts → VG stays Ready, loop reattached; - negative: a directory outside the allowed base dir is rejected with VGConfigurationApplied=False / ValidationFailed and nothing is provisioned. Documents the scenarios and their prerequisites (branch build, node free space) in e2e/README.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/README.md | 14 + ...e_configurator_helpers_filedevices_test.go | 149 ++++++++++ e2e/tests/sds_node_configurator_test.go | 266 +++++++++++++++++- 3 files changed, 422 insertions(+), 7 deletions(-) create mode 100644 e2e/tests/sds_node_configurator_helpers_filedevices_test.go diff --git a/e2e/README.md b/e2e/README.md index 353758f74..9c5951590 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -99,6 +99,20 @@ ginkgo -v --progress --label-filter=e2e-tests --focus="Should schedule Pod with - **BlockDevice discovery**: появление диска; корректные `status.nodeName`, `status.path`, `status.size`, `consumable`. - **LVMVolumeGroup**: создание на основе BlockDevice, статус и capacity. +### LVMVolumeGroup на файлах (fileDevices) + +Context `LVMVolumeGroup with file-backed devices` (Ginkgo-лейбл **`e2e-tests`**, файл `sds_node_configurator_test.go`, хелперы — `sds_node_configurator_helpers_filedevices_test.go`). В отличие от остальных сценариев **не требует отдельного диска / nested-VM**: LVG создаётся с `spec.fileDevices` на любой ноде с работающим агентом; проверки на ноде идут по SSH. Файлы кладутся в дефолтный базовый каталог `/opt/deckhouse/sds/file-devices`. + +Сценарии: + +- **Создание и удаление**: file-only VG (`1Gi`) → `Ready`; в `status.nodes[].fileDevices` — `filePath` под базовым каталогом, `loopDevice` `/dev/loop*`, `pvUUID`; на ноде существуют backing-файл, loop-устройство и VG. После удаления LVG — backing-файл удалён, loop отсоединён. +- **Thin-pool на файле**: file-only VG (`2Gi`) + thin-pool `50%` → `Ready`, data-LV thin-pool присутствует на ноде. +- **Расширение**: добавление второй записи `fileDevices` → в статусе две записи, `vgSize` растёт, `Ready`. +- **Reattach после рестарта агента**: рестарт пода агента → VG остаётся `Ready`, файл переподключён (loop снова привязан). +- **Негатив (allowlist)**: `directory` вне базового каталога (`/tmp/...`) → `VGConfigurationApplied=False`, reason `ValidationFailed`, VG и каталог на ноде не создаются. + +> Требования: кластер собран из этой ветки (поле `fileDevices` и настройка `fileDevicesDirectory` существуют только здесь); на выбранной ноде в `/opt/deckhouse/sds` должно быть ≥3Gi свободного места (иначе сработает проверка свободного места агента). + ### Stress: максимум VG на одной ноде Spec в `sds_node_configurator_stress_max_vgs_test.go`, Ginkgo-лейбл **`stress-test`** (остальные e2e — **`e2e-tests`**). CI smoke по умолчанию: `-ginkgo.label-filter=e2e-tests`. Подробнее — [E2E_USAGE.md](E2E_USAGE.md). diff --git a/e2e/tests/sds_node_configurator_helpers_filedevices_test.go b/e2e/tests/sds_node_configurator_helpers_filedevices_test.go new file mode 100644 index 000000000..2796d22fb --- /dev/null +++ b/e2e/tests/sds_node_configurator_helpers_filedevices_test.go @@ -0,0 +1,149 @@ +/* + Copyright 2026 Flant JSC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package tests + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" +) + +const ( + // e2eFileDevicesBaseDir must match the fileDevicesDirectory module default + // (openapi/config-values.yaml) and DefaultFileDevicesDirectory in the agent. + // File-backed tests place backing files here so the agent's base-dir + // allowlist accepts them. + e2eFileDevicesBaseDir = "/opt/deckhouse/sds/file-devices" + + e2eSNCNamespace = "d8-sds-node-configurator" + e2eSNCAgentApp = "sds-node-configurator" +) + +// e2ePickNodeRunningAgent returns the node name of a Ready, Running +// sds-node-configurator agent pod. File-backed LVMVolumeGroups need no +// dedicated disk, so any node with a healthy agent can host the test VG. +func e2ePickNodeRunningAgent(ctx context.Context, cl client.Client) string { + var chosen string + Eventually(func(g Gomega) { + var pods corev1.PodList + g.Expect(cl.List(ctx, &pods, client.InNamespace(e2eSNCNamespace), client.MatchingLabels{"app": e2eSNCAgentApp})).To(Succeed()) + for i := range pods.Items { + p := &pods.Items[i] + if p.Spec.NodeName == "" || p.DeletionTimestamp != nil || p.Status.Phase != corev1.PodRunning || !isPodReady(p) { + continue + } + chosen = p.Spec.NodeName + return + } + g.Expect(chosen).NotTo(BeEmpty(), "no Ready sds-node-configurator agent pod found in %s", e2eSNCNamespace) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + return chosen +} + +// e2eFileDevicesForNode returns the file-device status entries the LVG reports +// for node (nil if the node is absent or has no file devices yet). +func e2eFileDevicesForNode(lvg *v1alpha1.LVMVolumeGroup, node string) []v1alpha1.LVMVolumeGroupFileDevice { + for i := range lvg.Status.Nodes { + if lvg.Status.Nodes[i].Name == node { + return lvg.Status.Nodes[i].FileDevices + } + } + return nil +} + +// e2eNodePathExists reports whether path exists on the node (checked as root, +// because backing files live under root-owned /opt/deckhouse/sds). +func e2eNodePathExists(ctx context.Context, kubeconfig *rest.Config, node, sshUser, path string) (bool, error) { + out, err := e2eExecOnTestClusterNodeSSH(ctx, kubeconfig, node, sshUser, + fmt.Sprintf("sudo -n sh -c 'test -e %q && echo EXISTS || echo MISSING'", path)) + if err != nil { + return false, err + } + return strings.Contains(out, "EXISTS"), nil +} + +// e2eLoopBoundToFileOnNode reports whether any loop device is currently +// attached to path on the node (via `losetup -j`). +func e2eLoopBoundToFileOnNode(ctx context.Context, kubeconfig *rest.Config, node, sshUser, path string) (bool, string, error) { + out, err := e2eExecOnTestClusterNodeSSH(ctx, kubeconfig, node, sshUser, + fmt.Sprintf("sudo -n losetup -j %q 2>/dev/null || losetup -j %q 2>/dev/null", path, path)) + if err != nil { + return false, out, err + } + return strings.TrimSpace(out) != "", out, nil +} + +// e2eVGListedOnNode reports whether vgName is visible to vgs on the node. +func e2eVGListedOnNode(ctx context.Context, kubeconfig *rest.Config, node, sshUser, vgName string) (bool, string, error) { + out, err := e2eExecOnTestClusterNodeSSH(ctx, kubeconfig, node, sshUser, + "vgs -o vg_name --noheadings 2>/dev/null || sudo -n vgs -o vg_name --noheadings 2>/dev/null") + if err != nil { + return false, out, err + } + return e2eVgNameListedInVgsOutput(out, vgName), out, nil +} + +// e2eCreateFileBackedLVGAndWaitReady creates lvg and waits (up to +// e2eLVMVolumeGroupReadyTimeout) until it reports Phase Ready, dumping +// conditions while it waits. Returns the refreshed object. +func e2eCreateFileBackedLVGAndWaitReady(ctx context.Context, cl client.Client, lvg *v1alpha1.LVMVolumeGroup) *v1alpha1.LVMVolumeGroup { + Expect(cl.Create(ctx, lvg)).To(Succeed()) + var created v1alpha1.LVMVolumeGroup + Eventually(func(g Gomega) { + g.Expect(cl.Get(ctx, client.ObjectKeyFromObject(lvg), &created)).To(Succeed()) + if created.Status.Phase != v1alpha1.PhaseReady { + GinkgoWriter.Printf("LVMVolumeGroup %s phase=%s (waiting for Ready)\n", lvg.Name, created.Status.Phase) + for _, c := range created.Status.Conditions { + GinkgoWriter.Printf(" condition %s status=%s reason=%s msg=%s\n", c.Type, c.Status, c.Reason, c.Message) + } + } + g.Expect(created.Status.Phase).To(Equal(v1alpha1.PhaseReady), "Phase should be Ready, got %s", created.Status.Phase) + }, e2eLVMVolumeGroupReadyTimeout, 10*time.Second).Should(Succeed()) + return &created +} + +// e2eExpectNoFalseConditions fails if any status condition is False. +func e2eExpectNoFalseConditions(lvg *v1alpha1.LVMVolumeGroup) { + for _, c := range lvg.Status.Conditions { + Expect(c.Status).NotTo(Equal(metav1.ConditionFalse), + "condition %s has status False: reason=%s message=%s", c.Type, c.Reason, c.Message) + } +} + +// e2eDeleteLVGAndWaitGone deletes the LVMVolumeGroup and waits until its CR is +// gone. The finalizer is only cleared after the agent detaches the loop +// devices and removes the backing files, so a successful delete also confirms +// node-side cleanup ran. +func e2eDeleteLVGAndWaitGone(ctx context.Context, cl client.Client, name string, timeout time.Duration) { + Expect(client.IgnoreNotFound(cl.Delete(ctx, &v1alpha1.LVMVolumeGroup{ObjectMeta: metav1.ObjectMeta{Name: name}}))).To(Succeed()) + Eventually(func(g Gomega) { + var cur v1alpha1.LVMVolumeGroup + err := cl.Get(ctx, client.ObjectKey{Name: name}, &cur) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue(), "LVMVolumeGroup %s should be deleted", name) + }, timeout, 8*time.Second).Should(Succeed()) +} diff --git a/e2e/tests/sds_node_configurator_test.go b/e2e/tests/sds_node_configurator_test.go index 8088561a9..50c87961b 100644 --- a/e2e/tests/sds_node_configurator_test.go +++ b/e2e/tests/sds_node_configurator_test.go @@ -2446,13 +2446,13 @@ var _ = Describe("sds-node-configurator module e2e", Label("e2e-tests"), Ordered targetVM := clusterVMs[rand.Intn(len(clusterVMs))] type multiDisk struct { - diskName string - diskSize string - att *kubernetes.VirtualDiskAttachmentResult - bd *v1alpha1.BlockDevice - meta string - lvgName string - vgName string + diskName string + diskSize string + att *kubernetes.VirtualDiskAttachmentResult + bd *v1alpha1.BlockDevice + meta string + lvgName string + vgName string } disks := make([]multiDisk, 0, nDisks) for i := 0; i < nDisks; i++ { @@ -3077,6 +3077,258 @@ sudo -n vgchange "$VG" --addtag storage.deckhouse.io/enabled=true 2>&1 }) }) + Context("LVMVolumeGroup with file-backed devices", Label("e2e-tests"), func() { + const ( + lvgConditionVGConfigurationApplied = "VGConfigurationApplied" + reasonValidationFailed = "ValidationFailed" + ) + + var vmSSH string + + BeforeEach(func() { + ensureE2EK8sClient(testClusterResources, &k8sClient, e2eCtx) + vmSSH = e2eConfigVMSSHUser() + }) + + // Helper: build a file-only LVMVolumeGroup (no blockDeviceSelector) on node. + newFileLVG := func(node, lvgName, vgName string, fds []v1alpha1.LVMVolumeGroupFileDeviceSpec, thinPools []v1alpha1.LVMVolumeGroupThinPoolSpec) *v1alpha1.LVMVolumeGroup { + return &v1alpha1.LVMVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{Name: lvgName}, + Spec: v1alpha1.LVMVolumeGroupSpec{ + ActualVGNameOnTheNode: vgName, + Type: "Local", + Local: v1alpha1.LVMVolumeGroupLocalSpec{NodeName: node}, + FileDevices: fds, + ThinPools: thinPools, + }, + } + } + + It("Should create a file-backed LVMVolumeGroup and remove backing files on delete", func() { + node := e2ePickNodeRunningAgent(e2eCtx, k8sClient) + runID := strconv.FormatInt(time.Now().UnixNano(), 10) + nodeSuffix := strings.NewReplacer(".", "-", "_", "-").Replace(node) + lvgName := "e2e-lvg-fd-" + runID + "-" + nodeSuffix + vgName := "e2e-vg-fd-" + runID + + lvg := newFileLVG(node, lvgName, vgName, + []v1alpha1.LVMVolumeGroupFileDeviceSpec{{Directory: e2eFileDevicesBaseDir, Size: resource.MustParse("1Gi")}}, nil) + + By(fmt.Sprintf("Creating file-backed LVMVolumeGroup %s on node %s (dir %s)", lvgName, node, e2eFileDevicesBaseDir)) + deleted := false + defer func() { + if !deleted { + e2eDeleteLVGAndWaitGone(e2eCtx, k8sClient, lvgName, 10*time.Minute) + } + }() + created := e2eCreateFileBackedLVGAndWaitReady(e2eCtx, k8sClient, lvg) + + By("Verifying conditions (no errors)") + e2eExpectNoFalseConditions(created) + + By("Verifying status.nodes[].fileDevices") + fds := e2eFileDevicesForNode(created, node) + Expect(fds).To(HaveLen(1), "expected exactly one file device in status for node %s", node) + fd := fds[0] + Expect(fd.FilePath).To(HavePrefix(e2eFileDevicesBaseDir+"/"), "backing file must live under the base dir") + Expect(fd.LoopDevice).To(HavePrefix("/dev/loop"), "loop device path") + Expect(fd.PVUuid).NotTo(BeEmpty(), "file device should be a PV") + printLVMVolumeGroupInfo(created) + + By("Verifying backing file, loop attachment and VG exist on the node") + exists, err := e2eNodePathExists(e2eCtx, testClusterResources.Kubeconfig, node, vmSSH, fd.FilePath) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue(), "backing file %s should exist on node", fd.FilePath) + bound, loopOut, err := e2eLoopBoundToFileOnNode(e2eCtx, testClusterResources.Kubeconfig, node, vmSSH, fd.FilePath) + Expect(err).NotTo(HaveOccurred()) + Expect(bound).To(BeTrue(), "a loop device should be attached to %s; losetup:\n%s", fd.FilePath, loopOut) + vgListed, vgsOut, err := e2eVGListedOnNode(e2eCtx, testClusterResources.Kubeconfig, node, vmSSH, vgName) + Expect(err).NotTo(HaveOccurred()) + Expect(vgListed).To(BeTrue(), "VG %s should be visible on node; vgs:\n%s", vgName, vgsOut) + + By("Deleting the LVMVolumeGroup and verifying node-side cleanup") + filePath := fd.FilePath + e2eDeleteLVGAndWaitGone(e2eCtx, k8sClient, lvgName, 10*time.Minute) + deleted = true + Eventually(func(g Gomega) { + stillThere, err := e2eNodePathExists(e2eCtx, testClusterResources.Kubeconfig, node, vmSSH, filePath) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(stillThere).To(BeFalse(), "backing file %s should be removed after delete", filePath) + }, 3*time.Minute, 10*time.Second).Should(Succeed()) + stillBound, _, err := e2eLoopBoundToFileOnNode(e2eCtx, testClusterResources.Kubeconfig, node, vmSSH, filePath) + Expect(err).NotTo(HaveOccurred()) + Expect(stillBound).To(BeFalse(), "loop device should be detached from %s after delete", filePath) + + By("✓ File-backed VG created (file+loop+PV+VG), then fully cleaned up on delete") + }) + + It("Should create a thin-pool on a file-backed device", func() { + node := e2ePickNodeRunningAgent(e2eCtx, k8sClient) + runID := strconv.FormatInt(time.Now().UnixNano(), 10) + nodeSuffix := strings.NewReplacer(".", "-", "_", "-").Replace(node) + lvgName := "e2e-lvg-fdtp-" + runID + "-" + nodeSuffix + vgName := "e2e-vg-fdtp-" + runID + thinPoolName := "e2e-thin-fd" + + lvg := newFileLVG(node, lvgName, vgName, + []v1alpha1.LVMVolumeGroupFileDeviceSpec{{Directory: e2eFileDevicesBaseDir, Size: resource.MustParse("2Gi")}}, + []v1alpha1.LVMVolumeGroupThinPoolSpec{{Name: thinPoolName, Size: "50%", AllocationLimit: "100%"}}) + + By(fmt.Sprintf("Creating file-backed LVMVolumeGroup %s with thin-pool %s on node %s", lvgName, thinPoolName, node)) + defer func() { e2eDeleteLVGAndWaitGone(e2eCtx, k8sClient, lvgName, 10*time.Minute) }() + created := e2eCreateFileBackedLVGAndWaitReady(e2eCtx, k8sClient, lvg) + e2eExpectNoFalseConditions(created) + printLVMVolumeGroupInfo(created) + + By("Verifying the thin-pool is Ready in status") + var tp *v1alpha1.LVMVolumeGroupThinPoolStatus + for i := range created.Status.ThinPools { + if created.Status.ThinPools[i].Name == thinPoolName { + tp = &created.Status.ThinPools[i] + break + } + } + Expect(tp).NotTo(BeNil(), "thin-pool %q not found in status", thinPoolName) + Expect(tp.Ready).To(BeTrue(), "thin-pool should be Ready") + + By("Verifying the thin-pool data LV exists on the node") + present, out, err := e2eThinPoolDataLVPresentOnNode(e2eCtx, testClusterResources.Kubeconfig, node, vmSSH, vgName, thinPoolName) + Expect(err).NotTo(HaveOccurred()) + Expect(present).To(BeTrue(), "thin-pool data LV for %s/%s should exist on node; lvs:\n%s", vgName, thinPoolName, out) + + By("✓ Thin-pool created on a file-backed VG and reported Ready") + }) + + It("Should extend a file-backed VG by adding a fileDevices entry", func() { + node := e2ePickNodeRunningAgent(e2eCtx, k8sClient) + runID := strconv.FormatInt(time.Now().UnixNano(), 10) + nodeSuffix := strings.NewReplacer(".", "-", "_", "-").Replace(node) + lvgName := "e2e-lvg-fdext-" + runID + "-" + nodeSuffix + vgName := "e2e-vg-fdext-" + runID + + lvg := newFileLVG(node, lvgName, vgName, + []v1alpha1.LVMVolumeGroupFileDeviceSpec{{Directory: e2eFileDevicesBaseDir, Size: resource.MustParse("1Gi")}}, nil) + + By(fmt.Sprintf("Creating file-backed LVMVolumeGroup %s with one file device on node %s", lvgName, node)) + defer func() { e2eDeleteLVGAndWaitGone(e2eCtx, k8sClient, lvgName, 10*time.Minute) }() + created := e2eCreateFileBackedLVGAndWaitReady(e2eCtx, k8sClient, lvg) + Expect(e2eFileDevicesForNode(created, node)).To(HaveLen(1)) + vgSizeBefore := created.Status.VGSize.Value() + + By("Appending a second fileDevices entry (immutable existing entry + new one)") + Eventually(func(g Gomega) { + var cur v1alpha1.LVMVolumeGroup + g.Expect(k8sClient.Get(e2eCtx, client.ObjectKey{Name: lvgName}, &cur)).To(Succeed()) + // A different size yields a different backing-file path, so the + // new entry does not collide with the existing 1Gi one. + cur.Spec.FileDevices = append(cur.Spec.FileDevices, + v1alpha1.LVMVolumeGroupFileDeviceSpec{Directory: e2eFileDevicesBaseDir, Size: resource.MustParse("2Gi")}) + g.Expect(k8sClient.Update(e2eCtx, &cur)).To(Succeed()) + }, 1*time.Minute, 5*time.Second).Should(Succeed()) + + By("Waiting for the VG to grow with the second file device") + var extended v1alpha1.LVMVolumeGroup + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(e2eCtx, client.ObjectKey{Name: lvgName}, &extended)).To(Succeed()) + g.Expect(extended.Status.Phase).To(Equal(v1alpha1.PhaseReady), "phase should return to Ready after extend") + g.Expect(e2eFileDevicesForNode(&extended, node)).To(HaveLen(2), "status should list two file devices") + g.Expect(extended.Status.VGSize.Value()).To(BeNumerically(">", vgSizeBefore), "VG size should grow after extend") + }, e2eLVMVolumeGroupReadyTimeout, 10*time.Second).Should(Succeed()) + e2eExpectNoFalseConditions(&extended) + printLVMVolumeGroupInfo(&extended) + + By("✓ File-backed VG extended by adding a new fileDevices entry") + }) + + It("Should reattach file devices after the agent restarts", func() { + node := e2ePickNodeRunningAgent(e2eCtx, k8sClient) + runID := strconv.FormatInt(time.Now().UnixNano(), 10) + nodeSuffix := strings.NewReplacer(".", "-", "_", "-").Replace(node) + lvgName := "e2e-lvg-fdreat-" + runID + "-" + nodeSuffix + vgName := "e2e-vg-fdreat-" + runID + + lvg := newFileLVG(node, lvgName, vgName, + []v1alpha1.LVMVolumeGroupFileDeviceSpec{{Directory: e2eFileDevicesBaseDir, Size: resource.MustParse("1Gi")}}, nil) + + By(fmt.Sprintf("Creating file-backed LVMVolumeGroup %s on node %s", lvgName, node)) + defer func() { e2eDeleteLVGAndWaitGone(e2eCtx, k8sClient, lvgName, 10*time.Minute) }() + created := e2eCreateFileBackedLVGAndWaitReady(e2eCtx, k8sClient, lvg) + fdsBefore := e2eFileDevicesForNode(created, node) + Expect(fdsBefore).To(HaveLen(1)) + filePath := fdsBefore[0].FilePath + + By("Restarting the sds-node-configurator agent on the node (triggers ReattachFileDevices at startup)") + restartSDSNodeConfiguratorAgentOnNode(e2eCtx, k8sClient, node) + + By("Verifying the VG stays Ready and the file device is reattached") + var afterRestart v1alpha1.LVMVolumeGroup + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(e2eCtx, client.ObjectKey{Name: lvgName}, &afterRestart)).To(Succeed()) + g.Expect(afterRestart.Status.Phase).To(Equal(v1alpha1.PhaseReady), "phase should stay Ready after agent restart") + fds := e2eFileDevicesForNode(&afterRestart, node) + g.Expect(fds).To(HaveLen(1), "file device should still be reported after restart") + g.Expect(fds[0].FilePath).To(Equal(filePath), "backing file path should be unchanged") + g.Expect(fds[0].LoopDevice).To(HavePrefix("/dev/loop"), "loop device should be reattached") + }, 5*time.Minute, 10*time.Second).Should(Succeed()) + e2eExpectNoFalseConditions(&afterRestart) + + bound, loopOut, err := e2eLoopBoundToFileOnNode(e2eCtx, testClusterResources.Kubeconfig, node, vmSSH, filePath) + Expect(err).NotTo(HaveOccurred()) + Expect(bound).To(BeTrue(), "loop device should be attached to %s after restart; losetup:\n%s", filePath, loopOut) + printLVMVolumeGroupInfo(&afterRestart) + + By("✓ File device reattached and VG stayed Ready after agent restart") + }) + + It("Should reject a fileDevices directory outside the allowed base dir", func() { + node := e2ePickNodeRunningAgent(e2eCtx, k8sClient) + runID := strconv.FormatInt(time.Now().UnixNano(), 10) + nodeSuffix := strings.NewReplacer(".", "-", "_", "-").Replace(node) + lvgName := "e2e-lvg-fdbad-" + runID + "-" + nodeSuffix + vgName := "e2e-vg-fdbad-" + runID + badDir := "/tmp/e2e-filedevices-" + runID + + lvg := newFileLVG(node, lvgName, vgName, + []v1alpha1.LVMVolumeGroupFileDeviceSpec{{Directory: badDir, Size: resource.MustParse("1Gi")}}, nil) + + By(fmt.Sprintf("Creating LVMVolumeGroup %s with an out-of-base directory %s", lvgName, badDir)) + Expect(k8sClient.Create(e2eCtx, lvg)).To(Succeed()) + defer func() { e2eDeleteLVGAndWaitGone(e2eCtx, k8sClient, lvgName, 5*time.Minute) }() + + By("Expecting VGConfigurationApplied=False with ValidationFailed and the base-dir message") + Eventually(func(g Gomega) { + var cur v1alpha1.LVMVolumeGroup + g.Expect(k8sClient.Get(e2eCtx, client.ObjectKey{Name: lvgName}, &cur)).To(Succeed()) + g.Expect(cur.Status.Phase).NotTo(Equal(v1alpha1.PhaseReady)) + var cfg *metav1.Condition + for i := range cur.Status.Conditions { + if cur.Status.Conditions[i].Type == lvgConditionVGConfigurationApplied { + cfg = &cur.Status.Conditions[i] + break + } + } + g.Expect(cfg).NotTo(BeNil(), "VGConfigurationApplied condition should be present") + g.Expect(cfg.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cfg.Reason).To(Equal(reasonValidationFailed)) + g.Expect(cfg.Message).To(ContainSubstring("or a subdirectory of it")) + }, 3*time.Minute, 8*time.Second).Should(Succeed()) + + By("Verifying no VG and no backing directory were created on the node") + vgListed, vgsOut, err := e2eVGListedOnNode(e2eCtx, testClusterResources.Kubeconfig, node, vmSSH, vgName) + Expect(err).NotTo(HaveOccurred()) + Expect(vgListed).To(BeFalse(), "VG %s must not be created for a rejected spec; vgs:\n%s", vgName, vgsOut) + dirExists, err := e2eNodePathExists(e2eCtx, testClusterResources.Kubeconfig, node, vmSSH, badDir) + Expect(err).NotTo(HaveOccurred()) + Expect(dirExists).To(BeFalse(), "the out-of-base directory %s must not be created", badDir) + + var dump v1alpha1.LVMVolumeGroup + Expect(k8sClient.Get(e2eCtx, client.ObjectKey{Name: lvgName}, &dump)).To(Succeed()) + printLVMVolumeGroupInfo(&dump) + + By("✓ Out-of-base fileDevices directory rejected; nothing provisioned on the node") + }) + }) + ///////////////////////////////////////////////////// ---=== TESTS END HERE ===--- ///////////////////////////////////////////////////// }) // Describe: Sds Node Configurator