diff --git a/Makefile b/Makefile index ca125fcd8..9588cb9cc 100644 --- a/Makefile +++ b/Makefile @@ -233,6 +233,24 @@ $(BIN_PATH)/%: .static.%.$(STATIC) .PRECIOUS: $(foreach dir,$(BUILD_DIRS),.static.$(dir).1 .static.$(dir).) +# +# Eskimo words for snow... +# + +balloons: $(BIN_PATH)/nri-resource-policy-balloons +balloons-img: image.nri-resource-policy-balloons +t-a topology-aware: $(BIN_PATH)/nri-resource-policy-topology-aware +ta-img topolog-aware-img: image.nri-resource-policy-topology-aware +template: $(BIN_PATH)/nri-resource-policy-template +template-img: image.nri-resource-policy-template +memqos memory-qos: $(BIN_PATH)/nri-memory-qos +memqos-img memory-qos-img: image.nri-memory-qos +memtierd: $(BIN_PATH)/nri-memtierd +memtierd-img: image.nri-memtierd +sgx-epc sgx: $(BIN_PATH)/nri-sgx-epc +sgx-epc-img: image.nri-sgx-epc +config-manager: $(BIN_PATH)/config-manager + # # Image building test deployment generation targets # diff --git a/README-DRA-driver-proto.md b/README-DRA-driver-proto.md new file mode 100644 index 000000000..b432eb25e --- /dev/null +++ b/README-DRA-driver-proto.md @@ -0,0 +1,226 @@ +# Prototyping CPU DRA device abstraction / DRA-based CPU allocation + +## Background + +This prototype patch set bolts a DRA allocation frontend on top of the existing +topology aware resource policy plugin. The main intention with of this patch set +is to + +- provide something practical to play around with for the [feasibility study]( https://docs.google.com/document/d/1Tb_dC60YVCBr7cNYWuVLddUUTMcNoIt3zjd5-8rgug0/edit?tab=t.0#heading=h.iutbebngx80e) of enabling DRA-based CPU allocation, +- allow (relatively) easy experimentation with how to expose CPU as DRA +devices (IOW test various CPU DRA attributes) +- allow testing how DRA-based CPU allocation (using non-trivial CEL expressions) +would scale with cluster and cluster node size + +## Notes + +This patched NRI plugin, especially in its current state and form, is +*not a proposal* for a first real DRA-based CPU driver. + +## Prerequisites for Testing + +To test out this in a cluster, make sure you have + +1. A recent enough control plane with DRA v1 support (for instance Kubernetes v1.35.0). + +2. CDI enabled in your runtime configuration + +## Installation and Testing + +Once you have your cluster properly set upset up, you can pull this in to +your cluster with for testing with something like this: + +```bash +helm install --devel -n kube-system test oci://ghcr.io/klihub/nri-plugins/helm-charts/nri-resource-policy-topology-aware --version v0.12-dra-driver-unstable --set image.pullPolicy=Always --set extraEnv.OVERRIDE_SYS_ATOM_CPUS='2-5' --set extraEnv.OVERRIDE_SYS_CORE_CPUS='0\,1\,6-15' +``` + +Once the NRI plugin+DRA driver is up and running, you should see some CPUs +exposed as DRI devices. You can check the resource slices with the following +command + +```bash +[kli@n4c16-fedora-40-cloud-base-containerd ~]# kubectl get resourceslices +NAME NODE DRIVER POOL AGE +n4c16-fedora-40-cloud-base-containerd-native.cpu-jxfkj n4c16-fedora-40-cloud-base-containerd native.cpu pool0 4d2h +``` + +And the exposed devices like this: + +```bash +[kli@n4c16-fedora-40-cloud-base-containerd ~]# kubectl get resourceslices -oyaml | less +apiVersion: v1 +items: +- apiVersion: resource.k8s.io/v1 + kind: ResourceSlice + metadata: + creationTimestamp: "2025-06-10T06:01:54Z" + generateName: n4c16-fedora-40-cloud-base-containerd-native.cpu- + generation: 1 + name: n4c16-fedora-40-cloud-base-containerd-native.cpu-jxfkj + ownerReferences: + - apiVersion: v1 + controller: true + kind: Node + name: n4c16-fedora-40-cloud-base-containerd + uid: 90a99f1f-c1ca-4bea-8dbd-3cc821f744b1 + resourceVersion: "871388" + uid: 4639d31f-e508-4b0a-8378-867f6c1c7cb1 + spec: + devices: + - attributes: + cache0ID: + int: 0 + cache1ID: + int: 8 + cache2ID: + int: 16 + cache3ID: + int: 24 + cluster: + int: 0 + core: + int: 0 + coreType: + string: P-core + die: + int: 0 + isolated: + bool: false + localMemory: + int: 0 + package: + int: 0 + name: cpu1 + - attributes: + - attributes: + cache0ID: + int: 1 + cache1ID: + int: 9 + cache2ID: + int: 17 + cache3ID: + int: 24 + cluster: + int: 2 + core: + int: 1 + coreType: + string: E-core + die: + int: 0 + isolated: + bool: false + localMemory: + int: 0 + package: + int: 0 + name: cpu2 + - attributes: + cache0ID: + int: 1 + cache1ID: + int: 9 + cache2ID: + int: 17 + cache3ID: + int: 24 + cluster: + int: 2 + core: +... +``` + +If everything looks fine and you do have CPUs available as DRA devices, you +can test DRA-based CPU allocation with something like this. This allocates +a single P-core for the container. + +```yaml +apiVersion: resource.k8s.io/v1 +kind: ResourceClaimTemplate +metadata: + name: any-cores +spec: + spec: + devices: + requests: + - name: cpu + exactly: + deviceClassName: native.cpu +--- +apiVersion: resource.k8s.io/v1 +kind: ResourceClaimTemplate +metadata: + name: p-cores +spec: + spec: + devices: + requests: + - name: cpu + exactly: + deviceClassName: native.cpu + selectors: + - cel: + expression: device.attributes["native.cpu"].coreType == "P-core" + count: 1 +--- +apiVersion: resource.k8s.io/v1 +kind: ResourceClaimTemplate +metadata: + name: e-cores +spec: + spec: + devices: + requests: + - name: cpu + exactly: + deviceClassName: native.cpu + selectors: + - cel: + expression: device.attributes["native.cpu"].coreType == "E-core" + count: 1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: pcore-test + labels: + app: pod +spec: + containers: + - name: ctr0 + image: busybox + imagePullPolicy: IfNotPresent + args: + - /bin/sh + - -c + - trap 'exit 0' TERM; sleep 3600 & wait + resources: + requests: + cpu: 1 + memory: 100M + limits: + cpu: 1 + memory: 100M + claims: + - name: claim-pcores + resourceClaims: + - name: claim-pcores + resourceClaimTemplateName: p-cores + terminationGracePeriodSeconds: 1 +``` + +If you want to try a mixed native CPU + DRA-based allocation, try +increasing the CPU request and limit in the pods spec to 1500m CPUs +or CPUs and see what happens. + + +## Playing Around with CPU Abstractions + +If you want to play around with this (for instance modify the exposed CPU abstraction), the easiest way is to +1. [fork](https://github.com/containers/nri-plugins/fork) the [main NRI Reference Plugins](https://github.com/containers/nri-plugins) repo +2. enable github actions in your personal fork +3. make any changes you want (for instance, to alter the CPU abstraction, take a look at [cpu.DRA()](https://github.com/klihub/nri-plugins/blob/test/build/dra-driver/pkg/sysfs/dra.go) +4. Push your changes to ssh://git@github.com/$YOUR_FORK/nri-plugins/refs/heads/test/build/dra-driver. +5. Wait for the image and Helm chart publishing actions to succeed +6. Once done, you can pull the result in to your cluster with something like `helm install --devel -n kube-system test oci://ghcr.io/$YOUR_GITHUB_USERID/nri-plugins/helm-charts/nri-resource-policy-topology-aware --version v0.9-dra-driver-unstable` diff --git a/README.md b/README.md index d152ef7cb..7c63ba263 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,6 @@ Currently following plugins are available: [5]: https://containers.github.io/nri-plugins/stable/docs/memory/sgx-epc.html See the [NRI plugins documentation](https://containers.github.io/nri-plugins/) for more information. + +See the [DRA CPU driver prototype notes](README-DRA-driver-proto.md) for more information +about using the Topology Aware policy as a DRA CPU driver. diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 57c3bc32c..23b4a7e3f 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -372,6 +372,18 @@ func (p *balloons) UpdateResources(c cache.Container) error { return nil } +// AllocateClaim allocates CPUs for the claim. +func (p *balloons) AllocateClaim(claim policy.Claim) error { + log.Debug("allocating claim %s for pods %v...", claim.String(), claim.GetPods()) + return nil +} + +// ReleaseClaim releases CPUs of the claim. +func (p *balloons) ReleaseClaim(claim policy.Claim) error { + log.Debug("releasing claim %s for pods %v...", claim.String(), claim.GetPods()) + return nil +} + // HandleEvent handles policy-specific events. func (p *balloons) HandleEvent(*events.Policy) (bool, error) { log.Debug("(not) handling event...") diff --git a/cmd/plugins/template/policy/template-policy.go b/cmd/plugins/template/policy/template-policy.go index b1684ba7d..0d73c6074 100644 --- a/cmd/plugins/template/policy/template-policy.go +++ b/cmd/plugins/template/policy/template-policy.go @@ -108,6 +108,18 @@ func (p *policy) UpdateResources(c cache.Container) error { return nil } +// AllocateClaim alloctes CPUs for the claim. +func (p *policy) AllocateClaim(claim policyapi.Claim) error { + log.Debug("allocating claim %s for pods %v...", claim.String(), claim.GetPods()) + return nil +} + +// ReleaseClaim releases CPUs of the claim. +func (p *policy) ReleaseClaim(claim policyapi.Claim) error { + log.Debug("releasing claim %s for pods %v...", claim.String(), claim.GetPods()) + return nil +} + // HandleEvent handles policy-specific events. func (p *policy) HandleEvent(e *events.Policy) (bool, error) { log.Info("received policy event %s.%s with data %v...", e.Source, e.Type, e.Data) diff --git a/cmd/plugins/topology-aware/policy/cache.go b/cmd/plugins/topology-aware/policy/cache.go index 454cb8bd6..82cf5d3e2 100644 --- a/cmd/plugins/topology-aware/policy/cache.go +++ b/cmd/plugins/topology-aware/policy/cache.go @@ -126,6 +126,7 @@ func (p *policy) reinstateGrants(grants map[string]Grant) error { type cachedGrant struct { PrettyName string + Claimed string Exclusive string Part int CPUType cpuClass @@ -140,6 +141,7 @@ type cachedGrant struct { func newCachedGrant(cg Grant) *cachedGrant { ccg := &cachedGrant{} ccg.PrettyName = cg.GetContainer().PrettyName() + ccg.Claimed = cg.ClaimedCPUs().String() ccg.Exclusive = cg.ExclusiveCPUs().String() ccg.Part = cg.CPUPortion() ccg.CPUType = cg.CPUType() @@ -168,6 +170,7 @@ func (ccg *cachedGrant) ToGrant(policy *policy) (Grant, error) { container, ccg.CPUType, cpuset.MustParse(ccg.Exclusive), + cpuset.MustParse(ccg.Claimed), ccg.Part, ccg.MemType, ccg.ColdStart, diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index 9477f94b0..d4dc582ab 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -30,6 +30,7 @@ import ( "github.com/intel/goresctrl/pkg/sst" idset "github.com/intel/goresctrl/pkg/utils" v1 "k8s.io/api/core/v1" + resapi "k8s.io/api/resource/v1" ) type mockSystemNode struct { @@ -207,6 +208,10 @@ func (c *mockCPU) CoreKind() sysfs.CoreKind { return sysfs.PerformanceCore } +func (c *mockCPU) DRA(extras ...map[sysfs.QualifiedName]sysfs.Attribute) *resapi.Device { + panic("unimplmented") +} + type mockSystem struct { isolatedCPU int nodes []sysfs.Node @@ -348,6 +353,9 @@ func (fake *mockSystem) NodeDistance(idset.ID, idset.ID) int { func (fake *mockSystem) NodeHintToCPUs(string) string { return "" } +func (fake *mockSystem) CPUsAsDRADevices(ids []idset.ID) []resapi.Device { + panic("unimplemented") +} type mockContainer struct { name string @@ -408,6 +416,9 @@ func (m *mockContainer) GetAnnotation(string, interface{}) (string, bool) { func (m *mockContainer) GetEnv(string) (string, bool) { panic("unimplemented") } +func (m *mockContainer) GetEnvList() []string { + panic("unimplemented") +} func (m *mockContainer) GetAnnotations() map[string]string { panic("unimplemented") } @@ -752,6 +763,12 @@ func (m *mockCache) SetPolicyEntry(string, interface{}) { func (m *mockCache) GetPolicyEntry(string, interface{}) bool { return m.returnValueForGetPolicyEntry } +func (m *mockCache) SetEntry(string, interface{}) { + panic("unimplemented") +} +func (m *mockCache) GetEntry(string, interface{}) (interface{}, error) { + panic("unimplemented") +} func (m *mockCache) Save() error { return nil } diff --git a/cmd/plugins/topology-aware/policy/pools.go b/cmd/plugins/topology-aware/policy/pools.go index 87fe22a31..9082f5182 100644 --- a/cmd/plugins/topology-aware/policy/pools.go +++ b/cmd/plugins/topology-aware/policy/pools.go @@ -18,6 +18,8 @@ import ( "fmt" "math" "sort" + "strconv" + "strings" "github.com/containers/nri-plugins/pkg/utils/cpuset" corev1 "k8s.io/api/core/v1" @@ -25,6 +27,7 @@ import ( cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/resmgr/cache" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" + policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" system "github.com/containers/nri-plugins/pkg/sysfs" idset "github.com/intel/goresctrl/pkg/utils" ) @@ -349,7 +352,14 @@ func (p *policy) allocatePool(container cache.Container, poolHint string) (Grant offer *libmem.Offer ) - request := newRequest(container, p.memAllocator.Masks().AvailableTypes()) + claimed, unclaimed, err := p.getClaimedCPUs(container) + if err != nil { + return nil, err + } + + log.Info("**** claimed CPU(s): %s, unclaimed CPU: %d", claimed, unclaimed) + + request := newRequest(container, claimed, p.memAllocator.Masks().AvailableTypes()) if p.root.FreeSupply().ReservedCPUs().IsEmpty() && request.CPUType() == cpuReserved { // Fallback to allocating reserved CPUs from the shared pool @@ -440,6 +450,151 @@ func (p *policy) allocatePool(container cache.Container, poolHint string) (Grant return grant, nil } +func (p *policy) allocateClaim(claim policyapi.Claim) error { + hints := map[string]string{} + + // collect grants we need to free dues to conflciting exclusive allocation + realloc := []Grant{} + cpus := cpuset.New(claim.GetDevices()...) + for _, g := range p.allocations.grants { + if !cpus.Intersection(g.ExclusiveCPUs().Union(g.IsolatedCPUs())).IsEmpty() { + log.Info("***** releasing conflicting grant %s", g) + hints[g.GetContainer().GetID()] = g.GetCPUNode().Name() + realloc = append(realloc, g) + p.releasePool(g.GetContainer()) + p.updateSharedAllocations(&g) + } + } + + pool := p.getPoolForCPUs(cpus) + if pool == nil { + return fmt.Errorf("failed to find pool for claimed CPUs %s", cpus.String()) + } + + log.Info("*** target pool is %s", pool.Name()) + + // free more grants if we need to, for slicing off claimed CPUs + s := pool.FreeSupply() + needShared := s.SharableCPUs().Intersection(cpus).Size() + if available, needed := s.AllocatableSharedCPU(), 1000*needShared; available < needed { + log.Info("***** need to free %d mCPU shared capacity", needed-available) + grants := p.getLargestSharedUsers(pool) + log.Info("grants: %v", grants) + for _, g := range grants { + hints[g.GetContainer().GetID()] = g.GetCPUNode().Name() + realloc = append(realloc, g) + p.releasePool(g.GetContainer()) + p.updateSharedAllocations(&g) + if available, needed = s.AllocatableSharedCPU(), 1000*needShared; available >= needed { + break + } + } + } + s.ClaimCPUs(cpus) + + // TODO: sort old grants by QoS class or size and pool distance from root... + for _, oldg := range realloc { + c := oldg.GetContainer() + log.Info("***** reallocating %s with pool hint %s", c.PrettyName(), hints[c.GetID()]) + err := p.allocateResources(c, hints[c.GetID()]) + if err != nil { + log.Error("failed to reallocate container %s (%s), retrying with no pool hint", c.PrettyName(), c.GetID()) + } + err = p.allocateResources(c, "") + if err != nil { + log.Error("failed to reallocate container %s without pool hints", c.PrettyName(), c.GetID()) + } + } + + return nil +} + +// Find (tightest fitting) target pool for a set of cpus +func (p *policy) getPoolForCPUs(cpus cpuset.CPUSet) Node { + var pool Node + for _, n := range p.pools { + s := n.GetSupply() + poolCPUs := s.SharableCPUs().Union(s.IsolatedCPUs()).Union(s.ReservedCPUs()) + if poolCPUs.Intersection(cpus).Equals(cpus) { + if pool == nil { + pool = n + } else { + if n.RootDistance() > pool.RootDistance() { + pool = n + } + } + } + } + return pool +} + +// Get the largest shared CPU users in a pool and its parents. +func (p *policy) getLargestSharedUsers(pool Node) []Grant { + pools := map[string]struct{}{} + grants := []Grant{} + for n := pool; !n.IsNil(); n = n.Parent() { + pools[n.Name()] = struct{}{} + } + + for _, g := range p.allocations.grants { + if g.SharedPortion() > 0 { + grants = append(grants, g) + } + } + + sort.Slice(grants, func(i, j int) bool { + gi, gj := grants[i], grants[j] + di, dj := gi.GetCPUNode().RootDistance(), gj.GetCPUNode().RootDistance() + si, sj := gi.SharedPortion(), gj.SharedPortion() + if di < dj { + return true + } + return si <= sj + }) + + return grants +} + +func (p *policy) releaseClaim(claim policyapi.Claim) error { + cpus := cpuset.New(claim.GetDevices()...) + p.root.FreeSupply().UnclaimCPUs(cpus) + p.updateSharedAllocations(nil) + + return nil +} + +// Get the claimed CPUs injected into the environment of a container. +func (p *policy) getClaimedCPUs(c cache.Container) (cpuset.CPUSet, int, error) { + var ids []int + for _, env := range c.GetEnvList() { + if !strings.HasPrefix(env, "DRA_CPU") { + continue + } + val := strings.TrimSuffix(strings.TrimPrefix(env, "DRA_CPU"), "=1") + i, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return cpuset.New(), 0, fmt.Errorf("invalid DRA CPU env. var %q: %v", env, err) + } + ids = append(ids, int(i)) + } + + claimed := cpuset.New(ids...) + reqs, ok := c.GetResourceUpdates() + if !ok { + reqs = c.GetResourceRequirements() + } + request := reqs.Requests[corev1.ResourceCPU] + unclaimed := int(request.MilliValue()) - 1000*claimed.Size() + + if unclaimed < 0 { + return claimed, unclaimed, + fmt.Errorf("invalid claimed (%s CPU(s)) vs. requested/native CPU (%dm)", + claimed, request.MilliValue()) + } + + return claimed, unclaimed, nil +} + // setPreferredCpusetCpus pins container's CPUs according to what has been // allocated for it, taking into account if the container should run // with hyperthreads hidden. @@ -465,6 +620,7 @@ func (p *policy) applyGrant(grant Grant) { container := grant.GetContainer() cpuType := grant.CPUType() + claimed := grant.ClaimedCPUs() exclusive := grant.ExclusiveCPUs() reserved := grant.ReservedCPUs() shared := grant.SharedCPUs() @@ -474,16 +630,25 @@ func (p *policy) applyGrant(grant Grant) { kind := "" switch cpuType { case cpuNormal: - if exclusive.IsEmpty() { + if exclusive.IsEmpty() && claimed.IsEmpty() { cpus = shared kind = "shared" } else { - kind = "exclusive" + if !exclusive.IsEmpty() { + kind = "exclusive" + } + if !claimed.IsEmpty() { + if kind == "" { + kind = "claimed" + } else { + kind += "+claimed" + } + } if cpuPortion > 0 { kind += "+shared" - cpus = exclusive.Union(shared) + cpus = exclusive.Union(claimed).Union(shared) } else { - cpus = exclusive + cpus = exclusive.Union(claimed) } } case cpuReserved: @@ -618,9 +783,14 @@ func (p *policy) updateSharedAllocations(grant *Grant) { continue } + if other.SharedPortion() == 0 && !other.ClaimedCPUs().IsEmpty() { + log.Info(" => %s not affected (only claimed CPUs)...", other) + continue + } + if opt.PinCPU { shared := other.GetCPUNode().FreeSupply().SharableCPUs() - exclusive := other.ExclusiveCPUs() + exclusive := other.ExclusiveCPUs().Union(other.ClaimedCPUs()) if exclusive.IsEmpty() { p.setPreferredCpusetCpus(other.GetContainer(), shared, fmt.Sprintf(" => updating %s with shared CPUs of %s: %s...", @@ -756,6 +926,8 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco depth1, depth2 := node1.RootDistance(), node2.RootDistance() id1, id2 := node1.NodeID(), node2.NodeID() score1, score2 := scores[id1], scores[id2] + claim1, claim2 := node1.FreeSupply().ClaimedCPUs(), node2.FreeSupply().ClaimedCPUs() + claim := request.ClaimedCPUs() cpuType := request.CPUType() isolated1, reserved1, shared1 := score1.IsolatedCapacity(), score1.ReservedCapacity(), score1.SharedCapacity() isolated2, reserved2, shared2 := score2.IsolatedCapacity(), score2.ReservedCapacity(), score2.SharedCapacity() @@ -772,7 +944,7 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco // // Our scoring/score sorting algorithm is: // - // - insufficient isolated, reserved or shared capacity loses + // - insufficient claimed, isolated, reserved or shared capacity loses // - if we have affinity, the higher affinity score wins // - if we have topology hints // * better hint score wins @@ -799,8 +971,12 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco // Before this comparison is reached, nodes with insufficient uncompressible resources // (memory) have been filtered out. - // a node with insufficient isolated or shared capacity loses + // a node with insufficient claimed, isolated or shared capacity loses switch { + case claim.Difference(claim1).Size() == 0 && claim.Difference(claim2).Size() > 0: + return true + case claim.Difference(claim1).Size() > 0 && claim.Difference(claim2).Size() == 0: + return false case cpuType == cpuNormal && ((isolated2 < 0 && isolated1 >= 0) || (shared2 < 0 && shared1 >= 0)): log.Debug(" => %s loses, insufficent isolated or shared", node2.Name()) return true diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index 794312e76..c3a316651 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -65,6 +65,8 @@ type Supply interface { ReservedCPUs() cpuset.CPUSet // SharableCPUs returns the sharable cpuset in this supply. SharableCPUs() cpuset.CPUSet + // ClaimedCPUs returns the claimed CPUs for this supply. + ClaimedCPUs() cpuset.CPUSet // GrantedReserved returns the locally granted reserved CPU capacity in this supply. GrantedReserved() int // GrantedShared returns the locally granted shared CPU capacity in this supply. @@ -92,6 +94,12 @@ type Supply interface { DumpCapacity() string // DumpAllocatable returns a printable representation of the supply's alloctable resources. DumpAllocatable() string + + ClaimCPUs(cpuset.CPUSet) + claimCPUs(cpuset.CPUSet) + + UnclaimCPUs(cpuset.CPUSet) + unclaimCPUs(cpuset.CPUSet) } // Request represents CPU and memory resources requested by a container. @@ -106,6 +114,8 @@ type Request interface { CPUPrio() cpuPrio // SetCPUType sets the type of requested CPU. SetCPUType(cpuType cpuClass) + // ClaimedCPUs returns the CPUs claimed for this request. + ClaimedCPUs() cpuset.CPUSet // FullCPUs return the number of full CPUs requested. FullCPUs() int // CPUFraction returns the amount of fractional milli-CPU requested. @@ -147,6 +157,8 @@ type Grant interface { // CPUPortion returns granted milli-CPUs of non-full CPUs of CPUType(). // CPUPortion() == ReservedPortion() + SharedPortion(). CPUPortion() int + // ClaimedCPUs returns the claimed granted cpuset. + ClaimedCPUs() cpuset.CPUSet // ExclusiveCPUs returns the exclusively granted non-isolated cpuset. ExclusiveCPUs() cpuset.CPUSet // ReservedCPUs returns the reserved granted cpuset. @@ -217,6 +229,7 @@ type supply struct { isolated cpuset.CPUSet // isolated CPUs at this node reserved cpuset.CPUSet // reserved CPUs at this node sharable cpuset.CPUSet // sharable CPUs at this node + claimed cpuset.CPUSet // CPUs allocated through DRA grantedReserved int // amount of reserved CPUs allocated grantedShared int // amount of shareable CPUs allocated } @@ -226,6 +239,7 @@ var _ Supply = &supply{} // request implements our Request interface. type request struct { container cache.Container // container for this request + claimed cpuset.CPUSet // CPUs DRA-claimed for this container full int // number of full CPUs requested fraction int // amount of fractional CPU requested limit int // CPU limit, MaxInt for no limit @@ -252,6 +266,7 @@ type grant struct { container cache.Container // container CPU is granted to node Node // node CPU is supplied from exclusive cpuset.CPUSet // exclusive CPUs + claimed cpuset.CPUSet // claimed CPUs cpuType cpuClass // type of CPUs (normal, reserved, ...) cpuPortion int // milliCPUs granted from CPUs of cpuType memType memoryType // requested types of memory @@ -316,6 +331,11 @@ func (cs *supply) SharableCPUs() cpuset.CPUSet { return cs.sharable.Clone() } +// ClaimedCPUs returns the claimed CPUs in this supply. +func (cs *supply) ClaimedCPUs() cpuset.CPUSet { + return cs.claimed.Clone() +} + // GrantedReserved returns the locally granted reserved CPU capacity. func (cs *supply) GrantedReserved() int { return cs.grantedReserved @@ -398,6 +418,7 @@ func (cs *supply) AllocateCPU(r Request) (Grant, error) { cr := r.(*request) + claimed := cr.claimed full := cr.full fraction := cr.fraction @@ -465,7 +486,7 @@ func (cs *supply) AllocateCPU(r Request) (Grant, error) { cs.node.Name(), full, cs.sharable, cs.AllocatableSharedCPU()) } - grant := newGrant(cs.node, cr.GetContainer(), cpuType, exclusive, 0, 0, 0) + grant := newGrant(cs.node, cr.GetContainer(), cpuType, exclusive, claimed, 0, 0, 0) grant.AccountAllocateCPU() if fraction > 0 { @@ -714,6 +735,44 @@ func (cs *supply) DumpAllocatable() string { return allocatable } +func (cs *supply) ClaimCPUs(cpus cpuset.CPUSet) { + cs.claimCPUs(cpus) + + cs.GetNode().DepthFirst(func(n Node) { + n.FreeSupply().claimCPUs(cpus) + }) + for n := cs.GetNode(); !n.IsNil(); n = n.Parent() { + n.FreeSupply().claimCPUs(cpus) + } +} + +func (cs *supply) claimCPUs(cpus cpuset.CPUSet) { + cs.claimed = cs.claimed.Union(cpus) + cs.isolated = cs.isolated.Difference(cpus) + cs.sharable = cs.sharable.Difference(cpus) + cs.reserved = cs.reserved.Difference(cpus) +} + +func (cs *supply) UnclaimCPUs(cpus cpuset.CPUSet) { + cs.unclaimCPUs(cpus) + + cs.GetNode().DepthFirst(func(n Node) { + n.FreeSupply().unclaimCPUs(cpus) + }) + for n := cs.GetNode(); !n.IsNil(); n = n.Parent() { + n.FreeSupply().unclaimCPUs(cpus) + } +} + +func (cs *supply) unclaimCPUs(cpus cpuset.CPUSet) { + all := cs.GetNode().GetSupply() + + cs.isolated = cs.isolated.Union(all.IsolatedCPUs().Intersection(cpus)) + cs.sharable = cs.sharable.Union(all.SharableCPUs().Intersection(cpus)) + cs.reserved = cs.reserved.Union(all.ReservedCPUs().Intersection(cpus)) + cs.claimed = cs.claimed.Difference(cpus) +} + // prettyMem formats the given amount as k, M, G, or T units. func prettyMem(value int64) string { units := []string{"k", "M", "G", "T"} @@ -732,14 +791,22 @@ func prettyMem(value int64) string { } // newRequest creates a new request for the given container. -func newRequest(container cache.Container, types libmem.TypeMask) Request { +func newRequest(container cache.Container, claimed cpuset.CPUSet, types libmem.TypeMask) Request { pod, _ := container.GetPod() full, fraction, cpuLimit, isolate, cpuType, prio := cpuAllocationPreferences(pod, container) req, lim, mtype := memoryAllocationPreference(pod, container) coldStart := time.Duration(0) - log.Debug("%s: CPU preferences: cpuType=%s, full=%v, fraction=%v, isolate=%v, prio=%v", - container.PrettyName(), cpuType, full, fraction, isolate, prio) + if full >= claimed.Size() { + full -= claimed.Size() + } else { + if fraction >= 1000*claimed.Size() { + fraction -= 1000 * claimed.Size() + } + } + + log.Debug("%s: CPU preferences: cpuType=%s, claim=%s, full=%v, fraction=%v, isolate=%v, prio=%v", + container.PrettyName(), cpuType, claimed, full, fraction, isolate, prio) if mtype == memoryUnspec { mtype = defaultMemoryType &^ memoryHBM @@ -769,6 +836,7 @@ func newRequest(container cache.Container, types libmem.TypeMask) Request { return &request{ container: container, + claimed: claimed, full: full, fraction: fraction, limit: cpuLimit, @@ -825,6 +893,11 @@ func (cr *request) SetCPUType(cpuType cpuClass) { cr.cpuType = cpuType } +// ClaimedPUs return the claimed CPUs for this request. +func (cr *request) ClaimedCPUs() cpuset.CPUSet { + return cr.claimed +} + // FullCPUs return the number of full CPUs requested. func (cr *request) FullCPUs() int { return cr.full @@ -1135,12 +1208,13 @@ func (score *score) String() string { } // newGrant creates a CPU grant from the given node for the container. -func newGrant(n Node, c cache.Container, cpuType cpuClass, exclusive cpuset.CPUSet, cpuPortion int, mt memoryType, coldstart time.Duration) Grant { +func newGrant(n Node, c cache.Container, cpuType cpuClass, exclusive, claimed cpuset.CPUSet, cpuPortion int, mt memoryType, coldstart time.Duration) Grant { grant := &grant{ node: n, container: c, cpuType: cpuType, exclusive: exclusive, + claimed: claimed, cpuPortion: cpuPortion, memType: mt, coldStart: coldstart, @@ -1179,6 +1253,7 @@ func (cg *grant) Clone() Grant { node: cg.GetCPUNode(), container: cg.GetContainer(), exclusive: cg.ExclusiveCPUs(), + claimed: cg.ClaimedCPUs(), cpuType: cg.CPUType(), cpuPortion: cg.SharedPortion(), memType: cg.MemoryType(), @@ -1233,6 +1308,11 @@ func (cg *grant) ExclusiveCPUs() cpuset.CPUSet { return cg.exclusive } +// ClaimedCPUs returns the claimed CPUSet in this grant. +func (cg *grant) ClaimedCPUs() cpuset.CPUSet { + return cg.claimed +} + // ReservedCPUs returns the reserved CPUSet in the supply of this grant. func (cg *grant) ReservedCPUs() cpuset.CPUSet { return cg.node.GetSupply().ReservedCPUs() diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index c7f8b0fa4..897d4c267 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -150,6 +150,10 @@ func (p *policy) Start() error { } p.metrics = m + if err := p.options.PublishCPUs(p.allowed.Difference(p.reserved).List()); err != nil { + log.Errorf("failed to publish CPU DRA resources: %v", err) + } + return nil } @@ -291,6 +295,18 @@ func (p *policy) UpdateResources(container cache.Container) error { return nil } +// AllocateClaim allocates CPUs for the claim. +func (p *policy) AllocateClaim(claim policyapi.Claim) error { + log.Info("allocating claim %s for pods %v...", claim.String(), claim.GetPods()) + return p.allocateClaim(claim) +} + +// ReleaseClaim releases CPUs of the claim. +func (p *policy) ReleaseClaim(claim policyapi.Claim) error { + log.Info("releasing claim %s for pods %v...", claim.String(), claim.GetPods()) + return p.releaseClaim(claim) +} + // HandleEvent handles policy-specific events. func (p *policy) HandleEvent(e *events.Policy) (bool, error) { log.Debug("received policy event %s.%s with data %v...", e.Source, e.Type, e.Data) @@ -404,8 +420,9 @@ func (p *policy) ExportResourceData(c cache.Container) map[string]string { data := map[string]string{} shared := grant.SharedCPUs().String() - isolated := grant.ExclusiveCPUs().Intersection(grant.GetCPUNode().GetSupply().IsolatedCPUs()) + isolated := grant.ExclusiveCPUs().Union(grant.ClaimedCPUs()).Intersection(grant.GetCPUNode().GetSupply().IsolatedCPUs()) exclusive := grant.ExclusiveCPUs().Difference(isolated).String() + claimed := grant.ClaimedCPUs().String() if grant.SharedPortion() > 0 && shared != "" { data[policyapi.ExportSharedCPUs] = shared @@ -416,6 +433,9 @@ func (p *policy) ExportResourceData(c cache.Container) map[string]string { if exclusive != "" { data[policyapi.ExportExclusiveCPUs] = exclusive } + if claimed != "" { + data[policyapi.ExportClaimedCPUs] = claimed + } mems := grant.GetMemoryZone() dram := mems.And(p.memAllocator.Masks().NodesByTypes(libmem.TypeMaskDRAM)) @@ -516,6 +536,10 @@ func (p *policy) Reconfigure(newCfg interface{}) error { } p.metrics = m + if err := p.options.PublishCPUs(p.allowed.Difference(p.reserved).List()); err != nil { + log.Errorf("failed to publish CPU DRA resources: %v", err) + } + return nil } @@ -668,7 +692,11 @@ func (p *policy) newAllocations() allocations { // clone creates a copy of the allocation. func (a *allocations) clone() allocations { - o := allocations{policy: a.policy, grants: make(map[string]Grant)} + o := allocations{ + policy: a.policy, + grants: make(map[string]Grant), + //claims: make(map[string][]policyapi.Claim), + } for id, grant := range a.grants { o.grants[id] = grant.Clone() } diff --git a/deployment/helm/topology-aware/templates/clusterrole.yaml b/deployment/helm/topology-aware/templates/clusterrole.yaml index d5b76ffbb..2a58dd7e7 100644 --- a/deployment/helm/topology-aware/templates/clusterrole.yaml +++ b/deployment/helm/topology-aware/templates/clusterrole.yaml @@ -22,3 +22,27 @@ rules: - list - update - delete +- apiGroups: + - "resource.k8s.io" + resources: + - resourceslices + verbs: + - list + - watch + - create + - update + - delete +- apiGroups: + - "resource.k8s.io" + resources: + - resourceclaims + - deviceclasses + verbs: + - get +- apiGroups: + - "resource.k8s.io" + resources: + - resourceclaims/status + verbs: + - patch + - update diff --git a/deployment/helm/topology-aware/templates/daemonset.yaml b/deployment/helm/topology-aware/templates/daemonset.yaml index 86a7fec50..92a58545e 100644 --- a/deployment/helm/topology-aware/templates/daemonset.yaml +++ b/deployment/helm/topology-aware/templates/daemonset.yaml @@ -124,6 +124,12 @@ spec: - name: pod-resources-socket mountPath: /var/lib/kubelet/pod-resources readOnly: true + - name: host-cdi-dir + mountPath: /host/var/run/cdi + - name: kubelet-plugins + mountPath: /var/lib/kubelet/plugins + - name: kubelet-plugins-registry + mountPath: /var/lib/kubelet/plugins_registry {{- if .Values.podPriorityClassNodeCritical }} priorityClassName: system-node-critical {{- end }} @@ -147,6 +153,18 @@ spec: hostPath: path: /var/lib/kubelet/pod-resources type: DirectoryOrCreate + - name: host-cdi-dir + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate + - name: kubelet-plugins + hostPath: + path: /var/lib/kubelet/plugins + type: DirectoryOrCreate + - name: kubelet-plugins-registry + hostPath: + path: /var/lib/kubelet/plugins_registry + type: DirectoryOrCreate {{- if .Values.nri.runtime.patchConfig }} - name: containerd-config hostPath: diff --git a/deployment/helm/topology-aware/templates/native-cpu-device-class.yaml b/deployment/helm/topology-aware/templates/native-cpu-device-class.yaml new file mode 100644 index 000000000..2f4c7ccdb --- /dev/null +++ b/deployment/helm/topology-aware/templates/native-cpu-device-class.yaml @@ -0,0 +1,8 @@ +apiVersion: resource.k8s.io/v1 +kind: DeviceClass +metadata: + name: native.cpu +spec: + selectors: + - cel: + expression: device.driver == "native.cpu" diff --git a/docs/conf.py b/docs/conf.py index f93341bd0..ba73c1e09 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -127,7 +127,7 @@ def gomod_versions(modules): # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['build', 'build-aux', '_build', '.github', '_work', 'generate', 'README.md', 'TODO.md', 'SECURITY.md', 'CODE-OF-CONDUCT.md', 'docs/releases', 'test/e2e/README.md', 'docs/resource-policy/releases', 'docs/resource-policy/README.md','test/statistics-analysis/README.md', 'deployment/helm/*/*.md', '**/testdata'] +exclude_patterns = ['build', 'build-aux', '_build', '.github', '_work', 'generate', 'README.md', 'TODO.md', 'SECURITY.md', 'CODE-OF-CONDUCT.md', 'docs/releases', 'test/e2e/README.md', 'docs/resource-policy/releases', 'docs/resource-policy/README.md','test/statistics-analysis/README.md', 'deployment/helm/*/*.md', '**/testdata', 'README-DRA-driver-proto.md'] # -- Options for HTML output ------------------------------------------------- diff --git a/go.mod b/go.mod index e5fe6321a..1ad92532d 100644 --- a/go.mod +++ b/go.mod @@ -13,44 +13,46 @@ require ( github.com/intel/goresctrl v0.11.0 github.com/intel/memtierd v0.1.1 github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.1.2 - github.com/onsi/ginkgo/v2 v2.21.0 - github.com/onsi/gomega v1.35.1 + github.com/onsi/ginkgo/v2 v2.27.2 + github.com/onsi/gomega v1.38.2 github.com/pelletier/go-toml/v2 v2.1.0 - github.com/prometheus/client_golang v1.23.0 + github.com/prometheus/client_golang v1.23.2 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.38.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 go.opentelemetry.io/otel/exporters/prometheus v0.60.0 go.opentelemetry.io/otel/metric v1.38.0 go.opentelemetry.io/otel/sdk v1.38.0 go.opentelemetry.io/otel/sdk/metric v1.38.0 go.opentelemetry.io/otel/trace v1.38.0 - golang.org/x/sys v0.35.0 + golang.org/x/sys v0.38.0 golang.org/x/time v0.9.0 google.golang.org/grpc v1.75.0 - k8s.io/api v0.31.2 - k8s.io/apimachinery v0.33.1 - k8s.io/client-go v0.31.2 + k8s.io/api v0.35.0 + k8s.io/apimachinery v0.35.0 + k8s.io/client-go v0.35.0 + k8s.io/dynamic-resource-allocation v0.35.0 k8s.io/klog/v2 v2.130.1 - k8s.io/kubelet v0.31.2 - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 + k8s.io/kubelet v0.35.0 + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 sigs.k8s.io/controller-runtime v0.16.2 - sigs.k8s.io/yaml v1.4.0 + sigs.k8s.io/yaml v1.6.0 ) require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -59,54 +61,55 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect - github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/knqyf263/go-plugin v0.9.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.3.0 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/otlptranslator v0.0.2 // indirect github.com/prometheus/procfs v0.17.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.9 // indirect github.com/tetratelabs/wazero v1.10.1 // indirect github.com/x448/float16 v0.8.4 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.5 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect go.opentelemetry.io/proto/otlp v1.7.1 // indirect - golang.org/x/mod v0.26.0 // indirect - golang.org/x/net v0.43.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/tools v0.35.0 // indirect - golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/tools v0.38.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/protobuf v1.36.8 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/code-generator v0.33.1 // indirect - k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + k8s.io/code-generator v0.35.0 // indirect + k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index 8f3a66f9d..810a07d87 100644 --- a/go.sum +++ b/go.sum @@ -598,6 +598,8 @@ git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3p github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= @@ -614,8 +616,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -661,8 +663,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -682,9 +684,15 @@ github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/ github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= @@ -714,6 +722,8 @@ github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhO github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -759,8 +769,8 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= -github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -779,8 +789,6 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -801,8 +809,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -838,14 +846,14 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/intel/goresctrl v0.11.0 h1:FnlyrQxgjp9nm2hccwvBDc7qmf9uLxKf0DC6CKobnMA= github.com/intel/goresctrl v0.11.0/go.mod h1:F6jINn5WKuXXHAjmE7eTBjeHPmqT2kiWStrggYIm+UM= github.com/intel/memtierd v0.1.1 h1:hGSN0+dzjaUkwgkJrk6B9SU4dntggXLpXgs9Dm+jfz4= github.com/intel/memtierd v0.1.1/go.mod h1:NFDBvjoDS42gBK/c9q/CYCJ2pt/+g7UQwOOBvQli4z0= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -881,22 +889,27 @@ github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuz github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= @@ -907,22 +920,21 @@ github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= -github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= -github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ= github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= @@ -933,8 +945,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -944,11 +956,13 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -964,6 +978,14 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8= github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -975,6 +997,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.etcd.io/etcd/client/pkg/v3 v3.6.5 h1:Duz9fAzIZFhYWgRjp/FgNq2gO1jId9Yae/rLn3RrBP8= +go.etcd.io/etcd/client/pkg/v3 v3.6.5/go.mod h1:8Wx3eGRPiy0qOFMZT/hfvdos+DjEaPxdIDiCDUv/FQk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -992,12 +1016,12 @@ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 h1:wpMfgF8E1rkrT1Z6meFh1NDtownE9Ii3n3X2GJYjsaU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo= go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= @@ -1018,6 +1042,14 @@ go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOV go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1042,8 +1074,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1086,8 +1118,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1145,8 +1177,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1194,8 +1226,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1279,8 +1311,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -1289,8 +1321,8 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1307,8 +1339,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1380,10 +1412,10 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= -golang.org/x/tools/go/expect v0.1.0-deprecated h1:jY2C5HGYR5lqex3gEniOQL0r7Dq5+VGVgY1nudX5lXY= -golang.org/x/tools/go/expect v0.1.0-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1677,14 +1709,12 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1696,24 +1726,26 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= -k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= -k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= -k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= -k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc= -k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs= -k8s.io/code-generator v0.33.1 h1:ZLzIRdMsh3Myfnx9BaooX6iQry29UJjVfVG+BuS+UMw= -k8s.io/code-generator v0.33.1/go.mod h1:HUKT7Ubp6bOgIbbaPIs9lpd2Q02uqkMCMx9/GjDrWpY= -k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7 h1:2OX19X59HxDprNCVrWi6jb7LW1PoqTlYqEq5H2oetog= -k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= +k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= +k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o= +k8s.io/code-generator v0.35.0 h1:TvrtfKYZTm9oDF2z+veFKSCcgZE3Igv0svY+ehCmjHQ= +k8s.io/code-generator v0.35.0/go.mod h1:iS1gvVf3c/T71N5DOGYO+Gt3PdJ6B9LYSvIyQ4FHzgc= +k8s.io/dynamic-resource-allocation v0.35.0 h1:St6dsCCylLg3HiFPcyHzFF8YQO6yziUDaVRLGdkrNH8= +k8s.io/dynamic-resource-allocation v0.35.0/go.mod h1:uaFga3VJtwyfpfZwpuJG7mlurWGQaaiGUa+QZmooz2U= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= -k8s.io/kubelet v0.31.2 h1:6Hytyw4LqWqhgzoi7sPfpDGClu2UfxmPmaiXPC4FRgI= -k8s.io/kubelet v0.31.2/go.mod h1:0E4++3cMWi2cJxOwuaQP3eMBa7PSOvAFgkTPlVc/2FA= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/kubelet v0.35.0 h1:8cgJHCBCKLYuuQ7/Pxb/qWbJfX1LXIw7790ce9xHq7c= +k8s.io/kubelet v0.35.0/go.mod h1:ciRzAXn7C4z5iB7FhG1L2CGPPXLTVCABDlbXt/Zz8YA= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= @@ -1754,12 +1786,11 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 9f63cbae9..f97766f94 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -22,6 +22,7 @@ import ( "os" "sync" + "github.com/containers/nri-plugins/pkg/kubernetes/client" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -33,7 +34,6 @@ import ( "github.com/containers/nri-plugins/pkg/agent/podresapi" "github.com/containers/nri-plugins/pkg/agent/watch" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1" - k8sclient "k8s.io/client-go/kubernetes" logger "github.com/containers/nri-plugins/pkg/log" ) @@ -127,12 +127,11 @@ type Agent struct { kubeConfig string // kubeconfig path configFile string // configuration file to use instead of custom resource - cfgIf ConfigInterface // custom resource access interface - httpCli *http.Client // shared HTTP client - k8sCli *k8sclient.Clientset // kubernetes client - nrtCli *nrtapi.Client // NRT custom resources client - nrtLock sync.Mutex // serialize NRT custom resource updates - podResCli *podresapi.Client // pod resources API client + cfgIf ConfigInterface // custom resource access interface + k8sCli *client.Client // kubernetes client + nrtCli *nrtapi.Client // NRT custom resources client + nrtLock sync.Mutex // serialize NRT custom resource updates + podResCli *podresapi.Client // pod resources API client notifyFn NotifyFn // config resource change notification callback nodeWatch watch.Interface // kubernetes node watch @@ -264,6 +263,18 @@ func (a *Agent) Stop() { } } +func (a *Agent) NodeName() string { + return a.nodeName +} + +func (a *Agent) KubeClient() *client.Client { + return a.k8sCli +} + +func (a *Agent) KubeConfig() string { + return a.kubeConfig +} + var ( defaultConfig = &cfgapi.AgentConfig{ NodeResourceTopology: true, @@ -295,7 +306,8 @@ func (a *Agent) configure(newConfig metav1.Object) { log.Error("failed to setup NRT client: %w", err) break } - cli, err := nrtapi.NewForConfigAndClient(cfg, a.httpCli) + + cli, err := nrtapi.NewForConfigAndClient(cfg, a.k8sCli.HttpClient()) if err != nil { log.Error("failed to setup NRT client: %w", err) break @@ -331,37 +343,28 @@ func (a *Agent) hasLocalConfig() bool { } func (a *Agent) setupClients() error { + var err error + if a.hasLocalConfig() { log.Warn("running with local configuration, skipping cluster access client setup...") return nil } - // Create HTTP/REST client and K8s client on initial startup. Any failure - // to create these is a failure start up. - if a.httpCli == nil { - log.Info("setting up HTTP/REST client...") - restCfg, err := a.getRESTConfig() - if err != nil { - return err - } - - a.httpCli, err = rest.HTTPClientFor(restCfg) - if err != nil { - return fmt.Errorf("failed to setup kubernetes HTTP client: %w", err) - } + a.k8sCli, err = client.New(client.WithKubeOrInClusterConfig(a.kubeConfig)) + if err != nil { + return err + } - log.Info("setting up K8s client...") - a.k8sCli, err = k8sclient.NewForConfigAndClient(restCfg, a.httpCli) - if err != nil { - a.cleanupClients() - return fmt.Errorf("failed to setup kubernetes client: %w", err) - } + a.nrtCli, err = nrtapi.NewForConfigAndClient(a.k8sCli.RestConfig(), a.k8sCli.HttpClient()) + if err != nil { + a.cleanupClients() + return fmt.Errorf("failed to setup NRT client: %w", err) + } - kubeCfg := *restCfg - err = a.cfgIf.SetKubeClient(a.httpCli, &kubeCfg) - if err != nil { - return fmt.Errorf("failed to setup kubernetes config resource client: %w", err) - } + err = a.cfgIf.SetKubeClient(a.k8sCli.HttpClient(), a.k8sCli.RestConfig()) + if err != nil { + a.cleanupClients() + return fmt.Errorf("failed to setup kubernetes config resource client: %w", err) } a.configure(a.currentCfg) @@ -370,10 +373,9 @@ func (a *Agent) setupClients() error { } func (a *Agent) cleanupClients() { - if a.httpCli != nil { - a.httpCli.CloseIdleConnections() + if a.k8sCli != nil { + a.k8sCli.Close() } - a.httpCli = nil a.k8sCli = nil a.nrtCli = nil } diff --git a/pkg/kubernetes/client/client.go b/pkg/kubernetes/client/client.go new file mode 100644 index 000000000..c60243514 --- /dev/null +++ b/pkg/kubernetes/client/client.go @@ -0,0 +1,194 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "errors" + "net/http" + "strings" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// Option is an option that can be applied to a Client. +type Option func(*Client) error + +// Client enacapsulates our Kubernetes client. +type Client struct { + cfg *rest.Config + http *http.Client + *kubernetes.Clientset +} + +// GetConfigForFile returns a REST configuration for the given file. +func GetConfigForFile(kubeConfig string) (*rest.Config, error) { + return clientcmd.BuildConfigFromFlags("", kubeConfig) +} + +// InClusterConfig returns the in-cluster REST configuration. +func InClusterConfig() (*rest.Config, error) { + return rest.InClusterConfig() +} + +// WithKubeConfig returns a Client Option for using the given kubeconfig file. +func WithKubeConfig(file string) Option { + return func(c *Client) error { + cfg, err := GetConfigForFile(file) + if err != nil { + return err + } + return WithRestConfig(cfg)(c) + } +} + +// WithInClusterConfig returns a Client Option for using the in-cluster configuration. +func WithInClusterConfig() Option { + return func(c *Client) error { + cfg, err := rest.InClusterConfig() + if err != nil { + return err + } + return WithRestConfig(cfg)(c) + } +} + +// WithKubeOrInClusterConfig returns a Client Option for using in-cluster configuration +// if a configuration file is not given. +func WithKubeOrInClusterConfig(file string) Option { + if file == "" { + return WithInClusterConfig() + } + return WithKubeConfig(file) +} + +// WithRestConfig returns a Client Option for using the given REST configuration. +func WithRestConfig(cfg *rest.Config) Option { + return func(c *Client) error { + c.cfg = rest.CopyConfig(cfg) + return nil + } +} + +// WithHttpClient returns a Client Option for using/sharing the given HTTP client. +func WithHttpClient(hc *http.Client) Option { + return func(c *Client) error { + c.http = hc + return nil + } +} + +// WithAcceptContentTypes returns a Client Option for setting the accepted content types. +func WithAcceptContentTypes(contentTypes ...string) Option { + return func(c *Client) error { + if c.cfg == nil { + return errRetryWhenConfigSet + } + c.cfg.AcceptContentTypes = strings.Join(contentTypes, ",") + return nil + } +} + +// WithContentType returns a Client Option for setting the wire format content type. +func WithContentType(contentType string) Option { + return func(c *Client) error { + if c.cfg == nil { + return errRetryWhenConfigSet + } + c.cfg.ContentType = contentType + return nil + } +} + +const ( + ContentTypeJSON = "application/json" + ContentTypeProtobuf = "application/vnd.kubernetes.protobuf" +) + +var ( + // returned by options if applied too early, before a configuration is set + errRetryWhenConfigSet = errors.New("retry when client config is set") +) + +// New creates a new Client with the given options. +func New(options ...Option) (*Client, error) { + c := &Client{} + + var retry []Option + for _, o := range options { + if err := o(c); err != nil { + if err == errRetryWhenConfigSet { + retry = append(retry, o) + } else { + return nil, err + } + } + } + + if c.cfg == nil { + if err := WithInClusterConfig()(c); err != nil { + return nil, err + } + } + + for _, o := range retry { + if err := o(c); err != nil { + return nil, err + } + } + + if c.http == nil { + hc, err := rest.HTTPClientFor(c.cfg) + if err != nil { + return nil, err + } + c.http = hc + } + + client, err := kubernetes.NewForConfigAndClient(c.cfg, c.http) + if err != nil { + return nil, err + } + c.Clientset = client + + return c, nil +} + +// RestConfig returns a shallow copy of the REST configuration of the Client. +func (c *Client) RestConfig() *rest.Config { + cfg := *c.cfg + return &cfg +} + +// HttpClient returns the HTTP client of the Client. +func (c *Client) HttpClient() *http.Client { + return c.http +} + +// K8sClient returns the K8s Clientset of the Client. +func (c *Client) K8sClient() *kubernetes.Clientset { + return c.Clientset +} + +// Close closes the Client. +func (c *Client) Close() { + if c.http != nil { + c.http.CloseIdleConnections() + } + c.cfg = nil + c.http = nil + c.Clientset = nil +} diff --git a/pkg/kubernetes/watch/file.go b/pkg/kubernetes/watch/file.go new file mode 100644 index 000000000..cdc2417e2 --- /dev/null +++ b/pkg/kubernetes/watch/file.go @@ -0,0 +1,180 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watch + +import ( + "errors" + "io/fs" + "os" + "path" + "path/filepath" + "sync" + + "github.com/fsnotify/fsnotify" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8swatch "k8s.io/apimachinery/pkg/watch" +) + +// FileClient takes care of unmarshaling file content to a target object. +type FileClient interface { + Unmarshal([]byte, string) (runtime.Object, error) +} + +type fileWatch struct { + dir string + file string + client FileClient + fsw *fsnotify.Watcher + resultC chan Event + stopOnce sync.Once + stopC chan struct{} + doneC chan struct{} +} + +// File creates a k8s-compatible watch for the given file. Contents of the +// file are unmarshalled into the object of choice by the client. The file +// is monitored for changes and corresponding watch events are generated. +func File(client FileClient, file string) (Interface, error) { + absPath, err := filepath.Abs(file) + if err != nil { + return nil, err + } + + fsw, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + + if err = fsw.Add(filepath.Dir(absPath)); err != nil { + return nil, err + } + + fw := &fileWatch{ + dir: filepath.Dir(absPath), + file: filepath.Base(absPath), + client: client, + fsw: fsw, + resultC: make(chan Event, k8swatch.DefaultChanSize), + stopC: make(chan struct{}), + doneC: make(chan struct{}), + } + + obj, err := fw.readObject() + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + + fw.sendEvent(Added, obj) + + go fw.run() + + return fw, nil +} + +// Stop stops the watch. +func (w *fileWatch) Stop() { + w.stopOnce.Do(func() { + close(w.stopC) + <-w.doneC + w.stopC = nil + }) +} + +// ResultChan returns the channel for receiving events from the watch. +func (w *fileWatch) ResultChan() <-chan Event { + return w.resultC +} + +func (w *fileWatch) run() { + for { + select { + case <-w.stopC: + if err := w.fsw.Close(); err != nil { + log.Warn("%s failed to close fsnotify watcher: %v", w.name(), err) + } + close(w.resultC) + close(w.doneC) + return + + case e, ok := <-w.fsw.Events: + log.Debug("%s got event %+v", w.name(), e) + + if !ok { + w.sendEvent( + Error, + &metav1.Status{ + Status: metav1.StatusFailure, + Message: "failed to receive fsnotify event", + }, + ) + close(w.resultC) + close(w.doneC) + return + } + + if path.Base(e.Name) != w.file { + continue + } + + switch { + case (e.Op & (fsnotify.Create | fsnotify.Write)) != 0: + obj, err := w.readObject() + if err != nil { + log.Debug("%s failed to read/unmarshal: %v", w.name(), err) + continue + } + + if (e.Op & fsnotify.Create) != 0 { + w.sendEvent(Added, obj) + } else { + w.sendEvent(Added, obj) + } + + case (e.Op & (fsnotify.Remove | fsnotify.Rename)) != 0: + w.sendEvent(Deleted, nil) + } + } + } +} + +func (w *fileWatch) sendEvent(t EventType, obj runtime.Object) { + select { + case w.resultC <- Event{Type: t, Object: obj}: + default: + log.Warn("failed to deliver fileWatch %v event", t) + } +} + +func (w *fileWatch) readObject() (runtime.Object, error) { + file := path.Join(w.dir, w.file) + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + + obj, err := w.client.Unmarshal(data, file) + if err != nil { + return nil, err + } + + log.Debug("%s read object %+v", w.name(), obj) + + return obj, nil +} + +func (w *fileWatch) name() string { + return path.Join("filewatch/path:", path.Join(w.dir, w.file)) +} diff --git a/pkg/kubernetes/watch/object.go b/pkg/kubernetes/watch/object.go new file mode 100644 index 000000000..eefe3fed4 --- /dev/null +++ b/pkg/kubernetes/watch/object.go @@ -0,0 +1,160 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watch + +import ( + "path" + "sync" + "time" + + k8swatch "k8s.io/apimachinery/pkg/watch" +) + +const ( + createDelay = 1 * time.Second +) + +type watch struct { + sync.Mutex + subject string + client ObjectClient + w Interface + resultC chan Event + createC <-chan time.Time + + stopOnce sync.Once + stopC chan struct{} + doneC chan struct{} +} + +// ObjectClient takes care of low-level details of creating a wrapped watch. +type ObjectClient interface { + CreateWatch() (Interface, error) +} + +// Object creates a wrapped watch using the given client. The watch +// is transparently recreated upon expiration and any errors. +func Object(client ObjectClient, subject string) (Interface, error) { + w := &watch{ + subject: subject, + client: client, + resultC: make(chan Event, k8swatch.DefaultChanSize), + stopC: make(chan struct{}), + doneC: make(chan struct{}), + } + + if err := w.create(); err != nil { + return nil, err + } + + go w.run() + + return w, nil +} + +// Stop stops the watch. +func (w *watch) Stop() { + w.stopOnce.Do(func() { + close(w.stopC) + <-w.doneC + w.stop() + w.stopC = nil + }) +} + +// ResultChan returns the channel for receiving events from the watch. +func (w *watch) ResultChan() <-chan Event { + return w.resultC +} + +func (w *watch) eventChan() <-chan Event { + if w.w == nil { + return nil + } + + return w.w.ResultChan() +} + +func (w *watch) run() { + for { + select { + case <-w.stopC: + close(w.resultC) + close(w.doneC) + return + + case e, ok := <-w.eventChan(): + if !ok { + log.Debug("%s failed...", w.name()) + w.stop() + w.scheduleCreate() + continue + } + + if e.Type == Error { + log.Debug("%s failed with an error...", w.name()) + w.stop() + w.scheduleCreate() + continue + } + + select { + case w.resultC <- e: + default: + log.Debug("%s failed to deliver %v event", w.name(), e.Type) + w.stop() + w.scheduleCreate() + } + + case <-w.createC: + w.createC = nil + log.Debug("reopening %s...", w.name()) + if err := w.create(); err != nil { + w.scheduleCreate() + } + } + } +} + +func (w *watch) create() error { + w.stop() + + wif, err := w.client.CreateWatch() + if err != nil { + return err + } + w.w = wif + + return nil +} + +func (w *watch) scheduleCreate() { + if w.createC == nil { + w.createC = time.After(createDelay) + } +} + +func (w *watch) stop() { + if w.w == nil { + return + } + + w.w.Stop() + w.w = nil +} + +func (w *watch) name() string { + return path.Join("watch", w.subject) +} diff --git a/pkg/kubernetes/watch/watch.go b/pkg/kubernetes/watch/watch.go new file mode 100644 index 000000000..095215211 --- /dev/null +++ b/pkg/kubernetes/watch/watch.go @@ -0,0 +1,38 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watch + +import ( + logger "github.com/containers/nri-plugins/pkg/log" + k8swatch "k8s.io/apimachinery/pkg/watch" +) + +type ( + Interface = k8swatch.Interface + EventType = k8swatch.EventType + Event = k8swatch.Event +) + +const ( + Added = k8swatch.Added + Modified = k8swatch.Modified + Deleted = k8swatch.Deleted + Bookmark = k8swatch.Bookmark + Error = k8swatch.Error +) + +var ( + log = logger.Get("watch") +) diff --git a/pkg/log/yaml-formatter.go b/pkg/log/yaml-formatter.go new file mode 100644 index 000000000..ca6b581fb --- /dev/null +++ b/pkg/log/yaml-formatter.go @@ -0,0 +1,41 @@ +// Copyright 2019-2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "sigs.k8s.io/yaml" +) + +func AsYaml(o any) *YamlFormatter { + return &YamlFormatter{o: o} +} + +type YamlFormatter struct { + o any +} + +func (f *YamlFormatter) String() string { + if f == nil || f.o == nil { + return "" + } + + // Use a YAML marshaller to format the object. + data, err := yaml.Marshal(f.o) + if err != nil { + return "" + } + + return string(data) +} diff --git a/pkg/resmgr/cache/cache.go b/pkg/resmgr/cache/cache.go index fef120d12..da265b9fd 100644 --- a/pkg/resmgr/cache/cache.go +++ b/pkg/resmgr/cache/cache.go @@ -204,6 +204,8 @@ type Container interface { GetAnnotation(key string, objPtr interface{}) (string, bool) // GetEnv returns the value of a container environment variable. GetEnv(string) (string, bool) + // GetEnvList return the list of environment variables for the container. + GetEnvList() []string // GetMounts returns all the mounts of the container. GetMounts() []*Mount // GetDevices returns all the linux devices of the container. @@ -448,6 +450,11 @@ type Cache interface { // GetPolicyEntry gets the policy entry for a key. GetPolicyEntry(string, interface{}) bool + // SetEntry sets the entry for a key. + SetEntry(string, interface{}) + // GetEntry gets the entry for a key. + GetEntry(string, interface{}) (interface{}, error) + // Save requests a cache save. Save() error @@ -500,6 +507,8 @@ type cache struct { PolicyName string // name of the active policy policyData map[string]interface{} // opaque policy data PolicyJSON map[string]string // ditto in raw, unmarshaled form + entryData map[string]interface{} // opaque cached data + entryJSON map[string]string // ditto in raw, unmarshaled form pending map[string]struct{} // cache IDs of containers with pending changes @@ -525,6 +534,8 @@ func NewCache(options Options) (Cache, error) { NextID: 1, policyData: make(map[string]interface{}), PolicyJSON: make(map[string]string), + entryData: make(map[string]interface{}), + entryJSON: make(map[string]string), implicit: make(map[string]ImplicitAffinity), } @@ -829,6 +840,30 @@ func (cch *cache) GetContainers() []Container { return containers } +// Set the entry for a key. +func (cch *cache) SetEntry(key string, obj interface{}) { + cch.entryData[key] = obj +} + +var ErrNoEntry = errors.New("no cached entry found") + +// Get the entry for a key. +func (cch *cache) GetEntry(key string, ptr interface{}) (interface{}, error) { + if obj, ok := cch.entryData[key]; ok { + return obj, nil + } + if data, ok := cch.entryJSON[key]; ok { + err := json.Unmarshal([]byte(data), ptr) + if err != nil { + return nil, cacheError("failed to unmarshal entry for key '%s': %w", key, err) + } + cch.entryData[key] = ptr + return ptr, nil + } + + return nil, ErrNoEntry +} + // Set the policy entry for a key. func (cch *cache) SetPolicyEntry(key string, obj interface{}) { cch.policyData[key] = obj @@ -1092,6 +1127,7 @@ type snapshot struct { Pods map[string]*pod Containers map[string]*container NextID uint64 + Entries map[string]string PolicyName string PolicyJSON map[string]string } @@ -1105,6 +1141,7 @@ func (cch *cache) Snapshot() ([]byte, error) { NextID: cch.NextID, PolicyName: cch.PolicyName, PolicyJSON: cch.PolicyJSON, + Entries: make(map[string]string), } for id, p := range cch.Pods { @@ -1124,6 +1161,14 @@ func (cch *cache) Snapshot() ([]byte, error) { s.PolicyJSON[key] = string(data) } + for key, obj := range cch.entryData { + data, err := json.Marshal(obj) + if err != nil { + return nil, cacheError("failed to marshal entry '%s': %v", key, err) + } + s.Entries[key] = string(data) + } + data, err := json.Marshal(s) if err != nil { return nil, cacheError("failed to marshal cache: %v", err) @@ -1138,6 +1183,7 @@ func (cch *cache) Restore(data []byte) error { Pods: make(map[string]*pod), Containers: make(map[string]*container), PolicyJSON: make(map[string]string), + Entries: make(map[string]string), } if err := json.Unmarshal(data, &s); err != nil { @@ -1155,6 +1201,8 @@ func (cch *cache) Restore(data []byte) error { cch.PolicyJSON = s.PolicyJSON cch.PolicyName = s.PolicyName cch.policyData = make(map[string]interface{}) + cch.entryData = make(map[string]interface{}) + cch.entryJSON = s.Entries for _, p := range cch.Pods { p.cache = cch diff --git a/pkg/resmgr/cache/container.go b/pkg/resmgr/cache/container.go index f9ec72118..3aec8e62f 100644 --- a/pkg/resmgr/cache/container.go +++ b/pkg/resmgr/cache/container.go @@ -455,6 +455,10 @@ func (c *container) GetEnv(key string) (string, bool) { return "", false } +func (c *container) GetEnvList() []string { + return slices.Clone(c.Ctr.GetEnv()) +} + func (c *container) GetMounts() []*Mount { var mounts []*Mount diff --git a/pkg/resmgr/dra.go b/pkg/resmgr/dra.go new file mode 100644 index 000000000..75d57b08e --- /dev/null +++ b/pkg/resmgr/dra.go @@ -0,0 +1,486 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resmgr + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/containers/nri-plugins/pkg/kubernetes/client" + logger "github.com/containers/nri-plugins/pkg/log" + "github.com/containers/nri-plugins/pkg/resmgr/cache" + system "github.com/containers/nri-plugins/pkg/sysfs" + "google.golang.org/grpc" + resapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/dynamic-resource-allocation/kubeletplugin" + "k8s.io/dynamic-resource-allocation/resourceslice" +) + +type draPlugin struct { + driverName string + nodeName string + resmgr *resmgr + cancel context.CancelFunc + plugin *kubeletplugin.Helper + publishCh chan<- *resourceslice.DriverResources + claims savedClaims +} + +type UID = types.UID + +var ( + dra = logger.NewLogger("dra-driver") +) + +const ( + driverName = "native.cpu" + driverKind = driverName + "/device" + cdiVersion = "0.7.0" + cdiEnvVar = "DRA_CPU" +) + +func (resmgr *resmgr) publishCPUs(cpuIDs []system.ID) error { + if resmgr.dra == nil { + return fmt.Errorf("can't publish CPUs as DRA devices, no DRA plugin") + } + + if err := resmgr.dra.writeCDISpecFile(opt.HostRoot, cpuIDs); err != nil { + log.Errorf("failed to write CDI Spec file: %v", err) + return err + } + + cpuDevices := resmgr.system.CPUsAsDRADevices(cpuIDs) + if err := resmgr.dra.PublishResources(context.Background(), cpuDevices); err != nil { + log.Errorf("failed to publish DRA resources: %v", err) + return err + } + + return nil +} + +func newDRAPlugin(resmgr *resmgr) (*draPlugin, error) { + driverPath := filepath.Join(kubeletplugin.KubeletPluginsDir, driverName) + if err := os.MkdirAll(driverPath, 0750); err != nil { + return nil, fmt.Errorf("failed to create driver directory %s: %w", driverPath, err) + } + + p := &draPlugin{ + driverName: driverName, + nodeName: resmgr.agent.NodeName(), + resmgr: resmgr, + claims: make(map[UID]*draClaim), + } + + p.restoreClaims() + + return p, nil +} + +func (p *draPlugin) start() error { + p.run() + return nil +} + +func (p *draPlugin) run() { + var ( + ctx, cancel = context.WithCancel(context.Background()) + publishCh = make(chan *resourceslice.DriverResources, 1) + ) + + go func() { + for { + var resources *resourceslice.DriverResources + + select { + case <-ctx.Done(): + return + case r, ok := <-publishCh: + if !ok { + return + } + resources = r + } + + if p.plugin == nil { + if err := p.connect(); err != nil { + log.Errorf("failed start DRA plugin: %v", err) + continue + } + } + + if p.plugin != nil { + if err := p.plugin.PublishResources(ctx, *resources); err != nil { + log.Errorf("failed to publish DRA resources: %v", err) + } else { + log.Infof("published DRA resources, using %d pool(s)...", len(resources.Pools)) + } + } + + resources = nil + } + }() + + p.cancel = cancel + p.publishCh = publishCh +} + +func (p *draPlugin) connect() error { + kubeClient, err := client.New( + client.WithKubeOrInClusterConfig(p.resmgr.agent.KubeConfig()), + client.WithAcceptContentTypes(client.ContentTypeProtobuf, client.ContentTypeJSON), + client.WithContentType(client.ContentTypeProtobuf), + ) + if err != nil { + return fmt.Errorf("can't create kube client for DRA plugin: %w", err) + } + + options := []kubeletplugin.Option{ + kubeletplugin.DriverName(p.driverName), + kubeletplugin.NodeName(p.nodeName), + kubeletplugin.KubeClient(kubeClient.Clientset), + kubeletplugin.GRPCInterceptor(p.unaryInterceptor), + } + + log.Infof("using DRA driverName=%s nodeName=%s", p.driverName, p.nodeName) + + plugin, err := kubeletplugin.Start(context.Background(), p, options...) + if err != nil { + return fmt.Errorf("failed to start DRA plugin: %w", err) + } + + p.plugin = plugin + return nil +} + +func (p *draPlugin) stop() { + if p == nil { + return + } + + if p.plugin != nil { + p.plugin.Stop() + } + if p.cancel != nil { + p.cancel() + } + + p.plugin = nil + p.cancel = nil +} + +func (p *draPlugin) unaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + dra.Info("=> gRPC call %v, handler %v\n", *info, handler) + rpl, err := handler(ctx, req) + dra.Info("<= gRPC reply: %+v, %v\n", rpl, err) + return rpl, err +} + +func (p *draPlugin) IsRegistered() (bool, error) { + if p == nil || p.plugin == nil { + return false, errors.New("DRA plugin is not initialized") + } + + status := p.plugin.RegistrationStatus() + if status == nil { + return false, nil + } + + var err error + if status.Error != "" { + err = errors.New(status.Error) + } + + return status.PluginRegistered, err +} + +func (p *draPlugin) PublishResources(ctx context.Context, devices []resapi.Device) error { + resources := resourceslice.DriverResources{ + Pools: make(map[string]resourceslice.Pool), + } + + maxPerPool := resapi.ResourceSliceMaxDevices + for n := len(devices); n > 0; n = len(devices) { + if n > maxPerPool { + n = maxPerPool + } + resources.Pools["pool"+strconv.Itoa(len(resources.Pools))] = resourceslice.Pool{ + Slices: []resourceslice.Slice{ + { + Devices: devices[:n], + }, + }, + } + devices = devices[n:] + } + + log.Infof("publishing DRA resources, using %d pool(s)...", len(resources.Pools)) + + select { + case p.publishCh <- &resources: + return nil + default: + } + + return fmt.Errorf("failed to publish resources, failed to send on channel") +} + +func (p *draPlugin) writeCDISpecFile(hostRoot string, cpuIDs []system.ID) error { + spec := bytes.NewBuffer(nil) + fmt.Fprintf(spec, "cdiVersion: %s\nkind: %s\ndevices:\n", cdiVersion, driverKind) + for _, id := range cpuIDs { + fmt.Fprintf(spec, " - name: cpu%d\n", id) + fmt.Fprintf(spec, " containerEdits:\n") + fmt.Fprintf(spec, " env:\n") + fmt.Fprintf(spec, " - %s%d=1\n", cdiEnvVar, id) + } + + dir := filepath.Join(hostRoot, "/var/run/cdi") + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create CDI Spec directory: %w", err) + } + + if err := os.WriteFile(filepath.Join(dir, driverName+".yaml"), spec.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to write CDI Spec file: %w", err) + } + + return nil +} + +func (p *draPlugin) PrepareResourceClaims(ctx context.Context, claims []*resapi.ResourceClaim) (map[UID]kubeletplugin.PrepareResult, error) { + log.Infof("should prepare %d claims:", len(claims)) + + undoAndErrOut := func(err error, release []*resapi.ResourceClaim) error { + for _, c := range release { + if relc := p.claims.del(c.UID); relc != nil { + if undoErr := p.resmgr.policy.ReleaseClaim(relc); undoErr != nil { + log.Error("rollback error, failed to release claim %s: %v", c.UID, undoErr) + } + } + } + return err + } + + defer func() { + if err := p.saveClaims(); err != nil { + log.Error("failed to save claims: %v", err) + } + }() + p.resmgr.Lock() + defer p.resmgr.Unlock() + + result := make(map[UID]kubeletplugin.PrepareResult) + + for i, c := range claims { + if c == nil { + continue + } + + dra.Debug(" - claim #%d:", i) + specHdr := fmt.Sprintf(" ", i) + statusHdr := fmt.Sprintf(" ", i) + dra.DebugBlock(specHdr, "%s", logger.AsYaml(c.Spec)) + dra.DebugBlock(statusHdr, "%s", logger.AsYaml(c.Status)) + + if old, ok := p.claims.get(c.UID); ok { + log.Infof("claim %q already prepared, reusing it", c.UID) + result[c.UID] = *(old.GetResult()) + continue + } + + claim := &draClaim{ResourceClaim: c} + + if err := p.resmgr.policy.AllocateClaim(claim); err != nil { + log.Error("failed to prepare claim %q: %v", c.UID, err) + return nil, undoAndErrOut(err, claims[:i]) + } + + result[claim.GetUID()] = *(claim.GetResult()) + p.claims.add(claim) + } + + return result, nil +} + +func (p *draPlugin) UnprepareResourceClaims(ctx context.Context, claims []kubeletplugin.NamespacedObject) (map[UID]error, error) { + + log.Infof("should un-prepare %d claims:", len(claims)) + + defer func() { + if err := p.saveClaims(); err != nil { + log.Error("failed to save claims: %v", err) + } + }() + p.resmgr.Lock() + defer p.resmgr.Unlock() + + result := make(map[UID]error) + + for _, c := range claims { + log.Infof(" - un-claim %+v", c) + + if claim := p.claims.del(c.UID); claim != nil { + if err := p.resmgr.policy.ReleaseClaim(claim); err != nil { + log.Errorf("failed to release claim %s: %v", claim, err) + } + } + + result[c.UID] = nil + } + + if err := p.resmgr.nri.updateContainers(); err != nil { + log.Errorf("failed to update containers after releasing claim: %v", err) + } + + return result, nil +} + +func (p *draPlugin) ErrorHandler(ctx context.Context, err error, msg string) { + log.Errorf("resource slice publishing error: %v (%s)", err, msg) +} + +func (p *draPlugin) HandleError(ctx context.Context, err error, msg string) { + log.Errorf("resource slice publishing error: %v (%s)", err, msg) +} + +func (p *draPlugin) saveClaims() error { + p.resmgr.cache.SetEntry("claims", p.claims) + return p.resmgr.cache.Save() +} + +func (p *draPlugin) restoreClaims() { + claims := make(savedClaims) + restored, err := p.resmgr.cache.GetEntry("claims", &claims) + + if err != nil { + if err != cache.ErrNoEntry { + log.Error("failed to restore claims: %v", err) + } + p.claims = make(savedClaims) + } else { + if restored == nil { + p.claims = make(savedClaims) + } else { + p.claims = *restored.(*savedClaims) + } + } +} + +type draClaim struct { + *resapi.ResourceClaim + pods []UID + devs []system.ID +} + +func (c *draClaim) GetUID() UID { + if c == nil || c.ResourceClaim == nil { + return "" + } + return c.UID +} + +func (c *draClaim) GetPods() []UID { + if c == nil || c.ResourceClaim == nil { + return nil + } + if c.pods != nil { + return c.pods + } + + var pods []UID + for _, r := range c.Status.ReservedFor { + if r.Resource == "pods" { + pods = append(pods, r.UID) + } + } + c.pods = pods + + return c.pods +} + +func (c *draClaim) GetDevices() []system.ID { + if c == nil || c.ResourceClaim == nil { + return nil + } + + if c.devs != nil { + return c.devs + } + + var ids []system.ID + for _, r := range c.Status.Allocation.Devices.Results { + num := strings.TrimPrefix(r.Device, "cpu") + i, err := strconv.ParseInt(num, 10, 32) + if err != nil { + log.Errorf("failed to parse CPU ID %q: %v", num, err) + continue + } + ids = append(ids, system.ID(i)) + } + c.devs = ids + + return c.devs +} + +func (c *draClaim) GetResult() *kubeletplugin.PrepareResult { + result := &kubeletplugin.PrepareResult{} + for _, alloc := range c.Status.Allocation.Devices.Results { + result.Devices = append(result.Devices, + kubeletplugin.Device{ + Requests: []string{alloc.Request}, + DeviceName: alloc.Device, + CDIDeviceIDs: []string{driverKind + "=" + alloc.Device}, + }) + } + return result +} + +func (c *draClaim) String() string { + return fmt.Sprintf("", + c.GetUID(), c.GetDevices(), c.GetPods()) +} + +func (c *draClaim) MarshalJSON() ([]byte, error) { + return json.Marshal(c.ResourceClaim) +} + +func (c *draClaim) UnmarshalJSON(b []byte) error { + c.ResourceClaim = &resapi.ResourceClaim{} + return json.Unmarshal(b, c.ResourceClaim) +} + +type savedClaims map[UID]*draClaim + +func (s *savedClaims) add(c *draClaim) { + (*s)[c.UID] = c +} + +func (s *savedClaims) del(uid UID) *draClaim { + c := (*s)[uid] + delete(*s, uid) + return c +} + +func (s *savedClaims) get(uid UID) (*draClaim, bool) { + c, ok := (*s)[uid] + return c, ok +} diff --git a/pkg/resmgr/policy/policy.go b/pkg/resmgr/policy/policy.go index 760fb316e..7c8cb898a 100644 --- a/pkg/resmgr/policy/policy.go +++ b/pkg/resmgr/policy/policy.go @@ -20,6 +20,7 @@ import ( "sort" "k8s.io/apimachinery/pkg/api/resource" + apitypes "k8s.io/apimachinery/pkg/types" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/events" @@ -56,6 +57,8 @@ type ConstraintSet map[Domain]Constraint type Options struct { // SendEvent is the function for delivering events back to the resource manager. SendEvent SendEventFn + // PublishCPUs is the function for publishing CPUs as DRA devices. + PublishCPUs PublishCPUFn } // BackendOptions describes the options for a policy backend instance @@ -66,6 +69,8 @@ type BackendOptions struct { Cache cache.Cache // SendEvent is the function for delivering events up to the resource manager. SendEvent SendEventFn + // PublishCPUs publishes CPUs as DRA devices. + PublishCPUs PublishCPUFn // Config is the policy-specific configuration. Config interface{} } @@ -76,6 +81,22 @@ type CreateFn func(*BackendOptions) Backend // SendEventFn is the type for a function to send events back to the resource manager. type SendEventFn func(interface{}) error +// PublishCPUFn is the type for a function to publish CPUs as DRA devices. +type PublishCPUFn func([]ID) error + +// Claim represents a DRA Resource Claim, for CPUs. +type Claim interface { + GetUID() UID + GetPods() []UID + GetDevices() []ID + String() string +} + +type ( + UID = apitypes.UID + ID = system.ID +) + const ( // ExportedResources is the basename of the file container resources are exported to. ExportedResources = "resources.sh" @@ -85,6 +106,8 @@ const ( ExportIsolatedCPUs = "ISOLATED_CPUS" // ExportExclusiveCPUs is the shell variable used to export exclusive container CPUs. ExportExclusiveCPUs = "EXCLUSIVE_CPUS" + // ExportClaimedCPUs is the shell variable used to export DRA-claimed container CPUs. + ExportClaimedCPUs = "CLAIMED_CPUS" ) // Backend is the policy (decision making logic) interface exposed by implementations. @@ -110,6 +133,10 @@ type Backend interface { ReleaseResources(cache.Container) error // UpdateResources updates resource allocations of a container. UpdateResources(cache.Container) error + // AllocateClaim allocates CPUs for the claim. + AllocateClaim(Claim) error + // ReleaseClaim releases CPUs of the claim. + ReleaseClaim(Claim) error // HandleEvent processes the given event. The returned boolean indicates whether // changes have been made to any of the containers while handling the event. HandleEvent(*events.Policy) (bool, error) @@ -123,6 +150,8 @@ type Backend interface { type Policy interface { // ActivePolicy returns the name of the policy backend in use. ActivePolicy() string + // System returns the sysfs instance used by the policy. + System() system.System // Start starts up policy, prepare for serving resource management requests. Start(interface{}) error // Reconfigure the policy. @@ -135,6 +164,10 @@ type Policy interface { ReleaseResources(cache.Container) error // UpdateResources updates resource allocations of a container. UpdateResources(cache.Container) error + // AllocateClaim allocates CPUs for the claim. + AllocateClaim(Claim) error + // ReleaseClaim releases CPUs of the claim. + ReleaseClaim(Claim) error // HandleEvent passes on the given event to the active policy. The returned boolean // indicates whether changes have been made to any of the containers while handling // the event. @@ -244,15 +277,20 @@ func (p *policy) ActivePolicy() string { return "" } +func (p *policy) System() system.System { + return p.system +} + // Start starts up policy, preparing it for serving requests. func (p *policy) Start(cfg interface{}) error { log.Info("activating '%s' policy...", p.active.Name()) if err := p.active.Setup(&BackendOptions{ - Cache: p.cache, - System: p.system, - SendEvent: p.options.SendEvent, - Config: cfg, + Cache: p.cache, + System: p.system, + SendEvent: p.options.SendEvent, + PublishCPUs: p.options.PublishCPUs, + Config: cfg, }); err != nil { return err } @@ -300,6 +338,16 @@ func (p *policy) UpdateResources(c cache.Container) error { return p.active.UpdateResources(c) } +// AllocateClaim allocates CPUs for the claim. +func (p *policy) AllocateClaim(claim Claim) error { + return p.active.AllocateClaim(claim) +} + +// ReleaseClaim releases CPUs of the claim. +func (p *policy) ReleaseClaim(claim Claim) error { + return p.active.ReleaseClaim(claim) +} + // HandleEvent passes on the given event to the active policy. func (p *policy) HandleEvent(e *events.Policy) (bool, error) { return p.active.HandleEvent(e) diff --git a/pkg/resmgr/resource-manager.go b/pkg/resmgr/resource-manager.go index d75ec3d2d..9b02b1148 100644 --- a/pkg/resmgr/resource-manager.go +++ b/pkg/resmgr/resource-manager.go @@ -53,6 +53,7 @@ type resmgr struct { agent *agent.Agent cfg cfgapi.ResmgrConfig cache cache.Cache // cached state + system sysfs.System // sysfs in use policy policy.Policy // resource manager policy control control.Control // policy controllers/enforcement events chan interface{} // channel for delivering events @@ -60,6 +61,7 @@ type resmgr struct { nri *nriPlugin // NRI plugins, if we're running as such rdt *rdtControl // control for RDT allocation and monitoring blkio *blkioControl // control for block I/O prioritization and throttling + dra *draPlugin // DRA plugin, if we have one running bool } @@ -88,7 +90,7 @@ func NewResourceManager(backend policy.Backend, agt *agent.Agent) (ResourceManag return nil, err } - log.Info("running as an NRI plugin...") + log.Info("creating NRI plugin...") nrip, err := newNRIPlugin(m) if err != nil { return nil, err @@ -99,6 +101,13 @@ func NewResourceManager(backend policy.Backend, agt *agent.Agent) (ResourceManag return nil, err } + log.Info("creating DRA plugin...") + drap, err := newDRAPlugin(m) + if err != nil { + return nil, err + } + m.dra = drap + if err := m.setupEventProcessing(); err != nil { return nil, err } @@ -186,6 +195,10 @@ func (m *resmgr) start(cfg cfgapi.ResmgrConfig) error { return err } + if err := m.dra.start(); err != nil { + return err + } + if err := m.policy.Start(m.cfg.PolicyConfig()); err != nil { return err } @@ -222,6 +235,7 @@ func (m *resmgr) Stop() { defer m.Unlock() m.nri.stop() + m.dra.stop() } // setupCache creates a cache and reloads its last saved state if found. @@ -249,10 +263,17 @@ func (m *resmgr) setupPolicy(backend policy.Backend) error { log.Warnf("failed to set active policy: %v", err) } - p, err := policy.NewPolicy(backend, m.cache, &policy.Options{SendEvent: m.SendEvent}) + p, err := policy.NewPolicy(backend, m.cache, + &policy.Options{ + SendEvent: m.SendEvent, + PublishCPUs: m.publishCPUs, + }, + ) if err != nil { return resmgrError("failed to create policy %s: %v", backend.Name(), err) } + + m.system = p.System() m.policy = p return nil diff --git a/pkg/sysfs/dra.go b/pkg/sysfs/dra.go new file mode 100644 index 000000000..a6300809e --- /dev/null +++ b/pkg/sysfs/dra.go @@ -0,0 +1,119 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysfs + +import ( + "fmt" + "strconv" + + idset "github.com/intel/goresctrl/pkg/utils" + resapi "k8s.io/api/resource/v1" +) + +type ( + ID = idset.ID + QualifiedName = resapi.QualifiedName + Attribute = resapi.DeviceAttribute +) + +const ( + // Names for standard CPU device attributes. + AttrPackage = QualifiedName("package") + AttrDie = QualifiedName("die") + AttrCluster = QualifiedName("cluster") + AttrCore = QualifiedName("core") + AttrCoreType = QualifiedName("coreType") + AttrLocalMemory = QualifiedName("localMemory") + AttrIsolated = QualifiedName("isolated") + AttrMinFreq = QualifiedName("minFreq") + AttrMaxFreq = QualifiedName("maxFreq") + AttrBaseFreq = QualifiedName("baseFreq") +) + +// CPUsAsDRADevices returns the given CPUs as DRA devices. +func (sys *system) CPUsAsDRADevices(ids []ID) []resapi.Device { + devices := make([]resapi.Device, 0, len(ids)) + for _, id := range ids { + devices = append(devices, *(sys.CPU(id).DRA())) + } + return devices +} + +// DRA returns the CPU represented as a DRA device. +func (c *cpu) DRA(extras ...map[QualifiedName]Attribute) *resapi.Device { + dra := &resapi.Device{ + Name: "cpu" + strconv.Itoa(c.ID()), + Attributes: map[QualifiedName]Attribute{ + AttrPackage: Attr(c.PackageID()), + AttrDie: Attr(c.DieID()), + AttrCluster: Attr(c.ClusterID()), + AttrCore: Attr(c.CoreID()), + AttrCoreType: Attr(c.CoreKind().String()), + AttrLocalMemory: Attr(c.NodeID()), + AttrIsolated: Attr(c.Isolated()), + }, + } + + if base := c.FrequencyRange().Base; base > 0 { + dra.Attributes[AttrBaseFreq] = Attr(base) + } + if min := c.FrequencyRange().Min; min > 0 { + dra.Attributes[AttrMinFreq] = Attr(min) + } + if max := c.FrequencyRange().Max; max > 0 { + dra.Attributes[AttrMaxFreq] = Attr(max) + } + + for idx, cache := range c.GetCaches() { + dra.Attributes[QualifiedName(fmt.Sprintf("cache%dID", idx))] = Attr(cache.EnumID()) + } + + for _, m := range extras { + for name, value := range m { + if _, ok := dra.Attributes[name]; !ok { + dra.Attributes[name] = value + } + } + } + + return dra +} + +// Attr returns an attribute for the given value. +func Attr(value any) Attribute { + switch v := any(value).(type) { + case int64: + return Attribute{IntValue: &v} + case int: + val := int64(v) + return Attribute{IntValue: &val} + case uint64: + val := int64(v) + return Attribute{IntValue: &val} + case int32: + val := int64(v) + return Attribute{IntValue: &val} + case uint32: + val := int64(v) + return Attribute{IntValue: &val} + case string: + return Attribute{StringValue: &v} + case bool: + return Attribute{BoolValue: &v} + default: + val := fmt.Sprintf("", value) + return Attribute{StringValue: &val} + } +} diff --git a/pkg/sysfs/system.go b/pkg/sysfs/system.go index b013d9f4e..de78819dd 100644 --- a/pkg/sysfs/system.go +++ b/pkg/sysfs/system.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/containers/nri-plugins/pkg/utils/cpuset" + resapi "k8s.io/api/resource/v1" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/utils" @@ -134,6 +135,8 @@ type System interface { Isolated() cpuset.CPUSet NodeHintToCPUs(string) string + + CPUsAsDRADevices(ids []idset.ID) []resapi.Device } // System devices @@ -230,6 +233,7 @@ type CPU interface { GetLastLevelCaches() []*Cache GetLastLevelCacheCPUSet() cpuset.CPUSet CoreKind() CoreKind + DRA(extras ...map[QualifiedName]Attribute) *resapi.Device } type cpu struct { @@ -313,11 +317,12 @@ const ( // Cache has details about a CPU cache. type Cache struct { - id idset.ID // cache id - level int // cache type - kind CacheType // cache type - size uint64 // cache size - cpus idset.IDSet // CPUs sharing this cache + enumID int // enumerated cache id + id idset.ID // cache id + level int // cache type + kind CacheType // cache type + size uint64 // cache size + cpus idset.IDSet // CPUs sharing this cache } // cacheOverrides is a list of cache overrides for specific CPU sets. @@ -392,6 +397,11 @@ func (sys *system) Discover(flags DiscoveryFlag) error { if err := sys.discoverCPUs(); err != nil { return err } + if (flags & DiscoverCache) != 0 { + if err := sys.enumerateCaches(); err != nil { + return fmt.Errorf("failed to enumerate caches: %v", err) + } + } if err := sys.discoverNodes(); err != nil { return err } @@ -489,9 +499,10 @@ func (sys *system) Discover(flags DiscoveryFlag) error { sys.Debug(" base freq: %d", cpu.freq.Base) sys.Debug(" freq: %d - %d", cpu.freq.Min, cpu.freq.Max) sys.Debug(" epp: %d", cpu.epp) + sys.DebugBlock(" ", "%s", logger.AsYaml(cpu.DRA())) for idx, c := range cpu.caches { - sys.Debug(" cache #%d:", idx) + sys.Debug(" cache #%d (enumerated as #%d):", idx, c.enumID) sys.Debug(" id: %d", c.id) sys.Debug(" level: %d", c.level) sys.Debug(" kind: %s", c.kind) @@ -499,6 +510,9 @@ func (sys *system) Discover(flags DiscoveryFlag) error { sys.Debug(" cpus: %s", c.SharedCPUSet().String()) } } + + sys.DebugBlock(" ", "%s", + logger.AsYaml(sys.CPUsAsDRADevices(sys.CPUIDs()))) } return nil @@ -1308,6 +1322,13 @@ func (c *cpu) CoreKind() CoreKind { return c.coreKind } +func (c *Cache) EnumID() int { + if c == nil { + return 0 + } + return c.enumID +} + func (c *Cache) ID() int { if c == nil { return 0 @@ -2084,6 +2105,55 @@ func (sys *system) saveCache(c *Cache) *Cache { return c } +func (sys *system) enumerateCaches() error { + log.Debug("enumerating CPU caches...") + cMap := map[int]map[CacheType]map[idset.ID]*Cache{} + enumerated := []*Cache{} + for _, caches := range sys.caches { + for _, cacheMap := range caches { + for _, c := range cacheMap { + l, ok := cMap[c.level] + if !ok { + l = map[CacheType]map[idset.ID]*Cache{} + cMap[c.level] = l + } + k, ok := l[c.kind] + if !ok { + k = map[idset.ID]*Cache{} + l[c.kind] = k + } + _, ok = k[c.id] + if !ok { + k[c.id] = c + enumerated = append(enumerated, c) + } + } + } + } + + slices.SortFunc(enumerated, func(ci, cj *Cache) int { + if ci.level != cj.level { + return ci.level - cj.level + } + if ci.kind != cj.kind { + return int(ci.kind) - int(cj.kind) + } + return ci.id - cj.id + }) + + for id, c := range enumerated { + c.enumID = id + sys.Debug(" enumerated cache #%d:", c.enumID) + sys.Debug(" id: %d", c.id) + sys.Debug(" level: %d", c.level) + sys.Debug(" kind: %s", c.kind) + sys.Debug(" size: %dK", c.size/1024) + sys.Debug(" cpus: %s", c.SharedCPUSet().String()) + } + + return nil +} + // eppStrings initialized this way to better catch changes in the enum var eppStrings = func() [EPPUnknown]string { var e [EPPUnknown]string