From 2b3a577e9f77f09ce3875de4e899157e55b0ad5b Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 5 Jun 2025 23:34:28 +0300 Subject: [PATCH 01/15] Makefile: add shorthand aliasen for some targets. A dear child has many names... Signed-off-by: Krisztian Litkey --- Makefile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Makefile b/Makefile index ca125fcd8..9588cb9cc 100644 --- a/Makefile +++ b/Makefile @@ -233,6 +233,24 @@ $(BIN_PATH)/%: .static.%.$(STATIC) .PRECIOUS: $(foreach dir,$(BUILD_DIRS),.static.$(dir).1 .static.$(dir).) +# +# Eskimo words for snow... +# + +balloons: $(BIN_PATH)/nri-resource-policy-balloons +balloons-img: image.nri-resource-policy-balloons +t-a topology-aware: $(BIN_PATH)/nri-resource-policy-topology-aware +ta-img topolog-aware-img: image.nri-resource-policy-topology-aware +template: $(BIN_PATH)/nri-resource-policy-template +template-img: image.nri-resource-policy-template +memqos memory-qos: $(BIN_PATH)/nri-memory-qos +memqos-img memory-qos-img: image.nri-memory-qos +memtierd: $(BIN_PATH)/nri-memtierd +memtierd-img: image.nri-memtierd +sgx-epc sgx: $(BIN_PATH)/nri-sgx-epc +sgx-epc-img: image.nri-sgx-epc +config-manager: $(BIN_PATH)/config-manager + # # Image building test deployment generation targets # From 86e4c7a6efa415022b463c0ecd288b5829d02e22 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Mon, 9 Jun 2025 13:30:39 +0300 Subject: [PATCH 02/15] log: add AsYaml() for YAML-formatted log (blocks). AsYaml can be used to produced YAML-formatted log blocks with something like this: logger.DebugBlock(" ", "%s", log.AsYaml(my-obj)) Signed-off-by: Krisztian Litkey --- pkg/log/yaml-formatter.go | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 pkg/log/yaml-formatter.go diff --git a/pkg/log/yaml-formatter.go b/pkg/log/yaml-formatter.go new file mode 100644 index 000000000..ca6b581fb --- /dev/null +++ b/pkg/log/yaml-formatter.go @@ -0,0 +1,41 @@ +// Copyright 2019-2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "sigs.k8s.io/yaml" +) + +func AsYaml(o any) *YamlFormatter { + return &YamlFormatter{o: o} +} + +type YamlFormatter struct { + o any +} + +func (f *YamlFormatter) String() string { + if f == nil || f.o == nil { + return "" + } + + // Use a YAML marshaller to format the object. + data, err := yaml.Marshal(f.o) + if err != nil { + return "" + } + + return string(data) +} From 5dcb66dc3377f8e3300b8df74f7ec3aa85591ab5 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Tue, 16 Jul 2024 15:31:49 +0300 Subject: [PATCH 03/15] pkg/kubernetes: split out client setup code from agent. Split out K8s client setup code from agent to make it more generally available to any kind of plugin, not just resource management ones. Signed-off-by: Krisztian Litkey --- pkg/agent/agent.go | 60 ++++++------- pkg/kubernetes/client/client.go | 149 ++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 35 deletions(-) create mode 100644 pkg/kubernetes/client/client.go diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 9f63cbae9..7af43f808 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -22,6 +22,7 @@ import ( "os" "sync" + "github.com/containers/nri-plugins/pkg/kubernetes/client" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -33,7 +34,6 @@ import ( "github.com/containers/nri-plugins/pkg/agent/podresapi" "github.com/containers/nri-plugins/pkg/agent/watch" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1" - k8sclient "k8s.io/client-go/kubernetes" logger "github.com/containers/nri-plugins/pkg/log" ) @@ -127,12 +127,11 @@ type Agent struct { kubeConfig string // kubeconfig path configFile string // configuration file to use instead of custom resource - cfgIf ConfigInterface // custom resource access interface - httpCli *http.Client // shared HTTP client - k8sCli *k8sclient.Clientset // kubernetes client - nrtCli *nrtapi.Client // NRT custom resources client - nrtLock sync.Mutex // serialize NRT custom resource updates - podResCli *podresapi.Client // pod resources API client + cfgIf ConfigInterface // custom resource access interface + k8sCli *client.Client // kubernetes client + nrtCli *nrtapi.Client // NRT custom resources client + nrtLock sync.Mutex // serialize NRT custom resource updates + podResCli *podresapi.Client // pod resources API client notifyFn NotifyFn // config resource change notification callback nodeWatch watch.Interface // kubernetes node watch @@ -295,7 +294,8 @@ func (a *Agent) configure(newConfig metav1.Object) { log.Error("failed to setup NRT client: %w", err) break } - cli, err := nrtapi.NewForConfigAndClient(cfg, a.httpCli) + + cli, err := nrtapi.NewForConfigAndClient(cfg, a.k8sCli.HttpClient()) if err != nil { log.Error("failed to setup NRT client: %w", err) break @@ -331,37 +331,28 @@ func (a *Agent) hasLocalConfig() bool { } func (a *Agent) setupClients() error { + var err error + if a.hasLocalConfig() { log.Warn("running with local configuration, skipping cluster access client setup...") return nil } - // Create HTTP/REST client and K8s client on initial startup. Any failure - // to create these is a failure start up. - if a.httpCli == nil { - log.Info("setting up HTTP/REST client...") - restCfg, err := a.getRESTConfig() - if err != nil { - return err - } - - a.httpCli, err = rest.HTTPClientFor(restCfg) - if err != nil { - return fmt.Errorf("failed to setup kubernetes HTTP client: %w", err) - } + a.k8sCli, err = client.New(client.WithKubeOrInClusterConfig(a.kubeConfig)) + if err != nil { + return err + } - log.Info("setting up K8s client...") - a.k8sCli, err = k8sclient.NewForConfigAndClient(restCfg, a.httpCli) - if err != nil { - a.cleanupClients() - return fmt.Errorf("failed to setup kubernetes client: %w", err) - } + a.nrtCli, err = nrtapi.NewForConfigAndClient(a.k8sCli.RestConfig(), a.k8sCli.HttpClient()) + if err != nil { + a.cleanupClients() + return fmt.Errorf("failed to setup NRT client: %w", err) + } - kubeCfg := *restCfg - err = a.cfgIf.SetKubeClient(a.httpCli, &kubeCfg) - if err != nil { - return fmt.Errorf("failed to setup kubernetes config resource client: %w", err) - } + err = a.cfgIf.SetKubeClient(a.k8sCli.HttpClient(), a.k8sCli.RestConfig()) + if err != nil { + a.cleanupClients() + return fmt.Errorf("failed to setup kubernetes config resource client: %w", err) } a.configure(a.currentCfg) @@ -370,10 +361,9 @@ func (a *Agent) setupClients() error { } func (a *Agent) cleanupClients() { - if a.httpCli != nil { - a.httpCli.CloseIdleConnections() + if a.k8sCli != nil { + a.k8sCli.Close() } - a.httpCli = nil a.k8sCli = nil a.nrtCli = nil } diff --git a/pkg/kubernetes/client/client.go b/pkg/kubernetes/client/client.go new file mode 100644 index 000000000..0e50a575f --- /dev/null +++ b/pkg/kubernetes/client/client.go @@ -0,0 +1,149 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "net/http" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// Option is an option that can be applied to a Client. +type Option func(*Client) error + +// Client enacapsulates our Kubernetes client. +type Client struct { + cfg *rest.Config + http *http.Client + *kubernetes.Clientset +} + +// GetConfigForFile returns a REST configuration for the given file. +func GetConfigForFile(kubeConfig string) (*rest.Config, error) { + return clientcmd.BuildConfigFromFlags("", kubeConfig) +} + +// InClusterConfig returns the in-cluster REST configuration. +func InClusterConfig() (*rest.Config, error) { + return rest.InClusterConfig() +} + +// WithKubeConfig returns a Client Option for using the given kubeconfig file. +func WithKubeConfig(file string) Option { + return func(c *Client) error { + cfg, err := GetConfigForFile(file) + if err != nil { + return err + } + return WithRestConfig(cfg)(c) + } +} + +// WithInClusterConfig returns a Client Option for using the in-cluster configuration. +func WithInClusterConfig() Option { + return func(c *Client) error { + cfg, err := rest.InClusterConfig() + if err != nil { + return err + } + return WithRestConfig(cfg)(c) + } +} + +// WithKubeOrInClusterConfig returns a Client Option for using in-cluster configuration +// if a configuration file is not given. +func WithKubeOrInClusterConfig(file string) Option { + if file == "" { + return WithInClusterConfig() + } + return WithKubeConfig(file) +} + +// WithRestConfig returns a Client Option for using the given REST configuration. +func WithRestConfig(cfg *rest.Config) Option { + return func(c *Client) error { + c.cfg = cfg + return nil + } +} + +// WithHttpClient returns a Client Option for using/sharing the given HTTP client. +func WithHttpClient(hc *http.Client) Option { + return func(c *Client) error { + c.http = hc + return nil + } +} + +// New creates a new Client with the given options. +func New(options ...Option) (*Client, error) { + c := &Client{} + + for _, o := range options { + if err := o(c); err != nil { + return nil, err + } + } + + if c.cfg == nil { + if err := WithInClusterConfig()(c); err != nil { + return nil, err + } + } + + if c.http == nil { + hc, err := rest.HTTPClientFor(c.cfg) + if err != nil { + return nil, err + } + c.http = hc + } + + client, err := kubernetes.NewForConfigAndClient(c.cfg, c.http) + if err != nil { + return nil, err + } + c.Clientset = client + + return c, nil +} + +// RestConfig returns a shallow copy of the REST configuration of the Client. +func (c *Client) RestConfig() *rest.Config { + cfg := *c.cfg + return &cfg +} + +// HttpClient returns the HTTP client of the Client. +func (c *Client) HttpClient() *http.Client { + return c.http +} + +// K8sClient returns the K8s Clientset of the Client. +func (c *Client) K8sClient() *kubernetes.Clientset { + return c.Clientset +} + +// Close closes the Client. +func (c *Client) Close() { + if c.http != nil { + c.http.CloseIdleConnections() + } + c.cfg = nil + c.http = nil + c.Clientset = nil +} From 42ec10225c84084ae67761a062be9fd07e39739b Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 4 Jun 2025 09:21:57 +0300 Subject: [PATCH 04/15] pkg/kubernetes: allow setting accepted and wire content types. Allow setting the content types a client accepts and the type it uses on the wire. Provide constants for JSON and protobuf content types. Signed-off-by: Krisztian Litkey --- pkg/kubernetes/client/client.go | 49 +++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/pkg/kubernetes/client/client.go b/pkg/kubernetes/client/client.go index 0e50a575f..c60243514 100644 --- a/pkg/kubernetes/client/client.go +++ b/pkg/kubernetes/client/client.go @@ -15,7 +15,9 @@ package client import ( + "errors" "net/http" + "strings" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -76,7 +78,7 @@ func WithKubeOrInClusterConfig(file string) Option { // WithRestConfig returns a Client Option for using the given REST configuration. func WithRestConfig(cfg *rest.Config) Option { return func(c *Client) error { - c.cfg = cfg + c.cfg = rest.CopyConfig(cfg) return nil } } @@ -89,13 +91,50 @@ func WithHttpClient(hc *http.Client) Option { } } +// WithAcceptContentTypes returns a Client Option for setting the accepted content types. +func WithAcceptContentTypes(contentTypes ...string) Option { + return func(c *Client) error { + if c.cfg == nil { + return errRetryWhenConfigSet + } + c.cfg.AcceptContentTypes = strings.Join(contentTypes, ",") + return nil + } +} + +// WithContentType returns a Client Option for setting the wire format content type. +func WithContentType(contentType string) Option { + return func(c *Client) error { + if c.cfg == nil { + return errRetryWhenConfigSet + } + c.cfg.ContentType = contentType + return nil + } +} + +const ( + ContentTypeJSON = "application/json" + ContentTypeProtobuf = "application/vnd.kubernetes.protobuf" +) + +var ( + // returned by options if applied too early, before a configuration is set + errRetryWhenConfigSet = errors.New("retry when client config is set") +) + // New creates a new Client with the given options. func New(options ...Option) (*Client, error) { c := &Client{} + var retry []Option for _, o := range options { if err := o(c); err != nil { - return nil, err + if err == errRetryWhenConfigSet { + retry = append(retry, o) + } else { + return nil, err + } } } @@ -105,6 +144,12 @@ func New(options ...Option) (*Client, error) { } } + for _, o := range retry { + if err := o(c); err != nil { + return nil, err + } + } + if c.http == nil { hc, err := rest.HTTPClientFor(c.cfg) if err != nil { From 45a14d1ce2078f382cdbd95a784b2405fba028b9 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 17 Jul 2024 19:36:53 +0300 Subject: [PATCH 05/15] pkg/kubernetes: split out watch wrapper from agent. Split out K8s watch wrapper setup code from agent to make it generally available to any kind of plugin, not just resource management ones. Signed-off-by: Krisztian Litkey --- pkg/kubernetes/watch/file.go | 180 +++++++++++++++++++++++++++++++++ pkg/kubernetes/watch/object.go | 160 +++++++++++++++++++++++++++++ pkg/kubernetes/watch/watch.go | 38 +++++++ 3 files changed, 378 insertions(+) create mode 100644 pkg/kubernetes/watch/file.go create mode 100644 pkg/kubernetes/watch/object.go create mode 100644 pkg/kubernetes/watch/watch.go diff --git a/pkg/kubernetes/watch/file.go b/pkg/kubernetes/watch/file.go new file mode 100644 index 000000000..cdc2417e2 --- /dev/null +++ b/pkg/kubernetes/watch/file.go @@ -0,0 +1,180 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watch + +import ( + "errors" + "io/fs" + "os" + "path" + "path/filepath" + "sync" + + "github.com/fsnotify/fsnotify" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8swatch "k8s.io/apimachinery/pkg/watch" +) + +// FileClient takes care of unmarshaling file content to a target object. +type FileClient interface { + Unmarshal([]byte, string) (runtime.Object, error) +} + +type fileWatch struct { + dir string + file string + client FileClient + fsw *fsnotify.Watcher + resultC chan Event + stopOnce sync.Once + stopC chan struct{} + doneC chan struct{} +} + +// File creates a k8s-compatible watch for the given file. Contents of the +// file are unmarshalled into the object of choice by the client. The file +// is monitored for changes and corresponding watch events are generated. +func File(client FileClient, file string) (Interface, error) { + absPath, err := filepath.Abs(file) + if err != nil { + return nil, err + } + + fsw, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + + if err = fsw.Add(filepath.Dir(absPath)); err != nil { + return nil, err + } + + fw := &fileWatch{ + dir: filepath.Dir(absPath), + file: filepath.Base(absPath), + client: client, + fsw: fsw, + resultC: make(chan Event, k8swatch.DefaultChanSize), + stopC: make(chan struct{}), + doneC: make(chan struct{}), + } + + obj, err := fw.readObject() + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + + fw.sendEvent(Added, obj) + + go fw.run() + + return fw, nil +} + +// Stop stops the watch. +func (w *fileWatch) Stop() { + w.stopOnce.Do(func() { + close(w.stopC) + <-w.doneC + w.stopC = nil + }) +} + +// ResultChan returns the channel for receiving events from the watch. +func (w *fileWatch) ResultChan() <-chan Event { + return w.resultC +} + +func (w *fileWatch) run() { + for { + select { + case <-w.stopC: + if err := w.fsw.Close(); err != nil { + log.Warn("%s failed to close fsnotify watcher: %v", w.name(), err) + } + close(w.resultC) + close(w.doneC) + return + + case e, ok := <-w.fsw.Events: + log.Debug("%s got event %+v", w.name(), e) + + if !ok { + w.sendEvent( + Error, + &metav1.Status{ + Status: metav1.StatusFailure, + Message: "failed to receive fsnotify event", + }, + ) + close(w.resultC) + close(w.doneC) + return + } + + if path.Base(e.Name) != w.file { + continue + } + + switch { + case (e.Op & (fsnotify.Create | fsnotify.Write)) != 0: + obj, err := w.readObject() + if err != nil { + log.Debug("%s failed to read/unmarshal: %v", w.name(), err) + continue + } + + if (e.Op & fsnotify.Create) != 0 { + w.sendEvent(Added, obj) + } else { + w.sendEvent(Added, obj) + } + + case (e.Op & (fsnotify.Remove | fsnotify.Rename)) != 0: + w.sendEvent(Deleted, nil) + } + } + } +} + +func (w *fileWatch) sendEvent(t EventType, obj runtime.Object) { + select { + case w.resultC <- Event{Type: t, Object: obj}: + default: + log.Warn("failed to deliver fileWatch %v event", t) + } +} + +func (w *fileWatch) readObject() (runtime.Object, error) { + file := path.Join(w.dir, w.file) + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + + obj, err := w.client.Unmarshal(data, file) + if err != nil { + return nil, err + } + + log.Debug("%s read object %+v", w.name(), obj) + + return obj, nil +} + +func (w *fileWatch) name() string { + return path.Join("filewatch/path:", path.Join(w.dir, w.file)) +} diff --git a/pkg/kubernetes/watch/object.go b/pkg/kubernetes/watch/object.go new file mode 100644 index 000000000..eefe3fed4 --- /dev/null +++ b/pkg/kubernetes/watch/object.go @@ -0,0 +1,160 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watch + +import ( + "path" + "sync" + "time" + + k8swatch "k8s.io/apimachinery/pkg/watch" +) + +const ( + createDelay = 1 * time.Second +) + +type watch struct { + sync.Mutex + subject string + client ObjectClient + w Interface + resultC chan Event + createC <-chan time.Time + + stopOnce sync.Once + stopC chan struct{} + doneC chan struct{} +} + +// ObjectClient takes care of low-level details of creating a wrapped watch. +type ObjectClient interface { + CreateWatch() (Interface, error) +} + +// Object creates a wrapped watch using the given client. The watch +// is transparently recreated upon expiration and any errors. +func Object(client ObjectClient, subject string) (Interface, error) { + w := &watch{ + subject: subject, + client: client, + resultC: make(chan Event, k8swatch.DefaultChanSize), + stopC: make(chan struct{}), + doneC: make(chan struct{}), + } + + if err := w.create(); err != nil { + return nil, err + } + + go w.run() + + return w, nil +} + +// Stop stops the watch. +func (w *watch) Stop() { + w.stopOnce.Do(func() { + close(w.stopC) + <-w.doneC + w.stop() + w.stopC = nil + }) +} + +// ResultChan returns the channel for receiving events from the watch. +func (w *watch) ResultChan() <-chan Event { + return w.resultC +} + +func (w *watch) eventChan() <-chan Event { + if w.w == nil { + return nil + } + + return w.w.ResultChan() +} + +func (w *watch) run() { + for { + select { + case <-w.stopC: + close(w.resultC) + close(w.doneC) + return + + case e, ok := <-w.eventChan(): + if !ok { + log.Debug("%s failed...", w.name()) + w.stop() + w.scheduleCreate() + continue + } + + if e.Type == Error { + log.Debug("%s failed with an error...", w.name()) + w.stop() + w.scheduleCreate() + continue + } + + select { + case w.resultC <- e: + default: + log.Debug("%s failed to deliver %v event", w.name(), e.Type) + w.stop() + w.scheduleCreate() + } + + case <-w.createC: + w.createC = nil + log.Debug("reopening %s...", w.name()) + if err := w.create(); err != nil { + w.scheduleCreate() + } + } + } +} + +func (w *watch) create() error { + w.stop() + + wif, err := w.client.CreateWatch() + if err != nil { + return err + } + w.w = wif + + return nil +} + +func (w *watch) scheduleCreate() { + if w.createC == nil { + w.createC = time.After(createDelay) + } +} + +func (w *watch) stop() { + if w.w == nil { + return + } + + w.w.Stop() + w.w = nil +} + +func (w *watch) name() string { + return path.Join("watch", w.subject) +} diff --git a/pkg/kubernetes/watch/watch.go b/pkg/kubernetes/watch/watch.go new file mode 100644 index 000000000..095215211 --- /dev/null +++ b/pkg/kubernetes/watch/watch.go @@ -0,0 +1,38 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watch + +import ( + logger "github.com/containers/nri-plugins/pkg/log" + k8swatch "k8s.io/apimachinery/pkg/watch" +) + +type ( + Interface = k8swatch.Interface + EventType = k8swatch.EventType + Event = k8swatch.Event +) + +const ( + Added = k8swatch.Added + Modified = k8swatch.Modified + Deleted = k8swatch.Deleted + Bookmark = k8swatch.Bookmark + Error = k8swatch.Error +) + +var ( + log = logger.Get("watch") +) From 88140644c4bfced2beda58349886ec4d19986ef1 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 4 Jun 2025 09:20:32 +0300 Subject: [PATCH 06/15] agent: expose node name, kube client and config. Signed-off-by: Krisztian Litkey --- pkg/agent/agent.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 7af43f808..f97766f94 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -263,6 +263,18 @@ func (a *Agent) Stop() { } } +func (a *Agent) NodeName() string { + return a.nodeName +} + +func (a *Agent) KubeClient() *client.Client { + return a.k8sCli +} + +func (a *Agent) KubeConfig() string { + return a.kubeConfig +} + var ( defaultConfig = &cfgapi.AgentConfig{ NodeResourceTopology: true, From b631563f3d176f3dcf346e70667953e6575b6b38 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Mon, 9 Jun 2025 22:02:54 +0300 Subject: [PATCH 07/15] helm: add RBAC rules, mounts, and device class for DRA driver. Add the necessary RBAC rules (access resource slices and claims) and kubelet host mounts (plugin and plugin registry directories) to the topology-aware policy Helm chart. Add a device class for native.cpu DRA driver. Signed-off-by: Krisztian Litkey --- .../topology-aware/templates/clusterrole.yaml | 24 +++++++++++++++++++ .../topology-aware/templates/daemonset.yaml | 18 ++++++++++++++ .../templates/native-cpu-device-class.yaml | 8 +++++++ 3 files changed, 50 insertions(+) create mode 100644 deployment/helm/topology-aware/templates/native-cpu-device-class.yaml diff --git a/deployment/helm/topology-aware/templates/clusterrole.yaml b/deployment/helm/topology-aware/templates/clusterrole.yaml index d5b76ffbb..2a58dd7e7 100644 --- a/deployment/helm/topology-aware/templates/clusterrole.yaml +++ b/deployment/helm/topology-aware/templates/clusterrole.yaml @@ -22,3 +22,27 @@ rules: - list - update - delete +- apiGroups: + - "resource.k8s.io" + resources: + - resourceslices + verbs: + - list + - watch + - create + - update + - delete +- apiGroups: + - "resource.k8s.io" + resources: + - resourceclaims + - deviceclasses + verbs: + - get +- apiGroups: + - "resource.k8s.io" + resources: + - resourceclaims/status + verbs: + - patch + - update diff --git a/deployment/helm/topology-aware/templates/daemonset.yaml b/deployment/helm/topology-aware/templates/daemonset.yaml index 86a7fec50..92a58545e 100644 --- a/deployment/helm/topology-aware/templates/daemonset.yaml +++ b/deployment/helm/topology-aware/templates/daemonset.yaml @@ -124,6 +124,12 @@ spec: - name: pod-resources-socket mountPath: /var/lib/kubelet/pod-resources readOnly: true + - name: host-cdi-dir + mountPath: /host/var/run/cdi + - name: kubelet-plugins + mountPath: /var/lib/kubelet/plugins + - name: kubelet-plugins-registry + mountPath: /var/lib/kubelet/plugins_registry {{- if .Values.podPriorityClassNodeCritical }} priorityClassName: system-node-critical {{- end }} @@ -147,6 +153,18 @@ spec: hostPath: path: /var/lib/kubelet/pod-resources type: DirectoryOrCreate + - name: host-cdi-dir + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate + - name: kubelet-plugins + hostPath: + path: /var/lib/kubelet/plugins + type: DirectoryOrCreate + - name: kubelet-plugins-registry + hostPath: + path: /var/lib/kubelet/plugins_registry + type: DirectoryOrCreate {{- if .Values.nri.runtime.patchConfig }} - name: containerd-config hostPath: diff --git a/deployment/helm/topology-aware/templates/native-cpu-device-class.yaml b/deployment/helm/topology-aware/templates/native-cpu-device-class.yaml new file mode 100644 index 000000000..2f4c7ccdb --- /dev/null +++ b/deployment/helm/topology-aware/templates/native-cpu-device-class.yaml @@ -0,0 +1,8 @@ +apiVersion: resource.k8s.io/v1 +kind: DeviceClass +metadata: + name: native.cpu +spec: + selectors: + - cel: + expression: device.driver == "native.cpu" From b0efadc349264990397cdc8222c0a74e7e3f3f92 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 6 Jun 2025 19:17:15 +0300 Subject: [PATCH 08/15] cache: add opaque cache entries, container.GetEnvList(). Add an interface for caching policy agnostic data, similar to but simpler than {Get,Set}PolicyData(). Add an interface for querying the full container environment variable list. Signed-off-by: Krisztian Litkey --- .../topology-aware/policy/mocks_test.go | 9 ++++ pkg/resmgr/cache/cache.go | 48 +++++++++++++++++++ pkg/resmgr/cache/container.go | 4 ++ 3 files changed, 61 insertions(+) diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index 9477f94b0..2f0d2ee74 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -408,6 +408,9 @@ func (m *mockContainer) GetAnnotation(string, interface{}) (string, bool) { func (m *mockContainer) GetEnv(string) (string, bool) { panic("unimplemented") } +func (m *mockContainer) GetEnvList() []string { + panic("unimplemented") +} func (m *mockContainer) GetAnnotations() map[string]string { panic("unimplemented") } @@ -752,6 +755,12 @@ func (m *mockCache) SetPolicyEntry(string, interface{}) { func (m *mockCache) GetPolicyEntry(string, interface{}) bool { return m.returnValueForGetPolicyEntry } +func (m *mockCache) SetEntry(string, interface{}) { + panic("unimplemented") +} +func (m *mockCache) GetEntry(string, interface{}) (interface{}, error) { + panic("unimplemented") +} func (m *mockCache) Save() error { return nil } diff --git a/pkg/resmgr/cache/cache.go b/pkg/resmgr/cache/cache.go index fef120d12..da265b9fd 100644 --- a/pkg/resmgr/cache/cache.go +++ b/pkg/resmgr/cache/cache.go @@ -204,6 +204,8 @@ type Container interface { GetAnnotation(key string, objPtr interface{}) (string, bool) // GetEnv returns the value of a container environment variable. GetEnv(string) (string, bool) + // GetEnvList return the list of environment variables for the container. + GetEnvList() []string // GetMounts returns all the mounts of the container. GetMounts() []*Mount // GetDevices returns all the linux devices of the container. @@ -448,6 +450,11 @@ type Cache interface { // GetPolicyEntry gets the policy entry for a key. GetPolicyEntry(string, interface{}) bool + // SetEntry sets the entry for a key. + SetEntry(string, interface{}) + // GetEntry gets the entry for a key. + GetEntry(string, interface{}) (interface{}, error) + // Save requests a cache save. Save() error @@ -500,6 +507,8 @@ type cache struct { PolicyName string // name of the active policy policyData map[string]interface{} // opaque policy data PolicyJSON map[string]string // ditto in raw, unmarshaled form + entryData map[string]interface{} // opaque cached data + entryJSON map[string]string // ditto in raw, unmarshaled form pending map[string]struct{} // cache IDs of containers with pending changes @@ -525,6 +534,8 @@ func NewCache(options Options) (Cache, error) { NextID: 1, policyData: make(map[string]interface{}), PolicyJSON: make(map[string]string), + entryData: make(map[string]interface{}), + entryJSON: make(map[string]string), implicit: make(map[string]ImplicitAffinity), } @@ -829,6 +840,30 @@ func (cch *cache) GetContainers() []Container { return containers } +// Set the entry for a key. +func (cch *cache) SetEntry(key string, obj interface{}) { + cch.entryData[key] = obj +} + +var ErrNoEntry = errors.New("no cached entry found") + +// Get the entry for a key. +func (cch *cache) GetEntry(key string, ptr interface{}) (interface{}, error) { + if obj, ok := cch.entryData[key]; ok { + return obj, nil + } + if data, ok := cch.entryJSON[key]; ok { + err := json.Unmarshal([]byte(data), ptr) + if err != nil { + return nil, cacheError("failed to unmarshal entry for key '%s': %w", key, err) + } + cch.entryData[key] = ptr + return ptr, nil + } + + return nil, ErrNoEntry +} + // Set the policy entry for a key. func (cch *cache) SetPolicyEntry(key string, obj interface{}) { cch.policyData[key] = obj @@ -1092,6 +1127,7 @@ type snapshot struct { Pods map[string]*pod Containers map[string]*container NextID uint64 + Entries map[string]string PolicyName string PolicyJSON map[string]string } @@ -1105,6 +1141,7 @@ func (cch *cache) Snapshot() ([]byte, error) { NextID: cch.NextID, PolicyName: cch.PolicyName, PolicyJSON: cch.PolicyJSON, + Entries: make(map[string]string), } for id, p := range cch.Pods { @@ -1124,6 +1161,14 @@ func (cch *cache) Snapshot() ([]byte, error) { s.PolicyJSON[key] = string(data) } + for key, obj := range cch.entryData { + data, err := json.Marshal(obj) + if err != nil { + return nil, cacheError("failed to marshal entry '%s': %v", key, err) + } + s.Entries[key] = string(data) + } + data, err := json.Marshal(s) if err != nil { return nil, cacheError("failed to marshal cache: %v", err) @@ -1138,6 +1183,7 @@ func (cch *cache) Restore(data []byte) error { Pods: make(map[string]*pod), Containers: make(map[string]*container), PolicyJSON: make(map[string]string), + Entries: make(map[string]string), } if err := json.Unmarshal(data, &s); err != nil { @@ -1155,6 +1201,8 @@ func (cch *cache) Restore(data []byte) error { cch.PolicyJSON = s.PolicyJSON cch.PolicyName = s.PolicyName cch.policyData = make(map[string]interface{}) + cch.entryData = make(map[string]interface{}) + cch.entryJSON = s.Entries for _, p := range cch.Pods { p.cache = cch diff --git a/pkg/resmgr/cache/container.go b/pkg/resmgr/cache/container.go index f9ec72118..3aec8e62f 100644 --- a/pkg/resmgr/cache/container.go +++ b/pkg/resmgr/cache/container.go @@ -455,6 +455,10 @@ func (c *container) GetEnv(key string) (string, bool) { return "", false } +func (c *container) GetEnvList() []string { + return slices.Clone(c.Ctr.GetEnv()) +} + func (c *container) GetMounts() []*Mount { var mounts []*Mount From 9497e4d3a79703b5fa403d74d139b5ea12a5e430 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Tue, 10 Jun 2025 00:56:42 +0300 Subject: [PATCH 09/15] sysfs: enumerate caches, add CPU DRA device abstraction. Sort caches by level, kind, and id to enumerate caches globally. Add a common DRA device representation for CPUs. Signed-off-by: Krisztian Litkey --- .../topology-aware/policy/mocks_test.go | 8 + go.mod | 77 +++---- go.sum | 189 ++++++++++-------- pkg/sysfs/dra.go | 119 +++++++++++ pkg/sysfs/system.go | 82 +++++++- 5 files changed, 353 insertions(+), 122 deletions(-) create mode 100644 pkg/sysfs/dra.go diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index 2f0d2ee74..d4dc582ab 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -30,6 +30,7 @@ import ( "github.com/intel/goresctrl/pkg/sst" idset "github.com/intel/goresctrl/pkg/utils" v1 "k8s.io/api/core/v1" + resapi "k8s.io/api/resource/v1" ) type mockSystemNode struct { @@ -207,6 +208,10 @@ func (c *mockCPU) CoreKind() sysfs.CoreKind { return sysfs.PerformanceCore } +func (c *mockCPU) DRA(extras ...map[sysfs.QualifiedName]sysfs.Attribute) *resapi.Device { + panic("unimplmented") +} + type mockSystem struct { isolatedCPU int nodes []sysfs.Node @@ -348,6 +353,9 @@ func (fake *mockSystem) NodeDistance(idset.ID, idset.ID) int { func (fake *mockSystem) NodeHintToCPUs(string) string { return "" } +func (fake *mockSystem) CPUsAsDRADevices(ids []idset.ID) []resapi.Device { + panic("unimplemented") +} type mockContainer struct { name string diff --git a/go.mod b/go.mod index e5fe6321a..1ad92532d 100644 --- a/go.mod +++ b/go.mod @@ -13,44 +13,46 @@ require ( github.com/intel/goresctrl v0.11.0 github.com/intel/memtierd v0.1.1 github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.1.2 - github.com/onsi/ginkgo/v2 v2.21.0 - github.com/onsi/gomega v1.35.1 + github.com/onsi/ginkgo/v2 v2.27.2 + github.com/onsi/gomega v1.38.2 github.com/pelletier/go-toml/v2 v2.1.0 - github.com/prometheus/client_golang v1.23.0 + github.com/prometheus/client_golang v1.23.2 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.38.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 go.opentelemetry.io/otel/exporters/prometheus v0.60.0 go.opentelemetry.io/otel/metric v1.38.0 go.opentelemetry.io/otel/sdk v1.38.0 go.opentelemetry.io/otel/sdk/metric v1.38.0 go.opentelemetry.io/otel/trace v1.38.0 - golang.org/x/sys v0.35.0 + golang.org/x/sys v0.38.0 golang.org/x/time v0.9.0 google.golang.org/grpc v1.75.0 - k8s.io/api v0.31.2 - k8s.io/apimachinery v0.33.1 - k8s.io/client-go v0.31.2 + k8s.io/api v0.35.0 + k8s.io/apimachinery v0.35.0 + k8s.io/client-go v0.35.0 + k8s.io/dynamic-resource-allocation v0.35.0 k8s.io/klog/v2 v2.130.1 - k8s.io/kubelet v0.31.2 - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 + k8s.io/kubelet v0.35.0 + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 sigs.k8s.io/controller-runtime v0.16.2 - sigs.k8s.io/yaml v1.4.0 + sigs.k8s.io/yaml v1.6.0 ) require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -59,54 +61,55 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect - github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/knqyf263/go-plugin v0.9.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.3.0 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/otlptranslator v0.0.2 // indirect github.com/prometheus/procfs v0.17.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.9 // indirect github.com/tetratelabs/wazero v1.10.1 // indirect github.com/x448/float16 v0.8.4 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.5 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect go.opentelemetry.io/proto/otlp v1.7.1 // indirect - golang.org/x/mod v0.26.0 // indirect - golang.org/x/net v0.43.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/tools v0.35.0 // indirect - golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/tools v0.38.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/protobuf v1.36.8 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/code-generator v0.33.1 // indirect - k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + k8s.io/code-generator v0.35.0 // indirect + k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index 8f3a66f9d..810a07d87 100644 --- a/go.sum +++ b/go.sum @@ -598,6 +598,8 @@ git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3p github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= @@ -614,8 +616,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -661,8 +663,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -682,9 +684,15 @@ github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/ github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= @@ -714,6 +722,8 @@ github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhO github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -759,8 +769,8 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= -github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -779,8 +789,6 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -801,8 +809,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -838,14 +846,14 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/intel/goresctrl v0.11.0 h1:FnlyrQxgjp9nm2hccwvBDc7qmf9uLxKf0DC6CKobnMA= github.com/intel/goresctrl v0.11.0/go.mod h1:F6jINn5WKuXXHAjmE7eTBjeHPmqT2kiWStrggYIm+UM= github.com/intel/memtierd v0.1.1 h1:hGSN0+dzjaUkwgkJrk6B9SU4dntggXLpXgs9Dm+jfz4= github.com/intel/memtierd v0.1.1/go.mod h1:NFDBvjoDS42gBK/c9q/CYCJ2pt/+g7UQwOOBvQli4z0= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -881,22 +889,27 @@ github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuz github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= @@ -907,22 +920,21 @@ github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= -github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= -github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ= github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= @@ -933,8 +945,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -944,11 +956,13 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -964,6 +978,14 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8= github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -975,6 +997,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.etcd.io/etcd/client/pkg/v3 v3.6.5 h1:Duz9fAzIZFhYWgRjp/FgNq2gO1jId9Yae/rLn3RrBP8= +go.etcd.io/etcd/client/pkg/v3 v3.6.5/go.mod h1:8Wx3eGRPiy0qOFMZT/hfvdos+DjEaPxdIDiCDUv/FQk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -992,12 +1016,12 @@ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 h1:wpMfgF8E1rkrT1Z6meFh1NDtownE9Ii3n3X2GJYjsaU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo= go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= @@ -1018,6 +1042,14 @@ go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOV go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1042,8 +1074,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1086,8 +1118,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1145,8 +1177,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1194,8 +1226,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1279,8 +1311,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -1289,8 +1321,8 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1307,8 +1339,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1380,10 +1412,10 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= -golang.org/x/tools/go/expect v0.1.0-deprecated h1:jY2C5HGYR5lqex3gEniOQL0r7Dq5+VGVgY1nudX5lXY= -golang.org/x/tools/go/expect v0.1.0-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1677,14 +1709,12 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1696,24 +1726,26 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= -k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= -k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= -k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= -k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc= -k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs= -k8s.io/code-generator v0.33.1 h1:ZLzIRdMsh3Myfnx9BaooX6iQry29UJjVfVG+BuS+UMw= -k8s.io/code-generator v0.33.1/go.mod h1:HUKT7Ubp6bOgIbbaPIs9lpd2Q02uqkMCMx9/GjDrWpY= -k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7 h1:2OX19X59HxDprNCVrWi6jb7LW1PoqTlYqEq5H2oetog= -k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= +k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= +k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o= +k8s.io/code-generator v0.35.0 h1:TvrtfKYZTm9oDF2z+veFKSCcgZE3Igv0svY+ehCmjHQ= +k8s.io/code-generator v0.35.0/go.mod h1:iS1gvVf3c/T71N5DOGYO+Gt3PdJ6B9LYSvIyQ4FHzgc= +k8s.io/dynamic-resource-allocation v0.35.0 h1:St6dsCCylLg3HiFPcyHzFF8YQO6yziUDaVRLGdkrNH8= +k8s.io/dynamic-resource-allocation v0.35.0/go.mod h1:uaFga3VJtwyfpfZwpuJG7mlurWGQaaiGUa+QZmooz2U= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= -k8s.io/kubelet v0.31.2 h1:6Hytyw4LqWqhgzoi7sPfpDGClu2UfxmPmaiXPC4FRgI= -k8s.io/kubelet v0.31.2/go.mod h1:0E4++3cMWi2cJxOwuaQP3eMBa7PSOvAFgkTPlVc/2FA= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/kubelet v0.35.0 h1:8cgJHCBCKLYuuQ7/Pxb/qWbJfX1LXIw7790ce9xHq7c= +k8s.io/kubelet v0.35.0/go.mod h1:ciRzAXn7C4z5iB7FhG1L2CGPPXLTVCABDlbXt/Zz8YA= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= @@ -1754,12 +1786,11 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/pkg/sysfs/dra.go b/pkg/sysfs/dra.go new file mode 100644 index 000000000..a6300809e --- /dev/null +++ b/pkg/sysfs/dra.go @@ -0,0 +1,119 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysfs + +import ( + "fmt" + "strconv" + + idset "github.com/intel/goresctrl/pkg/utils" + resapi "k8s.io/api/resource/v1" +) + +type ( + ID = idset.ID + QualifiedName = resapi.QualifiedName + Attribute = resapi.DeviceAttribute +) + +const ( + // Names for standard CPU device attributes. + AttrPackage = QualifiedName("package") + AttrDie = QualifiedName("die") + AttrCluster = QualifiedName("cluster") + AttrCore = QualifiedName("core") + AttrCoreType = QualifiedName("coreType") + AttrLocalMemory = QualifiedName("localMemory") + AttrIsolated = QualifiedName("isolated") + AttrMinFreq = QualifiedName("minFreq") + AttrMaxFreq = QualifiedName("maxFreq") + AttrBaseFreq = QualifiedName("baseFreq") +) + +// CPUsAsDRADevices returns the given CPUs as DRA devices. +func (sys *system) CPUsAsDRADevices(ids []ID) []resapi.Device { + devices := make([]resapi.Device, 0, len(ids)) + for _, id := range ids { + devices = append(devices, *(sys.CPU(id).DRA())) + } + return devices +} + +// DRA returns the CPU represented as a DRA device. +func (c *cpu) DRA(extras ...map[QualifiedName]Attribute) *resapi.Device { + dra := &resapi.Device{ + Name: "cpu" + strconv.Itoa(c.ID()), + Attributes: map[QualifiedName]Attribute{ + AttrPackage: Attr(c.PackageID()), + AttrDie: Attr(c.DieID()), + AttrCluster: Attr(c.ClusterID()), + AttrCore: Attr(c.CoreID()), + AttrCoreType: Attr(c.CoreKind().String()), + AttrLocalMemory: Attr(c.NodeID()), + AttrIsolated: Attr(c.Isolated()), + }, + } + + if base := c.FrequencyRange().Base; base > 0 { + dra.Attributes[AttrBaseFreq] = Attr(base) + } + if min := c.FrequencyRange().Min; min > 0 { + dra.Attributes[AttrMinFreq] = Attr(min) + } + if max := c.FrequencyRange().Max; max > 0 { + dra.Attributes[AttrMaxFreq] = Attr(max) + } + + for idx, cache := range c.GetCaches() { + dra.Attributes[QualifiedName(fmt.Sprintf("cache%dID", idx))] = Attr(cache.EnumID()) + } + + for _, m := range extras { + for name, value := range m { + if _, ok := dra.Attributes[name]; !ok { + dra.Attributes[name] = value + } + } + } + + return dra +} + +// Attr returns an attribute for the given value. +func Attr(value any) Attribute { + switch v := any(value).(type) { + case int64: + return Attribute{IntValue: &v} + case int: + val := int64(v) + return Attribute{IntValue: &val} + case uint64: + val := int64(v) + return Attribute{IntValue: &val} + case int32: + val := int64(v) + return Attribute{IntValue: &val} + case uint32: + val := int64(v) + return Attribute{IntValue: &val} + case string: + return Attribute{StringValue: &v} + case bool: + return Attribute{BoolValue: &v} + default: + val := fmt.Sprintf("", value) + return Attribute{StringValue: &val} + } +} diff --git a/pkg/sysfs/system.go b/pkg/sysfs/system.go index b013d9f4e..de78819dd 100644 --- a/pkg/sysfs/system.go +++ b/pkg/sysfs/system.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/containers/nri-plugins/pkg/utils/cpuset" + resapi "k8s.io/api/resource/v1" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/utils" @@ -134,6 +135,8 @@ type System interface { Isolated() cpuset.CPUSet NodeHintToCPUs(string) string + + CPUsAsDRADevices(ids []idset.ID) []resapi.Device } // System devices @@ -230,6 +233,7 @@ type CPU interface { GetLastLevelCaches() []*Cache GetLastLevelCacheCPUSet() cpuset.CPUSet CoreKind() CoreKind + DRA(extras ...map[QualifiedName]Attribute) *resapi.Device } type cpu struct { @@ -313,11 +317,12 @@ const ( // Cache has details about a CPU cache. type Cache struct { - id idset.ID // cache id - level int // cache type - kind CacheType // cache type - size uint64 // cache size - cpus idset.IDSet // CPUs sharing this cache + enumID int // enumerated cache id + id idset.ID // cache id + level int // cache type + kind CacheType // cache type + size uint64 // cache size + cpus idset.IDSet // CPUs sharing this cache } // cacheOverrides is a list of cache overrides for specific CPU sets. @@ -392,6 +397,11 @@ func (sys *system) Discover(flags DiscoveryFlag) error { if err := sys.discoverCPUs(); err != nil { return err } + if (flags & DiscoverCache) != 0 { + if err := sys.enumerateCaches(); err != nil { + return fmt.Errorf("failed to enumerate caches: %v", err) + } + } if err := sys.discoverNodes(); err != nil { return err } @@ -489,9 +499,10 @@ func (sys *system) Discover(flags DiscoveryFlag) error { sys.Debug(" base freq: %d", cpu.freq.Base) sys.Debug(" freq: %d - %d", cpu.freq.Min, cpu.freq.Max) sys.Debug(" epp: %d", cpu.epp) + sys.DebugBlock(" ", "%s", logger.AsYaml(cpu.DRA())) for idx, c := range cpu.caches { - sys.Debug(" cache #%d:", idx) + sys.Debug(" cache #%d (enumerated as #%d):", idx, c.enumID) sys.Debug(" id: %d", c.id) sys.Debug(" level: %d", c.level) sys.Debug(" kind: %s", c.kind) @@ -499,6 +510,9 @@ func (sys *system) Discover(flags DiscoveryFlag) error { sys.Debug(" cpus: %s", c.SharedCPUSet().String()) } } + + sys.DebugBlock(" ", "%s", + logger.AsYaml(sys.CPUsAsDRADevices(sys.CPUIDs()))) } return nil @@ -1308,6 +1322,13 @@ func (c *cpu) CoreKind() CoreKind { return c.coreKind } +func (c *Cache) EnumID() int { + if c == nil { + return 0 + } + return c.enumID +} + func (c *Cache) ID() int { if c == nil { return 0 @@ -2084,6 +2105,55 @@ func (sys *system) saveCache(c *Cache) *Cache { return c } +func (sys *system) enumerateCaches() error { + log.Debug("enumerating CPU caches...") + cMap := map[int]map[CacheType]map[idset.ID]*Cache{} + enumerated := []*Cache{} + for _, caches := range sys.caches { + for _, cacheMap := range caches { + for _, c := range cacheMap { + l, ok := cMap[c.level] + if !ok { + l = map[CacheType]map[idset.ID]*Cache{} + cMap[c.level] = l + } + k, ok := l[c.kind] + if !ok { + k = map[idset.ID]*Cache{} + l[c.kind] = k + } + _, ok = k[c.id] + if !ok { + k[c.id] = c + enumerated = append(enumerated, c) + } + } + } + } + + slices.SortFunc(enumerated, func(ci, cj *Cache) int { + if ci.level != cj.level { + return ci.level - cj.level + } + if ci.kind != cj.kind { + return int(ci.kind) - int(cj.kind) + } + return ci.id - cj.id + }) + + for id, c := range enumerated { + c.enumID = id + sys.Debug(" enumerated cache #%d:", c.enumID) + sys.Debug(" id: %d", c.id) + sys.Debug(" level: %d", c.level) + sys.Debug(" kind: %s", c.kind) + sys.Debug(" size: %dK", c.size/1024) + sys.Debug(" cpus: %s", c.SharedCPUSet().String()) + } + + return nil +} + // eppStrings initialized this way to better catch changes in the enum var eppStrings = func() [EPPUnknown]string { var e [EPPUnknown]string From eec13711e66bab2c4370aa8239c81d04bed56c1e Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 5 Jun 2025 23:32:02 +0300 Subject: [PATCH 10/15] policy: add getter for sysfs.System being used. Signed-off-by: Krisztian Litkey --- pkg/resmgr/policy/policy.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/resmgr/policy/policy.go b/pkg/resmgr/policy/policy.go index 760fb316e..3967693e3 100644 --- a/pkg/resmgr/policy/policy.go +++ b/pkg/resmgr/policy/policy.go @@ -123,6 +123,8 @@ type Backend interface { type Policy interface { // ActivePolicy returns the name of the policy backend in use. ActivePolicy() string + // System returns the sysfs instance used by the policy. + System() system.System // Start starts up policy, prepare for serving resource management requests. Start(interface{}) error // Reconfigure the policy. @@ -244,6 +246,10 @@ func (p *policy) ActivePolicy() string { return "" } +func (p *policy) System() system.System { + return p.system +} + // Start starts up policy, preparing it for serving requests. func (p *policy) Start(cfg interface{}) error { log.Info("activating '%s' policy...", p.active.Name()) From 60ba3fd6a92f71ef678e2b4083aae42f25a53dfe Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 5 Jun 2025 23:30:41 +0300 Subject: [PATCH 11/15] policy: add plumbing for policies to act as CPU DRA drivers. Add the necessary plumbing to allow implementations act as a CPU DRA driver. In partice, add an option for publishing CPUs as DRA devices and interfaces to allocate and release CPUs as claims are prepared and unprepared. Signed-off-by: Krisztian Litkey --- .../balloons/policy/balloons-policy.go | 12 +++++ .../template/policy/template-policy.go | 12 +++++ .../policy/topology-aware-policy.go | 12 +++++ pkg/resmgr/policy/policy.go | 50 +++++++++++++++++-- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 57c3bc32c..23b4a7e3f 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -372,6 +372,18 @@ func (p *balloons) UpdateResources(c cache.Container) error { return nil } +// AllocateClaim allocates CPUs for the claim. +func (p *balloons) AllocateClaim(claim policy.Claim) error { + log.Debug("allocating claim %s for pods %v...", claim.String(), claim.GetPods()) + return nil +} + +// ReleaseClaim releases CPUs of the claim. +func (p *balloons) ReleaseClaim(claim policy.Claim) error { + log.Debug("releasing claim %s for pods %v...", claim.String(), claim.GetPods()) + return nil +} + // HandleEvent handles policy-specific events. func (p *balloons) HandleEvent(*events.Policy) (bool, error) { log.Debug("(not) handling event...") diff --git a/cmd/plugins/template/policy/template-policy.go b/cmd/plugins/template/policy/template-policy.go index b1684ba7d..0d73c6074 100644 --- a/cmd/plugins/template/policy/template-policy.go +++ b/cmd/plugins/template/policy/template-policy.go @@ -108,6 +108,18 @@ func (p *policy) UpdateResources(c cache.Container) error { return nil } +// AllocateClaim alloctes CPUs for the claim. +func (p *policy) AllocateClaim(claim policyapi.Claim) error { + log.Debug("allocating claim %s for pods %v...", claim.String(), claim.GetPods()) + return nil +} + +// ReleaseClaim releases CPUs of the claim. +func (p *policy) ReleaseClaim(claim policyapi.Claim) error { + log.Debug("releasing claim %s for pods %v...", claim.String(), claim.GetPods()) + return nil +} + // HandleEvent handles policy-specific events. func (p *policy) HandleEvent(e *events.Policy) (bool, error) { log.Info("received policy event %s.%s with data %v...", e.Source, e.Type, e.Data) diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index c7f8b0fa4..356aec885 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -291,6 +291,18 @@ func (p *policy) UpdateResources(container cache.Container) error { return nil } +// AllocateClaim allocates CPUs for the claim. +func (p *policy) AllocateClaim(claim policyapi.Claim) error { + log.Info("allocating claim %s for pods %v...", claim.String(), claim.GetPods()) + return nil +} + +// ReleaseClaim releases CPUs of the claim. +func (p *policy) ReleaseClaim(claim policyapi.Claim) error { + log.Info("releasing claim %s for pods %v...", claim.String(), claim.GetPods()) + return nil +} + // HandleEvent handles policy-specific events. func (p *policy) HandleEvent(e *events.Policy) (bool, error) { log.Debug("received policy event %s.%s with data %v...", e.Source, e.Type, e.Data) diff --git a/pkg/resmgr/policy/policy.go b/pkg/resmgr/policy/policy.go index 3967693e3..7c8cb898a 100644 --- a/pkg/resmgr/policy/policy.go +++ b/pkg/resmgr/policy/policy.go @@ -20,6 +20,7 @@ import ( "sort" "k8s.io/apimachinery/pkg/api/resource" + apitypes "k8s.io/apimachinery/pkg/types" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/events" @@ -56,6 +57,8 @@ type ConstraintSet map[Domain]Constraint type Options struct { // SendEvent is the function for delivering events back to the resource manager. SendEvent SendEventFn + // PublishCPUs is the function for publishing CPUs as DRA devices. + PublishCPUs PublishCPUFn } // BackendOptions describes the options for a policy backend instance @@ -66,6 +69,8 @@ type BackendOptions struct { Cache cache.Cache // SendEvent is the function for delivering events up to the resource manager. SendEvent SendEventFn + // PublishCPUs publishes CPUs as DRA devices. + PublishCPUs PublishCPUFn // Config is the policy-specific configuration. Config interface{} } @@ -76,6 +81,22 @@ type CreateFn func(*BackendOptions) Backend // SendEventFn is the type for a function to send events back to the resource manager. type SendEventFn func(interface{}) error +// PublishCPUFn is the type for a function to publish CPUs as DRA devices. +type PublishCPUFn func([]ID) error + +// Claim represents a DRA Resource Claim, for CPUs. +type Claim interface { + GetUID() UID + GetPods() []UID + GetDevices() []ID + String() string +} + +type ( + UID = apitypes.UID + ID = system.ID +) + const ( // ExportedResources is the basename of the file container resources are exported to. ExportedResources = "resources.sh" @@ -85,6 +106,8 @@ const ( ExportIsolatedCPUs = "ISOLATED_CPUS" // ExportExclusiveCPUs is the shell variable used to export exclusive container CPUs. ExportExclusiveCPUs = "EXCLUSIVE_CPUS" + // ExportClaimedCPUs is the shell variable used to export DRA-claimed container CPUs. + ExportClaimedCPUs = "CLAIMED_CPUS" ) // Backend is the policy (decision making logic) interface exposed by implementations. @@ -110,6 +133,10 @@ type Backend interface { ReleaseResources(cache.Container) error // UpdateResources updates resource allocations of a container. UpdateResources(cache.Container) error + // AllocateClaim allocates CPUs for the claim. + AllocateClaim(Claim) error + // ReleaseClaim releases CPUs of the claim. + ReleaseClaim(Claim) error // HandleEvent processes the given event. The returned boolean indicates whether // changes have been made to any of the containers while handling the event. HandleEvent(*events.Policy) (bool, error) @@ -137,6 +164,10 @@ type Policy interface { ReleaseResources(cache.Container) error // UpdateResources updates resource allocations of a container. UpdateResources(cache.Container) error + // AllocateClaim allocates CPUs for the claim. + AllocateClaim(Claim) error + // ReleaseClaim releases CPUs of the claim. + ReleaseClaim(Claim) error // HandleEvent passes on the given event to the active policy. The returned boolean // indicates whether changes have been made to any of the containers while handling // the event. @@ -255,10 +286,11 @@ func (p *policy) Start(cfg interface{}) error { log.Info("activating '%s' policy...", p.active.Name()) if err := p.active.Setup(&BackendOptions{ - Cache: p.cache, - System: p.system, - SendEvent: p.options.SendEvent, - Config: cfg, + Cache: p.cache, + System: p.system, + SendEvent: p.options.SendEvent, + PublishCPUs: p.options.PublishCPUs, + Config: cfg, }); err != nil { return err } @@ -306,6 +338,16 @@ func (p *policy) UpdateResources(c cache.Container) error { return p.active.UpdateResources(c) } +// AllocateClaim allocates CPUs for the claim. +func (p *policy) AllocateClaim(claim Claim) error { + return p.active.AllocateClaim(claim) +} + +// ReleaseClaim releases CPUs of the claim. +func (p *policy) ReleaseClaim(claim Claim) error { + return p.active.ReleaseClaim(claim) +} + // HandleEvent passes on the given event to the active policy. func (p *policy) HandleEvent(e *events.Policy) (bool, error) { return p.active.HandleEvent(e) From 84d6436d96111bbc19eb1cf3304cd10e6b4ce53b Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 6 Jun 2025 00:01:52 +0300 Subject: [PATCH 12/15] resmgr: add experimental DRA driver/plugin. Register as a 'native.cpu' DRA resource driver/plugin. Provide DRA CPU device publishing for policy implementations. Generate CDI Spec for the published CPU devices. Hook the active policy in for CPUs allocation and release for claim preparation and un- preparation. Signed-off-by: Krisztian Litkey --- pkg/resmgr/dra.go | 482 +++++++++++++++++++++++++++++++++ pkg/resmgr/resource-manager.go | 25 +- 2 files changed, 505 insertions(+), 2 deletions(-) create mode 100644 pkg/resmgr/dra.go diff --git a/pkg/resmgr/dra.go b/pkg/resmgr/dra.go new file mode 100644 index 000000000..2c6ca77d1 --- /dev/null +++ b/pkg/resmgr/dra.go @@ -0,0 +1,482 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resmgr + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/containers/nri-plugins/pkg/kubernetes/client" + logger "github.com/containers/nri-plugins/pkg/log" + "github.com/containers/nri-plugins/pkg/resmgr/cache" + system "github.com/containers/nri-plugins/pkg/sysfs" + "google.golang.org/grpc" + resapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/dynamic-resource-allocation/kubeletplugin" + "k8s.io/dynamic-resource-allocation/resourceslice" +) + +type draPlugin struct { + driverName string + nodeName string + resmgr *resmgr + cancel context.CancelFunc + plugin *kubeletplugin.Helper + publishCh chan<- *resourceslice.DriverResources + claims savedClaims +} + +type UID = types.UID + +var ( + dra = logger.NewLogger("dra-driver") +) + +const ( + driverName = "native.cpu" + driverKind = driverName + "/device" + cdiVersion = "0.7.0" + cdiEnvVar = "DRA_CPU" +) + +func (resmgr *resmgr) publishCPUs(cpuIDs []system.ID) error { + if resmgr.dra == nil { + return fmt.Errorf("can't publish CPUs as DRA devices, no DRA plugin") + } + + if err := resmgr.dra.writeCDISpecFile(opt.HostRoot, cpuIDs); err != nil { + log.Errorf("failed to write CDI Spec file: %v", err) + return err + } + + cpuDevices := resmgr.system.CPUsAsDRADevices(cpuIDs) + if err := resmgr.dra.PublishResources(context.Background(), cpuDevices); err != nil { + log.Errorf("failed to publish DRA resources: %v", err) + return err + } + + return nil +} + +func newDRAPlugin(resmgr *resmgr) (*draPlugin, error) { + driverPath := filepath.Join(kubeletplugin.KubeletPluginsDir, driverName) + if err := os.MkdirAll(driverPath, 0750); err != nil { + return nil, fmt.Errorf("failed to create driver directory %s: %w", driverPath, err) + } + + p := &draPlugin{ + driverName: driverName, + nodeName: resmgr.agent.NodeName(), + resmgr: resmgr, + claims: make(map[UID]*draClaim), + } + + p.restoreClaims() + + return p, nil +} + +func (p *draPlugin) start() error { + p.run() + return nil +} + +func (p *draPlugin) run() { + var ( + ctx, cancel = context.WithCancel(context.Background()) + publishCh = make(chan *resourceslice.DriverResources, 1) + ) + + go func() { + for { + var resources *resourceslice.DriverResources + + select { + case <-ctx.Done(): + return + case r, ok := <-publishCh: + if !ok { + return + } + resources = r + } + + if p.plugin == nil { + if err := p.connect(); err != nil { + log.Errorf("failed start DRA plugin: %v", err) + continue + } + } + + if p.plugin != nil { + if err := p.plugin.PublishResources(ctx, *resources); err != nil { + log.Errorf("failed to publish DRA resources: %v", err) + } else { + log.Infof("published DRA resources, using %d pool(s)...", len(resources.Pools)) + } + } + + resources = nil + } + }() + + p.cancel = cancel + p.publishCh = publishCh +} + +func (p *draPlugin) connect() error { + kubeClient, err := client.New( + client.WithKubeOrInClusterConfig(p.resmgr.agent.KubeConfig()), + client.WithAcceptContentTypes(client.ContentTypeProtobuf, client.ContentTypeJSON), + client.WithContentType(client.ContentTypeProtobuf), + ) + if err != nil { + return fmt.Errorf("can't create kube client for DRA plugin: %w", err) + } + + options := []kubeletplugin.Option{ + kubeletplugin.DriverName(p.driverName), + kubeletplugin.NodeName(p.nodeName), + kubeletplugin.KubeClient(kubeClient.Clientset), + kubeletplugin.GRPCInterceptor(p.unaryInterceptor), + } + + log.Infof("using DRA driverName=%s nodeName=%s", p.driverName, p.nodeName) + + plugin, err := kubeletplugin.Start(context.Background(), p, options...) + if err != nil { + return fmt.Errorf("failed to start DRA plugin: %w", err) + } + + p.plugin = plugin + return nil +} + +func (p *draPlugin) stop() { + if p == nil { + return + } + + if p.plugin != nil { + p.plugin.Stop() + } + if p.cancel != nil { + p.cancel() + } + + p.plugin = nil + p.cancel = nil +} + +func (p *draPlugin) unaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + dra.Info("=> gRPC call %v, handler %v\n", *info, handler) + rpl, err := handler(ctx, req) + dra.Info("<= gRPC reply: %+v, %v\n", rpl, err) + return rpl, err +} + +func (p *draPlugin) IsRegistered() (bool, error) { + if p == nil || p.plugin == nil { + return false, errors.New("DRA plugin is not initialized") + } + + status := p.plugin.RegistrationStatus() + if status == nil { + return false, nil + } + + var err error + if status.Error != "" { + err = errors.New(status.Error) + } + + return status.PluginRegistered, err +} + +func (p *draPlugin) PublishResources(ctx context.Context, devices []resapi.Device) error { + resources := resourceslice.DriverResources{ + Pools: make(map[string]resourceslice.Pool), + } + + maxPerPool := resapi.ResourceSliceMaxDevices + for n := len(devices); n > 0; n = len(devices) { + if n > maxPerPool { + n = maxPerPool + } + resources.Pools["pool"+strconv.Itoa(len(resources.Pools))] = resourceslice.Pool{ + Slices: []resourceslice.Slice{ + { + Devices: devices[:n], + }, + }, + } + devices = devices[n:] + } + + log.Infof("publishing DRA resources, using %d pool(s)...", len(resources.Pools)) + + select { + case p.publishCh <- &resources: + return nil + default: + } + + return fmt.Errorf("failed to publish resources, failed to send on channel") +} + +func (p *draPlugin) writeCDISpecFile(hostRoot string, cpuIDs []system.ID) error { + spec := bytes.NewBuffer(nil) + fmt.Fprintf(spec, "cdiVersion: %s\nkind: %s\ndevices:\n", cdiVersion, driverKind) + for _, id := range cpuIDs { + fmt.Fprintf(spec, " - name: cpu%d\n", id) + fmt.Fprintf(spec, " containerEdits:\n") + fmt.Fprintf(spec, " env:\n") + fmt.Fprintf(spec, " - %s%d=1\n", cdiEnvVar, id) + } + + dir := filepath.Join(hostRoot, "/var/run/cdi") + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create CDI Spec directory: %w", err) + } + + if err := os.WriteFile(filepath.Join(dir, driverName+".yaml"), spec.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to write CDI Spec file: %w", err) + } + + return nil +} + +func (p *draPlugin) PrepareResourceClaims(ctx context.Context, claims []*resapi.ResourceClaim) (map[UID]kubeletplugin.PrepareResult, error) { + log.Infof("should prepare %d claims:", len(claims)) + + undoAndErrOut := func(err error, release []*resapi.ResourceClaim) error { + for _, c := range release { + if relc := p.claims.del(c.UID); relc != nil { + if undoErr := p.resmgr.policy.ReleaseClaim(relc); undoErr != nil { + log.Error("rollback error, failed to release claim %s: %v", c.UID, undoErr) + } + } + } + return err + } + + defer func() { + if err := p.saveClaims(); err != nil { + log.Error("failed to save claims: %v", err) + } + }() + p.resmgr.Lock() + defer p.resmgr.Unlock() + + result := make(map[UID]kubeletplugin.PrepareResult) + + for i, c := range claims { + if c == nil { + continue + } + + dra.Debug(" - claim #%d:", i) + specHdr := fmt.Sprintf(" ", i) + statusHdr := fmt.Sprintf(" ", i) + dra.DebugBlock(specHdr, "%s", logger.AsYaml(c.Spec)) + dra.DebugBlock(statusHdr, "%s", logger.AsYaml(c.Status)) + + if old, ok := p.claims.get(c.UID); ok { + log.Infof("claim %q already prepared, reusing it", c.UID) + result[c.UID] = *(old.GetResult()) + continue + } + + claim := &draClaim{ResourceClaim: c} + + if err := p.resmgr.policy.AllocateClaim(claim); err != nil { + log.Error("failed to prepare claim %q: %v", c.UID, err) + return nil, undoAndErrOut(err, claims[:i]) + } + + result[claim.GetUID()] = *(claim.GetResult()) + p.claims.add(claim) + } + + return result, nil +} + +func (p *draPlugin) UnprepareResourceClaims(ctx context.Context, claims []kubeletplugin.NamespacedObject) (map[UID]error, error) { + + log.Infof("should un-prepare %d claims:", len(claims)) + + defer func() { + if err := p.saveClaims(); err != nil { + log.Error("failed to save claims: %v", err) + } + }() + p.resmgr.Lock() + defer p.resmgr.Unlock() + + result := make(map[UID]error) + + for _, c := range claims { + log.Infof(" - un-claim %+v", c) + + if claim := p.claims.del(c.UID); claim != nil { + if err := p.resmgr.policy.ReleaseClaim(claim); err != nil { + log.Errorf("failed to release claim %s: %v", claim, err) + } + } + + result[c.UID] = nil + } + + return result, nil +} + +func (p *draPlugin) ErrorHandler(ctx context.Context, err error, msg string) { + log.Errorf("resource slice publishing error: %v (%s)", err, msg) +} + +func (p *draPlugin) HandleError(ctx context.Context, err error, msg string) { + log.Errorf("resource slice publishing error: %v (%s)", err, msg) +} + +func (p *draPlugin) saveClaims() error { + p.resmgr.cache.SetEntry("claims", p.claims) + return p.resmgr.cache.Save() +} + +func (p *draPlugin) restoreClaims() { + claims := make(savedClaims) + restored, err := p.resmgr.cache.GetEntry("claims", &claims) + + if err != nil { + if err != cache.ErrNoEntry { + log.Error("failed to restore claims: %v", err) + } + p.claims = make(savedClaims) + } else { + if restored == nil { + p.claims = make(savedClaims) + } else { + p.claims = *restored.(*savedClaims) + } + } +} + +type draClaim struct { + *resapi.ResourceClaim + pods []UID + devs []system.ID +} + +func (c *draClaim) GetUID() UID { + if c == nil || c.ResourceClaim == nil { + return "" + } + return c.UID +} + +func (c *draClaim) GetPods() []UID { + if c == nil || c.ResourceClaim == nil { + return nil + } + if c.pods != nil { + return c.pods + } + + var pods []UID + for _, r := range c.Status.ReservedFor { + if r.Resource == "pods" { + pods = append(pods, r.UID) + } + } + c.pods = pods + + return c.pods +} + +func (c *draClaim) GetDevices() []system.ID { + if c == nil || c.ResourceClaim == nil { + return nil + } + + if c.devs != nil { + return c.devs + } + + var ids []system.ID + for _, r := range c.Status.Allocation.Devices.Results { + num := strings.TrimPrefix(r.Device, "cpu") + i, err := strconv.ParseInt(num, 10, 32) + if err != nil { + log.Errorf("failed to parse CPU ID %q: %v", num, err) + continue + } + ids = append(ids, system.ID(i)) + } + c.devs = ids + + return c.devs +} + +func (c *draClaim) GetResult() *kubeletplugin.PrepareResult { + result := &kubeletplugin.PrepareResult{} + for _, alloc := range c.Status.Allocation.Devices.Results { + result.Devices = append(result.Devices, + kubeletplugin.Device{ + Requests: []string{alloc.Request}, + DeviceName: alloc.Device, + CDIDeviceIDs: []string{driverKind + "=" + alloc.Device}, + }) + } + return result +} + +func (c *draClaim) String() string { + return fmt.Sprintf("", + c.GetUID(), c.GetDevices(), c.GetPods()) +} + +func (c *draClaim) MarshalJSON() ([]byte, error) { + return json.Marshal(c.ResourceClaim) +} + +func (c *draClaim) UnmarshalJSON(b []byte) error { + c.ResourceClaim = &resapi.ResourceClaim{} + return json.Unmarshal(b, c.ResourceClaim) +} + +type savedClaims map[UID]*draClaim + +func (s *savedClaims) add(c *draClaim) { + (*s)[c.UID] = c +} + +func (s *savedClaims) del(uid UID) *draClaim { + c := (*s)[uid] + delete(*s, uid) + return c +} + +func (s *savedClaims) get(uid UID) (*draClaim, bool) { + c, ok := (*s)[uid] + return c, ok +} diff --git a/pkg/resmgr/resource-manager.go b/pkg/resmgr/resource-manager.go index d75ec3d2d..9b02b1148 100644 --- a/pkg/resmgr/resource-manager.go +++ b/pkg/resmgr/resource-manager.go @@ -53,6 +53,7 @@ type resmgr struct { agent *agent.Agent cfg cfgapi.ResmgrConfig cache cache.Cache // cached state + system sysfs.System // sysfs in use policy policy.Policy // resource manager policy control control.Control // policy controllers/enforcement events chan interface{} // channel for delivering events @@ -60,6 +61,7 @@ type resmgr struct { nri *nriPlugin // NRI plugins, if we're running as such rdt *rdtControl // control for RDT allocation and monitoring blkio *blkioControl // control for block I/O prioritization and throttling + dra *draPlugin // DRA plugin, if we have one running bool } @@ -88,7 +90,7 @@ func NewResourceManager(backend policy.Backend, agt *agent.Agent) (ResourceManag return nil, err } - log.Info("running as an NRI plugin...") + log.Info("creating NRI plugin...") nrip, err := newNRIPlugin(m) if err != nil { return nil, err @@ -99,6 +101,13 @@ func NewResourceManager(backend policy.Backend, agt *agent.Agent) (ResourceManag return nil, err } + log.Info("creating DRA plugin...") + drap, err := newDRAPlugin(m) + if err != nil { + return nil, err + } + m.dra = drap + if err := m.setupEventProcessing(); err != nil { return nil, err } @@ -186,6 +195,10 @@ func (m *resmgr) start(cfg cfgapi.ResmgrConfig) error { return err } + if err := m.dra.start(); err != nil { + return err + } + if err := m.policy.Start(m.cfg.PolicyConfig()); err != nil { return err } @@ -222,6 +235,7 @@ func (m *resmgr) Stop() { defer m.Unlock() m.nri.stop() + m.dra.stop() } // setupCache creates a cache and reloads its last saved state if found. @@ -249,10 +263,17 @@ func (m *resmgr) setupPolicy(backend policy.Backend) error { log.Warnf("failed to set active policy: %v", err) } - p, err := policy.NewPolicy(backend, m.cache, &policy.Options{SendEvent: m.SendEvent}) + p, err := policy.NewPolicy(backend, m.cache, + &policy.Options{ + SendEvent: m.SendEvent, + PublishCPUs: m.publishCPUs, + }, + ) if err != nil { return resmgrError("failed to create policy %s: %v", backend.Name(), err) } + + m.system = p.System() m.policy = p return nil From b2f9e8fd3cd46db4ac47500e45ab09c8de9f8b87 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Mon, 9 Feb 2026 17:58:35 +0200 Subject: [PATCH 13/15] resmgr: flush pending updates after claim unprepare. Flush pending changes immediately using an unsolicited update to containers instead of waiting for the next NRI event to do so. Signed-off-by: Krisztian Litkey --- pkg/resmgr/dra.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/resmgr/dra.go b/pkg/resmgr/dra.go index 2c6ca77d1..75d57b08e 100644 --- a/pkg/resmgr/dra.go +++ b/pkg/resmgr/dra.go @@ -347,6 +347,10 @@ func (p *draPlugin) UnprepareResourceClaims(ctx context.Context, claims []kubele result[c.UID] = nil } + if err := p.resmgr.nri.updateContainers(); err != nil { + log.Errorf("failed to update containers after releasing claim: %v", err) + } + return result, nil } From cdaa70e1d8aa3ed99bd8340830fd4b78f369a2d4 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 5 Jun 2025 23:32:39 +0300 Subject: [PATCH 14/15] topology-aware: bolt on a DRA CPU driver frontend. Publish CPUs as DRA devices. Use a sledgehammer to bolt allocation and release of DRA-claimed CPUs on top of the current implementation. Signed-off-by: Krisztian Litkey --- cmd/plugins/topology-aware/policy/cache.go | 3 + cmd/plugins/topology-aware/policy/pools.go | 192 +++++++++++++++++- .../topology-aware/policy/resources.go | 90 +++++++- .../policy/topology-aware-policy.go | 24 ++- 4 files changed, 292 insertions(+), 17 deletions(-) diff --git a/cmd/plugins/topology-aware/policy/cache.go b/cmd/plugins/topology-aware/policy/cache.go index 454cb8bd6..82cf5d3e2 100644 --- a/cmd/plugins/topology-aware/policy/cache.go +++ b/cmd/plugins/topology-aware/policy/cache.go @@ -126,6 +126,7 @@ func (p *policy) reinstateGrants(grants map[string]Grant) error { type cachedGrant struct { PrettyName string + Claimed string Exclusive string Part int CPUType cpuClass @@ -140,6 +141,7 @@ type cachedGrant struct { func newCachedGrant(cg Grant) *cachedGrant { ccg := &cachedGrant{} ccg.PrettyName = cg.GetContainer().PrettyName() + ccg.Claimed = cg.ClaimedCPUs().String() ccg.Exclusive = cg.ExclusiveCPUs().String() ccg.Part = cg.CPUPortion() ccg.CPUType = cg.CPUType() @@ -168,6 +170,7 @@ func (ccg *cachedGrant) ToGrant(policy *policy) (Grant, error) { container, ccg.CPUType, cpuset.MustParse(ccg.Exclusive), + cpuset.MustParse(ccg.Claimed), ccg.Part, ccg.MemType, ccg.ColdStart, diff --git a/cmd/plugins/topology-aware/policy/pools.go b/cmd/plugins/topology-aware/policy/pools.go index 87fe22a31..9082f5182 100644 --- a/cmd/plugins/topology-aware/policy/pools.go +++ b/cmd/plugins/topology-aware/policy/pools.go @@ -18,6 +18,8 @@ import ( "fmt" "math" "sort" + "strconv" + "strings" "github.com/containers/nri-plugins/pkg/utils/cpuset" corev1 "k8s.io/api/core/v1" @@ -25,6 +27,7 @@ import ( cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/resmgr/cache" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" + policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" system "github.com/containers/nri-plugins/pkg/sysfs" idset "github.com/intel/goresctrl/pkg/utils" ) @@ -349,7 +352,14 @@ func (p *policy) allocatePool(container cache.Container, poolHint string) (Grant offer *libmem.Offer ) - request := newRequest(container, p.memAllocator.Masks().AvailableTypes()) + claimed, unclaimed, err := p.getClaimedCPUs(container) + if err != nil { + return nil, err + } + + log.Info("**** claimed CPU(s): %s, unclaimed CPU: %d", claimed, unclaimed) + + request := newRequest(container, claimed, p.memAllocator.Masks().AvailableTypes()) if p.root.FreeSupply().ReservedCPUs().IsEmpty() && request.CPUType() == cpuReserved { // Fallback to allocating reserved CPUs from the shared pool @@ -440,6 +450,151 @@ func (p *policy) allocatePool(container cache.Container, poolHint string) (Grant return grant, nil } +func (p *policy) allocateClaim(claim policyapi.Claim) error { + hints := map[string]string{} + + // collect grants we need to free dues to conflciting exclusive allocation + realloc := []Grant{} + cpus := cpuset.New(claim.GetDevices()...) + for _, g := range p.allocations.grants { + if !cpus.Intersection(g.ExclusiveCPUs().Union(g.IsolatedCPUs())).IsEmpty() { + log.Info("***** releasing conflicting grant %s", g) + hints[g.GetContainer().GetID()] = g.GetCPUNode().Name() + realloc = append(realloc, g) + p.releasePool(g.GetContainer()) + p.updateSharedAllocations(&g) + } + } + + pool := p.getPoolForCPUs(cpus) + if pool == nil { + return fmt.Errorf("failed to find pool for claimed CPUs %s", cpus.String()) + } + + log.Info("*** target pool is %s", pool.Name()) + + // free more grants if we need to, for slicing off claimed CPUs + s := pool.FreeSupply() + needShared := s.SharableCPUs().Intersection(cpus).Size() + if available, needed := s.AllocatableSharedCPU(), 1000*needShared; available < needed { + log.Info("***** need to free %d mCPU shared capacity", needed-available) + grants := p.getLargestSharedUsers(pool) + log.Info("grants: %v", grants) + for _, g := range grants { + hints[g.GetContainer().GetID()] = g.GetCPUNode().Name() + realloc = append(realloc, g) + p.releasePool(g.GetContainer()) + p.updateSharedAllocations(&g) + if available, needed = s.AllocatableSharedCPU(), 1000*needShared; available >= needed { + break + } + } + } + s.ClaimCPUs(cpus) + + // TODO: sort old grants by QoS class or size and pool distance from root... + for _, oldg := range realloc { + c := oldg.GetContainer() + log.Info("***** reallocating %s with pool hint %s", c.PrettyName(), hints[c.GetID()]) + err := p.allocateResources(c, hints[c.GetID()]) + if err != nil { + log.Error("failed to reallocate container %s (%s), retrying with no pool hint", c.PrettyName(), c.GetID()) + } + err = p.allocateResources(c, "") + if err != nil { + log.Error("failed to reallocate container %s without pool hints", c.PrettyName(), c.GetID()) + } + } + + return nil +} + +// Find (tightest fitting) target pool for a set of cpus +func (p *policy) getPoolForCPUs(cpus cpuset.CPUSet) Node { + var pool Node + for _, n := range p.pools { + s := n.GetSupply() + poolCPUs := s.SharableCPUs().Union(s.IsolatedCPUs()).Union(s.ReservedCPUs()) + if poolCPUs.Intersection(cpus).Equals(cpus) { + if pool == nil { + pool = n + } else { + if n.RootDistance() > pool.RootDistance() { + pool = n + } + } + } + } + return pool +} + +// Get the largest shared CPU users in a pool and its parents. +func (p *policy) getLargestSharedUsers(pool Node) []Grant { + pools := map[string]struct{}{} + grants := []Grant{} + for n := pool; !n.IsNil(); n = n.Parent() { + pools[n.Name()] = struct{}{} + } + + for _, g := range p.allocations.grants { + if g.SharedPortion() > 0 { + grants = append(grants, g) + } + } + + sort.Slice(grants, func(i, j int) bool { + gi, gj := grants[i], grants[j] + di, dj := gi.GetCPUNode().RootDistance(), gj.GetCPUNode().RootDistance() + si, sj := gi.SharedPortion(), gj.SharedPortion() + if di < dj { + return true + } + return si <= sj + }) + + return grants +} + +func (p *policy) releaseClaim(claim policyapi.Claim) error { + cpus := cpuset.New(claim.GetDevices()...) + p.root.FreeSupply().UnclaimCPUs(cpus) + p.updateSharedAllocations(nil) + + return nil +} + +// Get the claimed CPUs injected into the environment of a container. +func (p *policy) getClaimedCPUs(c cache.Container) (cpuset.CPUSet, int, error) { + var ids []int + for _, env := range c.GetEnvList() { + if !strings.HasPrefix(env, "DRA_CPU") { + continue + } + val := strings.TrimSuffix(strings.TrimPrefix(env, "DRA_CPU"), "=1") + i, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return cpuset.New(), 0, fmt.Errorf("invalid DRA CPU env. var %q: %v", env, err) + } + ids = append(ids, int(i)) + } + + claimed := cpuset.New(ids...) + reqs, ok := c.GetResourceUpdates() + if !ok { + reqs = c.GetResourceRequirements() + } + request := reqs.Requests[corev1.ResourceCPU] + unclaimed := int(request.MilliValue()) - 1000*claimed.Size() + + if unclaimed < 0 { + return claimed, unclaimed, + fmt.Errorf("invalid claimed (%s CPU(s)) vs. requested/native CPU (%dm)", + claimed, request.MilliValue()) + } + + return claimed, unclaimed, nil +} + // setPreferredCpusetCpus pins container's CPUs according to what has been // allocated for it, taking into account if the container should run // with hyperthreads hidden. @@ -465,6 +620,7 @@ func (p *policy) applyGrant(grant Grant) { container := grant.GetContainer() cpuType := grant.CPUType() + claimed := grant.ClaimedCPUs() exclusive := grant.ExclusiveCPUs() reserved := grant.ReservedCPUs() shared := grant.SharedCPUs() @@ -474,16 +630,25 @@ func (p *policy) applyGrant(grant Grant) { kind := "" switch cpuType { case cpuNormal: - if exclusive.IsEmpty() { + if exclusive.IsEmpty() && claimed.IsEmpty() { cpus = shared kind = "shared" } else { - kind = "exclusive" + if !exclusive.IsEmpty() { + kind = "exclusive" + } + if !claimed.IsEmpty() { + if kind == "" { + kind = "claimed" + } else { + kind += "+claimed" + } + } if cpuPortion > 0 { kind += "+shared" - cpus = exclusive.Union(shared) + cpus = exclusive.Union(claimed).Union(shared) } else { - cpus = exclusive + cpus = exclusive.Union(claimed) } } case cpuReserved: @@ -618,9 +783,14 @@ func (p *policy) updateSharedAllocations(grant *Grant) { continue } + if other.SharedPortion() == 0 && !other.ClaimedCPUs().IsEmpty() { + log.Info(" => %s not affected (only claimed CPUs)...", other) + continue + } + if opt.PinCPU { shared := other.GetCPUNode().FreeSupply().SharableCPUs() - exclusive := other.ExclusiveCPUs() + exclusive := other.ExclusiveCPUs().Union(other.ClaimedCPUs()) if exclusive.IsEmpty() { p.setPreferredCpusetCpus(other.GetContainer(), shared, fmt.Sprintf(" => updating %s with shared CPUs of %s: %s...", @@ -756,6 +926,8 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco depth1, depth2 := node1.RootDistance(), node2.RootDistance() id1, id2 := node1.NodeID(), node2.NodeID() score1, score2 := scores[id1], scores[id2] + claim1, claim2 := node1.FreeSupply().ClaimedCPUs(), node2.FreeSupply().ClaimedCPUs() + claim := request.ClaimedCPUs() cpuType := request.CPUType() isolated1, reserved1, shared1 := score1.IsolatedCapacity(), score1.ReservedCapacity(), score1.SharedCapacity() isolated2, reserved2, shared2 := score2.IsolatedCapacity(), score2.ReservedCapacity(), score2.SharedCapacity() @@ -772,7 +944,7 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco // // Our scoring/score sorting algorithm is: // - // - insufficient isolated, reserved or shared capacity loses + // - insufficient claimed, isolated, reserved or shared capacity loses // - if we have affinity, the higher affinity score wins // - if we have topology hints // * better hint score wins @@ -799,8 +971,12 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco // Before this comparison is reached, nodes with insufficient uncompressible resources // (memory) have been filtered out. - // a node with insufficient isolated or shared capacity loses + // a node with insufficient claimed, isolated or shared capacity loses switch { + case claim.Difference(claim1).Size() == 0 && claim.Difference(claim2).Size() > 0: + return true + case claim.Difference(claim1).Size() > 0 && claim.Difference(claim2).Size() == 0: + return false case cpuType == cpuNormal && ((isolated2 < 0 && isolated1 >= 0) || (shared2 < 0 && shared1 >= 0)): log.Debug(" => %s loses, insufficent isolated or shared", node2.Name()) return true diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index 794312e76..c3a316651 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -65,6 +65,8 @@ type Supply interface { ReservedCPUs() cpuset.CPUSet // SharableCPUs returns the sharable cpuset in this supply. SharableCPUs() cpuset.CPUSet + // ClaimedCPUs returns the claimed CPUs for this supply. + ClaimedCPUs() cpuset.CPUSet // GrantedReserved returns the locally granted reserved CPU capacity in this supply. GrantedReserved() int // GrantedShared returns the locally granted shared CPU capacity in this supply. @@ -92,6 +94,12 @@ type Supply interface { DumpCapacity() string // DumpAllocatable returns a printable representation of the supply's alloctable resources. DumpAllocatable() string + + ClaimCPUs(cpuset.CPUSet) + claimCPUs(cpuset.CPUSet) + + UnclaimCPUs(cpuset.CPUSet) + unclaimCPUs(cpuset.CPUSet) } // Request represents CPU and memory resources requested by a container. @@ -106,6 +114,8 @@ type Request interface { CPUPrio() cpuPrio // SetCPUType sets the type of requested CPU. SetCPUType(cpuType cpuClass) + // ClaimedCPUs returns the CPUs claimed for this request. + ClaimedCPUs() cpuset.CPUSet // FullCPUs return the number of full CPUs requested. FullCPUs() int // CPUFraction returns the amount of fractional milli-CPU requested. @@ -147,6 +157,8 @@ type Grant interface { // CPUPortion returns granted milli-CPUs of non-full CPUs of CPUType(). // CPUPortion() == ReservedPortion() + SharedPortion(). CPUPortion() int + // ClaimedCPUs returns the claimed granted cpuset. + ClaimedCPUs() cpuset.CPUSet // ExclusiveCPUs returns the exclusively granted non-isolated cpuset. ExclusiveCPUs() cpuset.CPUSet // ReservedCPUs returns the reserved granted cpuset. @@ -217,6 +229,7 @@ type supply struct { isolated cpuset.CPUSet // isolated CPUs at this node reserved cpuset.CPUSet // reserved CPUs at this node sharable cpuset.CPUSet // sharable CPUs at this node + claimed cpuset.CPUSet // CPUs allocated through DRA grantedReserved int // amount of reserved CPUs allocated grantedShared int // amount of shareable CPUs allocated } @@ -226,6 +239,7 @@ var _ Supply = &supply{} // request implements our Request interface. type request struct { container cache.Container // container for this request + claimed cpuset.CPUSet // CPUs DRA-claimed for this container full int // number of full CPUs requested fraction int // amount of fractional CPU requested limit int // CPU limit, MaxInt for no limit @@ -252,6 +266,7 @@ type grant struct { container cache.Container // container CPU is granted to node Node // node CPU is supplied from exclusive cpuset.CPUSet // exclusive CPUs + claimed cpuset.CPUSet // claimed CPUs cpuType cpuClass // type of CPUs (normal, reserved, ...) cpuPortion int // milliCPUs granted from CPUs of cpuType memType memoryType // requested types of memory @@ -316,6 +331,11 @@ func (cs *supply) SharableCPUs() cpuset.CPUSet { return cs.sharable.Clone() } +// ClaimedCPUs returns the claimed CPUs in this supply. +func (cs *supply) ClaimedCPUs() cpuset.CPUSet { + return cs.claimed.Clone() +} + // GrantedReserved returns the locally granted reserved CPU capacity. func (cs *supply) GrantedReserved() int { return cs.grantedReserved @@ -398,6 +418,7 @@ func (cs *supply) AllocateCPU(r Request) (Grant, error) { cr := r.(*request) + claimed := cr.claimed full := cr.full fraction := cr.fraction @@ -465,7 +486,7 @@ func (cs *supply) AllocateCPU(r Request) (Grant, error) { cs.node.Name(), full, cs.sharable, cs.AllocatableSharedCPU()) } - grant := newGrant(cs.node, cr.GetContainer(), cpuType, exclusive, 0, 0, 0) + grant := newGrant(cs.node, cr.GetContainer(), cpuType, exclusive, claimed, 0, 0, 0) grant.AccountAllocateCPU() if fraction > 0 { @@ -714,6 +735,44 @@ func (cs *supply) DumpAllocatable() string { return allocatable } +func (cs *supply) ClaimCPUs(cpus cpuset.CPUSet) { + cs.claimCPUs(cpus) + + cs.GetNode().DepthFirst(func(n Node) { + n.FreeSupply().claimCPUs(cpus) + }) + for n := cs.GetNode(); !n.IsNil(); n = n.Parent() { + n.FreeSupply().claimCPUs(cpus) + } +} + +func (cs *supply) claimCPUs(cpus cpuset.CPUSet) { + cs.claimed = cs.claimed.Union(cpus) + cs.isolated = cs.isolated.Difference(cpus) + cs.sharable = cs.sharable.Difference(cpus) + cs.reserved = cs.reserved.Difference(cpus) +} + +func (cs *supply) UnclaimCPUs(cpus cpuset.CPUSet) { + cs.unclaimCPUs(cpus) + + cs.GetNode().DepthFirst(func(n Node) { + n.FreeSupply().unclaimCPUs(cpus) + }) + for n := cs.GetNode(); !n.IsNil(); n = n.Parent() { + n.FreeSupply().unclaimCPUs(cpus) + } +} + +func (cs *supply) unclaimCPUs(cpus cpuset.CPUSet) { + all := cs.GetNode().GetSupply() + + cs.isolated = cs.isolated.Union(all.IsolatedCPUs().Intersection(cpus)) + cs.sharable = cs.sharable.Union(all.SharableCPUs().Intersection(cpus)) + cs.reserved = cs.reserved.Union(all.ReservedCPUs().Intersection(cpus)) + cs.claimed = cs.claimed.Difference(cpus) +} + // prettyMem formats the given amount as k, M, G, or T units. func prettyMem(value int64) string { units := []string{"k", "M", "G", "T"} @@ -732,14 +791,22 @@ func prettyMem(value int64) string { } // newRequest creates a new request for the given container. -func newRequest(container cache.Container, types libmem.TypeMask) Request { +func newRequest(container cache.Container, claimed cpuset.CPUSet, types libmem.TypeMask) Request { pod, _ := container.GetPod() full, fraction, cpuLimit, isolate, cpuType, prio := cpuAllocationPreferences(pod, container) req, lim, mtype := memoryAllocationPreference(pod, container) coldStart := time.Duration(0) - log.Debug("%s: CPU preferences: cpuType=%s, full=%v, fraction=%v, isolate=%v, prio=%v", - container.PrettyName(), cpuType, full, fraction, isolate, prio) + if full >= claimed.Size() { + full -= claimed.Size() + } else { + if fraction >= 1000*claimed.Size() { + fraction -= 1000 * claimed.Size() + } + } + + log.Debug("%s: CPU preferences: cpuType=%s, claim=%s, full=%v, fraction=%v, isolate=%v, prio=%v", + container.PrettyName(), cpuType, claimed, full, fraction, isolate, prio) if mtype == memoryUnspec { mtype = defaultMemoryType &^ memoryHBM @@ -769,6 +836,7 @@ func newRequest(container cache.Container, types libmem.TypeMask) Request { return &request{ container: container, + claimed: claimed, full: full, fraction: fraction, limit: cpuLimit, @@ -825,6 +893,11 @@ func (cr *request) SetCPUType(cpuType cpuClass) { cr.cpuType = cpuType } +// ClaimedPUs return the claimed CPUs for this request. +func (cr *request) ClaimedCPUs() cpuset.CPUSet { + return cr.claimed +} + // FullCPUs return the number of full CPUs requested. func (cr *request) FullCPUs() int { return cr.full @@ -1135,12 +1208,13 @@ func (score *score) String() string { } // newGrant creates a CPU grant from the given node for the container. -func newGrant(n Node, c cache.Container, cpuType cpuClass, exclusive cpuset.CPUSet, cpuPortion int, mt memoryType, coldstart time.Duration) Grant { +func newGrant(n Node, c cache.Container, cpuType cpuClass, exclusive, claimed cpuset.CPUSet, cpuPortion int, mt memoryType, coldstart time.Duration) Grant { grant := &grant{ node: n, container: c, cpuType: cpuType, exclusive: exclusive, + claimed: claimed, cpuPortion: cpuPortion, memType: mt, coldStart: coldstart, @@ -1179,6 +1253,7 @@ func (cg *grant) Clone() Grant { node: cg.GetCPUNode(), container: cg.GetContainer(), exclusive: cg.ExclusiveCPUs(), + claimed: cg.ClaimedCPUs(), cpuType: cg.CPUType(), cpuPortion: cg.SharedPortion(), memType: cg.MemoryType(), @@ -1233,6 +1308,11 @@ func (cg *grant) ExclusiveCPUs() cpuset.CPUSet { return cg.exclusive } +// ClaimedCPUs returns the claimed CPUSet in this grant. +func (cg *grant) ClaimedCPUs() cpuset.CPUSet { + return cg.claimed +} + // ReservedCPUs returns the reserved CPUSet in the supply of this grant. func (cg *grant) ReservedCPUs() cpuset.CPUSet { return cg.node.GetSupply().ReservedCPUs() diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index 356aec885..897d4c267 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -150,6 +150,10 @@ func (p *policy) Start() error { } p.metrics = m + if err := p.options.PublishCPUs(p.allowed.Difference(p.reserved).List()); err != nil { + log.Errorf("failed to publish CPU DRA resources: %v", err) + } + return nil } @@ -294,13 +298,13 @@ func (p *policy) UpdateResources(container cache.Container) error { // AllocateClaim allocates CPUs for the claim. func (p *policy) AllocateClaim(claim policyapi.Claim) error { log.Info("allocating claim %s for pods %v...", claim.String(), claim.GetPods()) - return nil + return p.allocateClaim(claim) } // ReleaseClaim releases CPUs of the claim. func (p *policy) ReleaseClaim(claim policyapi.Claim) error { log.Info("releasing claim %s for pods %v...", claim.String(), claim.GetPods()) - return nil + return p.releaseClaim(claim) } // HandleEvent handles policy-specific events. @@ -416,8 +420,9 @@ func (p *policy) ExportResourceData(c cache.Container) map[string]string { data := map[string]string{} shared := grant.SharedCPUs().String() - isolated := grant.ExclusiveCPUs().Intersection(grant.GetCPUNode().GetSupply().IsolatedCPUs()) + isolated := grant.ExclusiveCPUs().Union(grant.ClaimedCPUs()).Intersection(grant.GetCPUNode().GetSupply().IsolatedCPUs()) exclusive := grant.ExclusiveCPUs().Difference(isolated).String() + claimed := grant.ClaimedCPUs().String() if grant.SharedPortion() > 0 && shared != "" { data[policyapi.ExportSharedCPUs] = shared @@ -428,6 +433,9 @@ func (p *policy) ExportResourceData(c cache.Container) map[string]string { if exclusive != "" { data[policyapi.ExportExclusiveCPUs] = exclusive } + if claimed != "" { + data[policyapi.ExportClaimedCPUs] = claimed + } mems := grant.GetMemoryZone() dram := mems.And(p.memAllocator.Masks().NodesByTypes(libmem.TypeMaskDRAM)) @@ -528,6 +536,10 @@ func (p *policy) Reconfigure(newCfg interface{}) error { } p.metrics = m + if err := p.options.PublishCPUs(p.allowed.Difference(p.reserved).List()); err != nil { + log.Errorf("failed to publish CPU DRA resources: %v", err) + } + return nil } @@ -680,7 +692,11 @@ func (p *policy) newAllocations() allocations { // clone creates a copy of the allocation. func (a *allocations) clone() allocations { - o := allocations{policy: a.policy, grants: make(map[string]Grant)} + o := allocations{ + policy: a.policy, + grants: make(map[string]Grant), + //claims: make(map[string][]policyapi.Claim), + } for id, grant := range a.grants { o.grants[id] = grant.Clone() } From 9deb65b7f45a5177b2d6aeceae7144bde181cf23 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Sat, 14 Jun 2025 11:59:16 +0300 Subject: [PATCH 15/15] docs: add a README for the DRA prototype. Signed-off-by: Krisztian Litkey --- README-DRA-driver-proto.md | 226 +++++++++++++++++++++++++++++++++++++ README.md | 3 + docs/conf.py | 2 +- 3 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 README-DRA-driver-proto.md diff --git a/README-DRA-driver-proto.md b/README-DRA-driver-proto.md new file mode 100644 index 000000000..b432eb25e --- /dev/null +++ b/README-DRA-driver-proto.md @@ -0,0 +1,226 @@ +# Prototyping CPU DRA device abstraction / DRA-based CPU allocation + +## Background + +This prototype patch set bolts a DRA allocation frontend on top of the existing +topology aware resource policy plugin. The main intention with of this patch set +is to + +- provide something practical to play around with for the [feasibility study]( https://docs.google.com/document/d/1Tb_dC60YVCBr7cNYWuVLddUUTMcNoIt3zjd5-8rgug0/edit?tab=t.0#heading=h.iutbebngx80e) of enabling DRA-based CPU allocation, +- allow (relatively) easy experimentation with how to expose CPU as DRA +devices (IOW test various CPU DRA attributes) +- allow testing how DRA-based CPU allocation (using non-trivial CEL expressions) +would scale with cluster and cluster node size + +## Notes + +This patched NRI plugin, especially in its current state and form, is +*not a proposal* for a first real DRA-based CPU driver. + +## Prerequisites for Testing + +To test out this in a cluster, make sure you have + +1. A recent enough control plane with DRA v1 support (for instance Kubernetes v1.35.0). + +2. CDI enabled in your runtime configuration + +## Installation and Testing + +Once you have your cluster properly set upset up, you can pull this in to +your cluster with for testing with something like this: + +```bash +helm install --devel -n kube-system test oci://ghcr.io/klihub/nri-plugins/helm-charts/nri-resource-policy-topology-aware --version v0.12-dra-driver-unstable --set image.pullPolicy=Always --set extraEnv.OVERRIDE_SYS_ATOM_CPUS='2-5' --set extraEnv.OVERRIDE_SYS_CORE_CPUS='0\,1\,6-15' +``` + +Once the NRI plugin+DRA driver is up and running, you should see some CPUs +exposed as DRI devices. You can check the resource slices with the following +command + +```bash +[kli@n4c16-fedora-40-cloud-base-containerd ~]# kubectl get resourceslices +NAME NODE DRIVER POOL AGE +n4c16-fedora-40-cloud-base-containerd-native.cpu-jxfkj n4c16-fedora-40-cloud-base-containerd native.cpu pool0 4d2h +``` + +And the exposed devices like this: + +```bash +[kli@n4c16-fedora-40-cloud-base-containerd ~]# kubectl get resourceslices -oyaml | less +apiVersion: v1 +items: +- apiVersion: resource.k8s.io/v1 + kind: ResourceSlice + metadata: + creationTimestamp: "2025-06-10T06:01:54Z" + generateName: n4c16-fedora-40-cloud-base-containerd-native.cpu- + generation: 1 + name: n4c16-fedora-40-cloud-base-containerd-native.cpu-jxfkj + ownerReferences: + - apiVersion: v1 + controller: true + kind: Node + name: n4c16-fedora-40-cloud-base-containerd + uid: 90a99f1f-c1ca-4bea-8dbd-3cc821f744b1 + resourceVersion: "871388" + uid: 4639d31f-e508-4b0a-8378-867f6c1c7cb1 + spec: + devices: + - attributes: + cache0ID: + int: 0 + cache1ID: + int: 8 + cache2ID: + int: 16 + cache3ID: + int: 24 + cluster: + int: 0 + core: + int: 0 + coreType: + string: P-core + die: + int: 0 + isolated: + bool: false + localMemory: + int: 0 + package: + int: 0 + name: cpu1 + - attributes: + - attributes: + cache0ID: + int: 1 + cache1ID: + int: 9 + cache2ID: + int: 17 + cache3ID: + int: 24 + cluster: + int: 2 + core: + int: 1 + coreType: + string: E-core + die: + int: 0 + isolated: + bool: false + localMemory: + int: 0 + package: + int: 0 + name: cpu2 + - attributes: + cache0ID: + int: 1 + cache1ID: + int: 9 + cache2ID: + int: 17 + cache3ID: + int: 24 + cluster: + int: 2 + core: +... +``` + +If everything looks fine and you do have CPUs available as DRA devices, you +can test DRA-based CPU allocation with something like this. This allocates +a single P-core for the container. + +```yaml +apiVersion: resource.k8s.io/v1 +kind: ResourceClaimTemplate +metadata: + name: any-cores +spec: + spec: + devices: + requests: + - name: cpu + exactly: + deviceClassName: native.cpu +--- +apiVersion: resource.k8s.io/v1 +kind: ResourceClaimTemplate +metadata: + name: p-cores +spec: + spec: + devices: + requests: + - name: cpu + exactly: + deviceClassName: native.cpu + selectors: + - cel: + expression: device.attributes["native.cpu"].coreType == "P-core" + count: 1 +--- +apiVersion: resource.k8s.io/v1 +kind: ResourceClaimTemplate +metadata: + name: e-cores +spec: + spec: + devices: + requests: + - name: cpu + exactly: + deviceClassName: native.cpu + selectors: + - cel: + expression: device.attributes["native.cpu"].coreType == "E-core" + count: 1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: pcore-test + labels: + app: pod +spec: + containers: + - name: ctr0 + image: busybox + imagePullPolicy: IfNotPresent + args: + - /bin/sh + - -c + - trap 'exit 0' TERM; sleep 3600 & wait + resources: + requests: + cpu: 1 + memory: 100M + limits: + cpu: 1 + memory: 100M + claims: + - name: claim-pcores + resourceClaims: + - name: claim-pcores + resourceClaimTemplateName: p-cores + terminationGracePeriodSeconds: 1 +``` + +If you want to try a mixed native CPU + DRA-based allocation, try +increasing the CPU request and limit in the pods spec to 1500m CPUs +or CPUs and see what happens. + + +## Playing Around with CPU Abstractions + +If you want to play around with this (for instance modify the exposed CPU abstraction), the easiest way is to +1. [fork](https://github.com/containers/nri-plugins/fork) the [main NRI Reference Plugins](https://github.com/containers/nri-plugins) repo +2. enable github actions in your personal fork +3. make any changes you want (for instance, to alter the CPU abstraction, take a look at [cpu.DRA()](https://github.com/klihub/nri-plugins/blob/test/build/dra-driver/pkg/sysfs/dra.go) +4. Push your changes to ssh://git@github.com/$YOUR_FORK/nri-plugins/refs/heads/test/build/dra-driver. +5. Wait for the image and Helm chart publishing actions to succeed +6. Once done, you can pull the result in to your cluster with something like `helm install --devel -n kube-system test oci://ghcr.io/$YOUR_GITHUB_USERID/nri-plugins/helm-charts/nri-resource-policy-topology-aware --version v0.9-dra-driver-unstable` diff --git a/README.md b/README.md index d152ef7cb..7c63ba263 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,6 @@ Currently following plugins are available: [5]: https://containers.github.io/nri-plugins/stable/docs/memory/sgx-epc.html See the [NRI plugins documentation](https://containers.github.io/nri-plugins/) for more information. + +See the [DRA CPU driver prototype notes](README-DRA-driver-proto.md) for more information +about using the Topology Aware policy as a DRA CPU driver. diff --git a/docs/conf.py b/docs/conf.py index f93341bd0..ba73c1e09 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -127,7 +127,7 @@ def gomod_versions(modules): # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['build', 'build-aux', '_build', '.github', '_work', 'generate', 'README.md', 'TODO.md', 'SECURITY.md', 'CODE-OF-CONDUCT.md', 'docs/releases', 'test/e2e/README.md', 'docs/resource-policy/releases', 'docs/resource-policy/README.md','test/statistics-analysis/README.md', 'deployment/helm/*/*.md', '**/testdata'] +exclude_patterns = ['build', 'build-aux', '_build', '.github', '_work', 'generate', 'README.md', 'TODO.md', 'SECURITY.md', 'CODE-OF-CONDUCT.md', 'docs/releases', 'test/e2e/README.md', 'docs/resource-policy/releases', 'docs/resource-policy/README.md','test/statistics-analysis/README.md', 'deployment/helm/*/*.md', '**/testdata', 'README-DRA-driver-proto.md'] # -- Options for HTML output -------------------------------------------------