Experiment/reservation race pending

cmd/binder/app/app.go

@@ -114,6 +114,10 @@ func New(options *Options, config *rest.Config) (*App, error) {
 		options.ResourceReservationPodResources.Value,
 		options.ResourceReservationPodSecurityContext.Value,
 		options.ResourceReservationContainerSecurityContext.Value)
+	if options.ReservationSchedulerName != "" {
+		rrs = rrs.WithExternalReservation(options.ReservationSchedulerName,
+			options.ReservationGpuResourceName, options.ReservationPodAnnotations)
+	}
 
 	reconcilerParams := &controllers.ReconcilerParams{
 		MaxConcurrentReconciles:     options.MaxConcurrentReconciles,
@@ -174,6 +178,27 @@ func (app *App) Run(ctx context.Context) error {
 		setupLog.Error(err, "unable to create controller", "controller", "BindRequest")
 		return err
 	}
+
+	if err = (&controllers.ReserveAheadReconciler{
+		Client:              app.manager.GetClient(),
+		Scheme:              app.manager.GetScheme(),
+		ResourceReservation: app.rrs,
+		SchedulerName:       app.Options.SchedulerName,
+		Enabled:             app.Options.ReservationSchedulerName != "",
+	}).SetupWithManager(app.manager, app.reconcilerParams); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "ReserveAhead")
+		return err
+	}
+
+	if err = (&controllers.ReservationCascadeReconciler{
+		Client:               app.manager.GetClient(),
+		Scheme:               app.manager.GetScheme(),
+		ReservationNamespace: app.Options.ResourceReservationNamespace,
+		Enabled:              app.Options.ReservationSchedulerName != "",
+	}).SetupWithManager(app.manager, app.reconcilerParams); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "ReservationCascade")
+		return err
+	}
 	// +kubebuilder:scaffold:builder
 
 	setupLog.Info("starting manager")

cmd/binder/app/options.go

@@ -36,6 +36,9 @@ type Options struct {
 	FakeGPUNodes                                bool
 	Plugins                                     flags.JSONFlag[binderplugins.Config]
 	RuntimeClassName                            string
+	ReservationSchedulerName                    string
+	ReservationGpuResourceName                  string
+	ReservationPodAnnotations                   map[string]string
 }
 
 func InitOptions(fs *pflag.FlagSet) *Options {
@@ -109,6 +112,19 @@ func InitOptions(fs *pflag.FlagSet) *Options {
 	fs.StringVar(&options.RuntimeClassName,
 		"runtime-class-name", "",
 		"Runtime class for GPU reservation pods. Defaults to empty (no runtime class).")
+	fs.StringVar(&options.ReservationSchedulerName,
+		"reservation-scheduler-name", "",
+		"When set, reservation pods are created unbound with this spec.schedulerName so an "+
+			"external scheduler (e.g. slurm-bridge-scheduler) places them, instead of being "+
+			"pre-bound to the scheduler-selected node. Empty keeps the default pinned behavior.")
+	fs.StringVar(&options.ReservationGpuResourceName,
+		"reservation-gpu-resource-name", "",
+		"Resource key the reservation pod requests for a whole GPU in external mode. "+
+			"Empty defaults to nvidia.com/gpu; e.g. deviceclass.resource.kubernetes.io/gpu.nvidia.com.")
+	fs.StringToStringVar(&options.ReservationPodAnnotations,
+		"reservation-pod-annotations", nil,
+		"Extra annotations to stamp on externally-scheduled reservation pods (k=v,k2=v2), "+
+			"e.g. the slurmjob.slinky.slurm.net/{qos,exclusive} annotations slurm-bridge requires.")
 
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
 

deployments/kai-scheduler/crds/kai.scheduler_configs.yaml

@@ -4048,6 +4048,30 @@ spec:
                       If enabled, this prevents pods of each microservice from being scheduled on the same node.
                       If another podAntiAffinity term is defined (either globally or locally for a specific microservice), this will be ignored.
                     type: boolean
+                  reservationGpuResourceName:
+                    description: |-
+                      ReservationGpuResourceName overrides the resource key the reservation pod requests for a
+                      whole GPU (only used when ReservationSchedulerName is set). Empty defaults to
+                      "nvidia.com/gpu"; set it to "deviceclass.resource.kubernetes.io/gpu.nvidia.com" to request
+                      a DRA-backed extended resource that slurm-bridge accepts and Slurm accounts.
+                    type: string
+                  reservationPodAnnotations:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      ReservationPodAnnotations are extra annotations stamped onto externally-scheduled
+                      reservation pods (only used when ReservationSchedulerName is set), e.g. the
+                      slurmjob.slinky.slurm.net/{qos,exclusive} annotations slurm-bridge requires to build a
+                      Slurm job. Ignored in the default pinned mode.
+                    type: object
+                  reservationSchedulerName:
+                    description: |-
+                      ReservationSchedulerName, when set, makes the binder create GPU-sharing reservation
+                      pods with this `spec.schedulerName` (instead of pre-binding them via `spec.nodeName`),
+                      so an external scheduler (e.g. slurm-bridge-scheduler) places the whole-GPU reservation
+                      pod and KAI only sub-divides fractions onto it. Empty/nil preserves the default behavior
+                      where the binder pins the reservation pod to the scheduler-selected node.
+                    type: string
                   schedulerName:
                     description: |-
                       SchedulerName specifies the name of the KAI scheduler. Pods must set this value

deployments/kai-scheduler/templates/_helpers.tpl

@@ -72,6 +72,16 @@ spec:
     {{- if .Values.global.jsonLog }}
     jsonLog: true
     {{- end }}
+    {{- if .Values.global.reservationSchedulerName }}
+    reservationSchedulerName: {{ .Values.global.reservationSchedulerName | quote }}
+    {{- end }}
+    {{- if .Values.global.reservationGpuResourceName }}
+    reservationGpuResourceName: {{ .Values.global.reservationGpuResourceName | quote }}
+    {{- end }}
+    {{- if .Values.global.reservationPodAnnotations }}
+    reservationPodAnnotations:
+      {{- toYaml .Values.global.reservationPodAnnotations | nindent 6 }}
+    {{- end }}
     {{- if .Values.global.affinity }}
     affinity:
       {{- toYaml .Values.global.affinity | nindent 6 }}

pkg/apis/kai/v1/global.go

@@ -74,6 +74,28 @@ type GlobalConfig struct {
 	// +kubebuilder:validation:Optional
 	SchedulerName *string `json:"schedulerName,omitempty"`
 
+	// ReservationSchedulerName, when set, makes the binder create GPU-sharing reservation
+	// pods with this `spec.schedulerName` (instead of pre-binding them via `spec.nodeName`),
+	// so an external scheduler (e.g. slurm-bridge-scheduler) places the whole-GPU reservation
+	// pod and KAI only sub-divides fractions onto it. Empty/nil preserves the default behavior
+	// where the binder pins the reservation pod to the scheduler-selected node.
+	// +kubebuilder:validation:Optional
+	ReservationSchedulerName *string `json:"reservationSchedulerName,omitempty"`
+
+	// ReservationPodAnnotations are extra annotations stamped onto externally-scheduled
+	// reservation pods (only used when ReservationSchedulerName is set), e.g. the
+	// slurmjob.slinky.slurm.net/{qos,exclusive} annotations slurm-bridge requires to build a
+	// Slurm job. Ignored in the default pinned mode.
+	// +kubebuilder:validation:Optional
+	ReservationPodAnnotations map[string]string `json:"reservationPodAnnotations,omitempty"`
+
+	// ReservationGpuResourceName overrides the resource key the reservation pod requests for a
+	// whole GPU (only used when ReservationSchedulerName is set). Empty defaults to
+	// "nvidia.com/gpu"; set it to "deviceclass.resource.kubernetes.io/gpu.nvidia.com" to request
+	// a DRA-backed extended resource that slurm-bridge accepts and Slurm accounts.
+	// +kubebuilder:validation:Optional
+	ReservationGpuResourceName *string `json:"reservationGpuResourceName,omitempty"`
+
 	// NodePoolLabelKey is the label name by with to filter nodes, pods and other resources that the scheduler is watching
 	// +kubebuilder:validation:Optional
 	NodePoolLabelKey *string `json:"nodePoolLabelKey,omitempty"`