fix(infiniband): make accelerator discovery pluggable

.claude/CLAUDE.md

@@ -18,13 +18,16 @@ Topograph discovers the physical network topology of a cluster (NVLink domains,
 
 Providers differ by environment. The canonical `topology.Graph` is stable. Engines only translate — they do not discover.
 
+Within a provider, network-fabric and accelerator-domain discovery may be composed independently through `pkg/accelerator`; the provider remains responsible for combining both dimensions into the canonical graph.
+
 This separation is load-bearing. If you find yourself reading the fabric in an engine, or emitting scheduler-specific output from a provider, stop and reconsider.
 
 ### Repository map
 
 ```
 cmd/                  # Entry points: topograph, node-observer, node-data-broker, kwok-nodes
 pkg/
+  accelerator/        # Pluggable accelerator-domain discovery composed by providers
   providers/          # One directory per provider: aws, gcp, oci, nebius, netq, dra, infiniband, lambdai, test
   engines/            # One directory per engine: k8s, nfd, slinky, slurm
   topology/           # Canonical Graph, Vertex tree, and topology constants (DO NOT CHANGE CASUALLY)

AGENTS.md

@@ -18,13 +18,16 @@ Topograph discovers the physical network topology of a cluster (NVLink domains,
 
 Providers differ by environment. The canonical `topology.Graph` is stable. Engines only translate — they do not discover.
 
+Within a provider, network-fabric and accelerator-domain discovery may be composed independently through `pkg/accelerator`; the provider remains responsible for combining both dimensions into the canonical graph.
+
 This separation is load-bearing. If you find yourself reading the fabric in an engine, or emitting scheduler-specific output from a provider, stop and reconsider.
 
 ### Repository map
 
 ```
 cmd/                  # Entry points: topograph, node-observer, node-data-broker, kwok-nodes
 pkg/
+  accelerator/        # Pluggable accelerator-domain discovery composed by providers
   providers/          # One directory per provider: aws, gcp, oci, nebius, netq, dra, infiniband, lambdai, test
   engines/            # One directory per engine: k8s, nfd, slinky, slurm
   topology/           # Canonical Graph, Vertex tree, and topology constants (DO NOT CHANGE CASUALLY)

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ### Added
 
+- Pluggable accelerator-domain discovery for InfiniBand providers, independently selectable from fabric discovery with `nvidia-smi`, an explicitly configured Kubernetes Node label, or no accelerator source. Discovery is disabled when `accelerator` is omitted or empty; a non-empty section must set `source` explicitly. Helm defaults the `nvidia-smi` workload location to the `gpu-operator` namespace and `nvidia-device-plugin-daemonset` DaemonSet when those values are omitted.
 - Helm `kubeClient.qps` and `kubeClient.burst` values for tuning the DRA provider and the Kubernetes, NFD, and Slinky engine clients through deployment-level `KUBE_QPS` and `KUBE_BURST` settings.
 - The Kubernetes engine now publishes `accelerator.topograph.run/sub-domain` when a provider supplies `InstanceTopology.XclrSubDomainID`.
 - The NFD engine now publishes separate `xclr-domain` and `xclr-sub-domain` attributes and groups.
@@ -26,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ### Changed
 
+- InfiniBand providers now query NVL partition IDs with the `nvidia-smi` CSV query interface, merge identical per-GPU rows, reject unavailable (`N/A`) fields, and normalize the result to `ClusterUUID.CliqueId`.
 - **BREAKING:** The default Kubernetes topology labels now use the vendor-neutral Topograph domains `fabric.topograph.run/tier-N`, `accelerator.topograph.run/domain`, and `accelerator.topograph.run/sub-domain`. Consumers must update topology keys, selectors, allowlists, and scheduling policies to use the new labels.
 - The node-observer now processes its existing topology-generation triggers through a client-go rate-limiting work queue, coalescing event bursts into a single cluster-wide reconciliation while preserving existing trigger and retry behavior.
 - Simulation models now define accelerator topology through inherited `switches[].annotations` and `blocks[].annotations` using `accelerator.topology.test/domain` and optional `accelerator.topology.test/sub-domain`.

charts/topograph/templates/NOTES.txt

@@ -42,6 +42,11 @@
 
   NOTE: node-data-broker applies node annotations once when each broker pod starts.
 {{- end }}
+{{- if eq .Values.provider.name "infiniband-k8s" }}
+
+  NOTE: InfiniBand fabric discovery uses ibnetdiscover; accelerator-domain
+        discovery source is {{ include "nodeDataBroker.acceleratorSource" . }}.
+{{- end }}
 {{- if eq .Values.engine.name "nfd" }}
 
   NOTE: The NFD engine writes NodeFeature and NodeFeatureGroup resources in

charts/topograph/templates/_validation.tpl

@@ -20,6 +20,40 @@
   {{- fail "env.KUBE_BURST is managed by the chart; configure kubeClient.burst instead" }}
 {{- end }}
 
+{{- if or (eq .Values.provider.name "infiniband-k8s") (eq .Values.provider.name "infiniband-bm") }}
+{{- $params := default dict .Values.provider.params }}
+{{- $acceleratorValue := get $params "accelerator" }}
+{{- $accelerator := default dict $acceleratorValue }}
+{{- $source := "none" }}
+{{- if hasKey $params "accelerator" }}
+{{- if and (kindIs "map" $acceleratorValue) (eq (len $acceleratorValue) 0) }}
+{{- $source = "none" }}
+{{- else }}
+{{- $source = lower (toString (get $accelerator "source")) }}
+{{- if eq $source "" }}
+  {{- fail "provider.params.accelerator.source must be set when provider.params.accelerator is present" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- if not (has $source (list "nvidia-smi" "kubernetes-label" "none")) }}
+  {{- fail (printf "unsupported provider.params.accelerator.source %q" $source) }}
+{{- end }}
+
+{{- if and (eq .Values.provider.name "infiniband-k8s") (eq $source "kubernetes-label") }}
+{{- $kubernetesLabel := default dict (get $accelerator "kubernetesLabel") }}
+{{- $key := trim (toString (get $kubernetesLabel "key")) }}
+{{- if eq $key "" }}
+  {{- fail "provider.params.accelerator.kubernetesLabel.key must be set for source kubernetes-label" }}
+{{- end }}
+{{- end }}
+
+{{- if and (eq .Values.provider.name "infiniband-bm") (eq $source "kubernetes-label") }}
+  {{- fail "provider.params.accelerator.source kubernetes-label is not supported by infiniband-bm" }}
+{{- end }}
+
+{{- end }}
+
 {{- if eq .Values.provider.name "gcp" }}
 {{- $params := default dict .Values.provider.params }}
 

charts/topograph/templates/nodeDataBroker/_helpers.tpl

@@ -53,6 +53,50 @@ Create the name of the RBAC resources.
 {{- include "nodeDataBroker.fullname" . }}
 {{- end }}
 
+{{/* Resolve the configured accelerator source. */}}
+{{- define "nodeDataBroker.acceleratorSource" -}}
+{{- $providerParams := default dict .Values.provider.params -}}
+{{- $acceleratorValue := get $providerParams "accelerator" -}}
+{{- $accelerator := default dict $acceleratorValue -}}
+{{- $source := "none" -}}
+{{- if hasKey $providerParams "accelerator" -}}
+{{- if and (kindIs "map" $acceleratorValue) (eq (len $acceleratorValue) 0) -}}
+{{- $source = "none" -}}
+{{- else -}}
+{{- $source = get $accelerator "source" -}}
+{{- if empty $source -}}
+{{- fail "provider.params.accelerator.source must be set when provider.params.accelerator is present" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- lower (toString $source) -}}
+{{- end }}
+
+{{/*
+Render the provider configuration used by node-data-broker. The broker needs
+the GPU Operator workload location when nvidia-smi discovery is enabled, so
+materialize its defaults in the generated configuration while preserving
+explicit overrides.
+*/}}
+{{- define "nodeDataBroker.providerConfig" -}}
+{{- $provider := deepCopy .Values.provider -}}
+{{- if eq (include "nodeDataBroker.acceleratorSource" .) "nvidia-smi" -}}
+{{- $params := default dict (get $provider "params") -}}
+{{- $accelerator := default dict (get $params "accelerator") -}}
+{{- $nvidiaSmi := default dict (get $accelerator "nvidiaSmi") -}}
+{{- if empty (trim (toString (get $nvidiaSmi "gpuOperatorNamespace"))) -}}
+{{- $_ := set $nvidiaSmi "gpuOperatorNamespace" "gpu-operator" -}}
+{{- end -}}
+{{- if empty (trim (toString (get $nvidiaSmi "devicePluginDaemonSet"))) -}}
+{{- $_ := set $nvidiaSmi "devicePluginDaemonSet" "nvidia-device-plugin-daemonset" -}}
+{{- end -}}
+{{- $_ := set $accelerator "nvidiaSmi" $nvidiaSmi -}}
+{{- $_ := set $params "accelerator" $accelerator -}}
+{{- $_ := set $provider "params" $params -}}
+{{- end -}}
+{{- toYaml $provider -}}
+{{- end }}
+
 {{/*
 Create the name of a generated ConfigMap mount.
 */}}

charts/topograph/templates/nodeDataBroker/configmap.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.nodeDataBroker.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "nodeDataBroker.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nodeDataBroker.labels" . | nindent 4 }}
+data:
+  node-data-broker-config.yaml: |-
+    provider:
+      {{- include "nodeDataBroker.providerConfig" . | nindent 6 }}
+    healthzPort: {{ .Values.nodeDataBroker.port }}
+{{- end }}

charts/topograph/templates/nodeDataBroker/daemonset.yaml

@@ -1,6 +1,4 @@
 {{- if .Values.nodeDataBroker.enabled }}
-{{- $providerParams := default dict .Values.provider.params }}
-{{- $useGpuCliqueLabel := and (eq .Values.provider.name "infiniband-k8s") (eq (lower (toString (get $providerParams "useGpuCliqueLabel"))) "true") }}
 {{- $configMapMounts := default list .Values.nodeDataBroker.configMapMounts }}
 apiVersion: apps/v1
 kind: DaemonSet
@@ -15,6 +13,8 @@ spec:
       {{- include "nodeDataBroker.selectorLabels" . | nindent 6 }}
   template:
     metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/nodeDataBroker/configmap.yaml") . | sha256sum }}
       labels:
         {{- include "nodeDataBroker.labels" . | nindent 8 }}
     spec:
@@ -38,15 +38,9 @@ spec:
           command:
             - /usr/local/bin/node-data-broker
           args:
-            - --provider={{ .Values.provider.name }}
+            - -c
+            - /etc/topograph/node-data-broker-config.yaml
             - -v={{ .Values.verbosity }}
-            - --port={{ .Values.nodeDataBroker.port }}
-            {{- if $useGpuCliqueLabel }}
-            - --set=useGpuCliqueLabel=true
-            {{- end }}
-            {{- range .Values.nodeDataBroker.extraArgs }}
-            - --set={{ . }}
-            {{- end }}
           env:
             - name: NODE_NAME
               valueFrom:
@@ -85,8 +79,10 @@ spec:
               port: http
           resources:
             {{- toYaml .Values.nodeDataBroker.resources | nindent 12 }}
-          {{- if or $configMapMounts .Values.nodeDataBroker.volumeMounts }}
           volumeMounts:
+            - name: config-volume
+              mountPath: /etc/topograph
+              readOnly: true
             {{- range $configMapMounts }}
             - name: {{ include "nodeDataBroker.configMapMountVolumeName" (dict "name" .name) }}
               mountPath: {{ required "nodeDataBroker.configMapMounts[].mountPath is required" .mountPath | quote }}
@@ -98,9 +94,11 @@ spec:
             {{- with .Values.nodeDataBroker.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
-          {{- end }}
-      {{- if or $configMapMounts .Values.nodeDataBroker.volumes }}
       volumes:
+        - name: config-volume
+          configMap:
+            defaultMode: 420
+            name: {{ include "nodeDataBroker.fullname" . }}
         {{- range $configMapMounts }}
         - name: {{ include "nodeDataBroker.configMapMountVolumeName" (dict "name" .name) }}
           configMap:
@@ -109,7 +107,6 @@ spec:
         {{- with .Values.nodeDataBroker.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-      {{- end }}
       {{- with .Values.nodeDataBroker.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/topograph/templates/nodeDataBroker/rbac.yaml

@@ -1,6 +1,5 @@
 {{- if and .Values.nodeDataBroker.enabled .Values.nodeDataBroker.rbac.create }}
-{{- $providerParams := default dict .Values.provider.params }}
-{{- $useGpuCliqueLabel := and (eq .Values.provider.name "infiniband-k8s") (eq (lower (toString (get $providerParams "useGpuCliqueLabel"))) "true") }}
+{{- $acceleratorSource := include "nodeDataBroker.acceleratorSource" . }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -9,7 +8,7 @@ rules:
 - apiGroups: [""]
   resources: [nodes]
   verbs: [get,update]
-{{- if and (eq .Values.provider.name "infiniband-k8s") (not $useGpuCliqueLabel) }}
+{{- if and (eq .Values.provider.name "infiniband-k8s") (eq $acceleratorSource "nvidia-smi") }}
 - apiGroups: [apps]
   resources: [daemonsets]
   verbs: [get]