diff --git a/gremlin/Chart.yaml b/gremlin/Chart.yaml index 65e4e37..be142c2 100644 --- a/gremlin/Chart.yaml +++ b/gremlin/Chart.yaml @@ -1,5 +1,5 @@ name: gremlin -version: 0.26.0 +version: 0.27.0 description: The Gremlin Inc client application apiVersion: v1 home: https://www.gremlin.com diff --git a/gremlin/README.md b/gremlin/README.md index f9b786b..f408aac 100644 --- a/gremlin/README.md +++ b/gremlin/README.md @@ -74,6 +74,11 @@ their default values. See values.yaml for all available options. | `gremlin.proxy.url` | Specifies the http proxy the agent should use to communicate with api.gremlin.com. | `""` (ignored) | | | `gremlin.extraEnv` | Specify any arbitrary environment variables to pass to the Gremlin Agent daemonset. | `[]` | | `gremlin.features.discoverDestinationService.enabled` | Enable discovery of a destination service in a service mesh to resolve hostnames | `false` | +| `gremlin.gpu.enabled` | Expose host GPU/OpenCL drivers to the agent for the GPU attack | `false` | +| `gremlin.gpu.cdiDevice` | CDI device to inject via pod annotation (for CDI-based runtimes) | `""` | +| `gremlin.gpu.projectOpenclIcd` | Project a vendor's OpenCL ICD registry file into the container | `true` | +| `gremlin.gpu.vendors` | Vendor blocks to target; one DaemonSet is created per entry | `[nvidia, amd]` | +| `gremlin.gpu.` | Per-vendor config block: `nodeSelector`, `runtimeClassName`, `env`, `volumes`, `volumeMounts`, `openclIcd` | see `values.yaml` | | `ssl.certFile` | Add a certificate file to Gremlin's set of certificate authorities. This argument expects a file containing the certificate(s) you wish to add. When set, this chart creates secret (`ssl-cert-file`) with the contents and passes it to both agents. This value is ignored when blank or absent. | `""` (ignored) | | `ssl.certDir` | sets the SSL_CERT_DIR environment variable on the both agents. Unlike ssl.certFile, this value accepts only a path to an existing directory on the Kubernetes nodes. This value is ignored when blank or absent. | `""` (ignored) | @@ -213,6 +218,39 @@ helm install gremlin gremlin/gremlin \ --set-file ssl.certFile=$HOME/Workspace/proxy/ca.pem ``` +### With GPU Support + +To let the GPU attack enumerate and target GPUs, enable `gremlin.gpu` and list the vendors your cluster has in `gremlin.gpu.vendors` (both `nvidia` and `amd` by default). + +```shell +helm install gremlin gremlin/gremlin \ + --namespace gremlin \ + --set gremlin.secret.managed=true \ + --set gremlin.secret.teamID=$GREMLIN_TEAM_ID \ + --set gremlin.secret.clusterID=$GREMLIN_CLUSTER_ID \ + --set-file gremlin.secret.certificate=/path/to/gremlin.cert \ + --set-file gremlin.secret.key=/path/to/gremlin.key \ + --set gremlin.gpu.enabled=true \ + --set gremlin.gpu.vendors={nvidia} +``` + +_note_: The `nvidia` preset runs the agent under the `nvidia` RuntimeClass. If the Gremlin pod fails to start (for example, a `RuntimeClass not found` error), the RuntimeClass likely doesn't exist on your cluster. This chart does not create RuntimeClass objects. Ensure the RuntimeClass named by the vendor block exists (it is normally provided by the NVIDIA GPU Operator or your platform), or set the vendor's `runtimeClassName` to `""`. + +#### One DaemonSet per vendor + +Only some nodes have GPUs, and different nodes may have different GPU vendors, so a single GPU DaemonSet cannot run cluster-wide: nodes lacking the vendor's RuntimeClass or device mounts would fail to start the Gremlin pod. So whenever `gremlin.gpu.enabled` is set, the chart renders: + +- one GPU DaemonSet per entry in `gremlin.gpu.vendors` (`-gremlin-gpu-`), scheduled via node affinity onto that vendor's nodes and carrying that vendor's GPU configuration, and +- one DaemonSet (`-gremlin-gpu-none`) for every node that belongs to none of those vendors. + +Each vendor's DaemonSet is rendered whether or not the cluster currently has nodes of that vendor; a vendor with no matching nodes simply schedules no pods. Trim `gremlin.gpu.vendors` to the vendors you care about to avoid the extra DaemonSets. + +Scheduling uses each vendor block's `nodeSelector` (node labels such as `nvidia.com/gpu.present` / `amd.com/gpu.present`, set by the NVIDIA GPU Operator / Node Feature Discovery and the AMD GPU labeller): its DaemonSet requires those labels, and the `gpu-none` DaemonSet requires their absence. GPU nodes must therefore carry the vendor's `nodeSelector` labels for its DaemonSet to schedule. Any `affinity` you set is preserved — the vendor requirement is ANDed into it. + +This chart does not create RuntimeClass objects; any RuntimeClass named by a vendor block must already exist on the cluster. + +_note_: When GPU support is disabled the agent DaemonSet keeps its original `-gremlin` name. Enabling GPU support replaces it with the per-vendor DaemonSets above (including `-gremlin-gpu-none`), so Helm deletes the old DaemonSet and its pods are recreated by the new ones. The per-vendor DaemonSets also carry an extra `gremlin.com/gpu` pod-selector label so each only manages its own pods. + ## Uninstallation ```shell diff --git a/gremlin/templates/_daemonset.tpl b/gremlin/templates/_daemonset.tpl new file mode 100644 index 0000000..00b52b8 --- /dev/null +++ b/gremlin/templates/_daemonset.tpl @@ -0,0 +1,299 @@ +{{/* +gremlin.daemonset renders one Gremlin DaemonSet from an already-resolved context. Deciding how many +DaemonSets exist and what goes in each is the caller's job (see daemonset.yaml); this template only +renders what it is handed. + +Context: dict + "root" $ # the chart root context + "name" # DaemonSet name + "selectorLabels" # extra selector/pod labels, so DaemonSets sharing a cluster do not + # fight over each other's pods (optional; default none) + "gpu" # a gremlin.gpu. config block (optional; default no GPU config) + "affinity" # rendered affinity (optional; defaults to .Values.affinity) +*/}} +{{- define "gremlin.daemonset" -}} +{{- $root := .root -}} +{{- $name := .name -}} +{{- $selectorLabels := default (dict) .selectorLabels -}} +{{- $gpu := default (dict) .gpu -}} +{{- $affinity := .affinity -}} +{{- if and (not $affinity) $root.Values.affinity -}}{{- $affinity = toYaml $root.Values.affinity -}}{{- end -}} +{{- /* CDI device: global gremlin.gpu.cdiDevice, optionally overridden per vendor block */ -}} +{{- $cdiDevice := "" -}} +{{- with $gpu -}}{{- $cdiDevice = default $root.Values.gremlin.gpu.cdiDevice .cdiDevice -}}{{- end -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "gremlin.name" $root }} + helm.sh/chart: {{ include "gremlin.chart" $root }} + app.kubernetes.io/instance: {{ $root.Release.Name }} + app.kubernetes.io/managed-by: {{ $root.Release.Service }} + version: v1 + {{- with $selectorLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if $root.Values.gremlin.podLabels }} + {{- toYaml $root.Values.gremlin.podLabels | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "gremlin.name" $root }} + {{- with $selectorLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if $root.Values.gremlin.updateStrategy }} + updateStrategy: + {{- toYaml $root.Values.gremlin.updateStrategy | nindent 4 }} + {{- end }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "gremlin.name" $root }} + helm.sh/chart: {{ include "gremlin.chart" $root }} + app.kubernetes.io/instance: {{ $root.Release.Name }} + app.kubernetes.io/managed-by: {{ $root.Release.Service }} + version: v1 + {{- with $selectorLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if $root.Values.gremlin.podLabels }} + {{- toYaml $root.Values.gremlin.podLabels | nindent 8 }} + {{- end }} + {{- if or $root.Values.gremlin.apparmor $root.Values.gremlin.installApparmorProfile $root.Values.gremlin.podSecurity.seccomp.enabled $root.Values.gremlin.podSecurity.securityContextConstraints.create $root.Values.gremlin.podAnnotations $cdiDevice }} + annotations: + {{- if $root.Values.gremlin.apparmor }} + container.apparmor.security.beta.kubernetes.io/{{ $root.Chart.Name }}: {{ $root.Values.gremlin.apparmor }} + {{- else if $root.Values.gremlin.installApparmorProfile }} + container.apparmor.security.beta.kubernetes.io/{{ $root.Chart.Name }}: {{ "localhost/gremlin-agent" }} + {{- end }} + {{- if $root.Values.gremlin.podSecurity.seccomp.enabled }} + container.seccomp.security.alpha.kubernetes.io/{{ $root.Chart.Name }}: {{ $root.Values.gremlin.podSecurity.seccomp.profile }} + {{- end }} + {{- if $root.Values.gremlin.podSecurity.securityContextConstraints.create }} + openshift.io/required-scc: "gremlin" + {{- end }} + {{- if $cdiDevice }} + cdi.k8s.io/gremlin-gpu: {{ $cdiDevice | quote }} + {{- end }} + {{- if $root.Values.gremlin.podAnnotations }} + {{- toYaml $root.Values.gremlin.podAnnotations | nindent 8 }} + {{- end }} + {{- end }} + spec: + serviceAccountName: gremlin + {{- with $gpu.runtimeClassName }} + runtimeClassName: {{ . }} + {{- end }} + {{- with $affinity }} + affinity: {{ . | trimSuffix "\n" | nindent 8 }} + {{- end }} + {{- if $root.Values.nodeSelector }} + nodeSelector: {{ toYaml $root.Values.nodeSelector | trimSuffix "\n" | nindent 8 }} + {{- end }} + {{- if $root.Values.tolerations }} + tolerations: {{ toYaml $root.Values.tolerations | trimSuffix "\n" | nindent 8 }} + {{- end }} + dnsPolicy: {{ $root.Values.gremlin.dnsPolicy }} + hostPID: {{ $root.Values.gremlin.hostPID }} + hostNetwork: {{ $root.Values.gremlin.hostNetwork }} + {{- if $root.Values.image.pullSecret }} + imagePullSecrets: + - name: {{ $root.Values.image.pullSecret }} + {{- end }} + {{- if and $root.Values.gremlin.podSecurity.seccomp.enabled (eq "localhost/gremlin" $root.Values.gremlin.podSecurity.seccomp.profile) }} + initContainers: + - name: seccomp-init + image: {{ $root.Values.image.repository }}:{{ $root.Values.image.tag }} + imagePullPolicy: {{ $root.Values.image.pullPolicy }} + volumeMounts: + - mountPath: {{ $root.Values.gremlin.podSecurity.seccomp.root }} + name: seccomp-root + - mountPath: /gremlin + name: seccomp-profile + command: + - cp + - /gremlin/seccomp.json + - {{ $root.Values.gremlin.podSecurity.seccomp.root }}/gremlin + {{- end }} + containers: + - name: {{ $root.Chart.Name }} + image: {{ $root.Values.image.repository }}:{{ $root.Values.image.tag }} + args: [ "daemon" ] + imagePullPolicy: {{ $root.Values.image.pullPolicy }} + {{- if $root.Values.gremlin.resources }} + resources: {{ toYaml $root.Values.gremlin.resources | nindent 10 }} + {{- end }} + securityContext: + privileged: {{ $root.Values.gremlin.podSecurity.privileged }} + allowPrivilegeEscalation: {{ $root.Values.gremlin.podSecurity.allowPrivilegeEscalation }} + capabilities: + add: {{ toYaml $root.Values.gremlin.podSecurity.capabilities | nindent 14 }} + {{- if $root.Values.gremlin.podSecurity.seLinuxOptions }} + seLinuxOptions: {{ toYaml $root.Values.gremlin.podSecurity.seLinuxOptions | nindent 12 }} + {{- end }} + readOnlyRootFilesystem: {{ $root.Values.gremlin.podSecurity.readOnlyRootFilesystem }} + env: + - name: GREMLIN_TEAM_ID + {{- /* If we aren't managing this secret and a teamID was supplied, assume teamID is not in the external secret */}} + {{- if (and (not $root.Values.gremlin.secret.managed) (default $root.Values.gremlin.teamID $root.Values.gremlin.secret.teamID)) }} + value: {{ default $root.Values.gremlin.teamID $root.Values.gremlin.secret.teamID | quote }} + {{- else }} + valueFrom: + secretKeyRef: + name: {{ include "gremlin.secretName" $root }} + key: GREMLIN_TEAM_ID + {{- end }} + + {{- if (eq (include "gremlin.secretType" $root) "secret") }} + - name: GREMLIN_TEAM_SECRET + valueFrom: + secretKeyRef: + name: {{ include "gremlin.secretName" $root }} + key: GREMLIN_TEAM_SECRET + {{- else }} + - name: GREMLIN_TEAM_CERTIFICATE_OR_FILE + {{- /* If managed outside of this chart, or if the value is a literal, reference the secret as a file */}} + {{- if or (not $root.Values.gremlin.secret.managed) (hasPrefix "-----BEGIN" $root.Values.gremlin.secret.certificate) }} + value: file:///var/lib/gremlin/cert/gremlin.cert + {{- else }} + value: {{ $root.Values.gremlin.secret.certificate }} + {{- end }} + - name: GREMLIN_TEAM_PRIVATE_KEY_OR_FILE + {{- /* If managed outside of this chart, or if the value is a literal, reference the secret as a file */}} + {{- if or (not $root.Values.gremlin.secret.managed) (hasPrefix "-----BEGIN" $root.Values.gremlin.secret.certificate) }} + value: file:///var/lib/gremlin/cert/gremlin.key + {{- else }} + value: {{ $root.Values.gremlin.secret.key }} + {{- end }} + {{- end }} + - name: GREMLIN_IDENTIFIER + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: GREMLIN_CLIENT_TAGS + value: {{ $root.Values.gremlin.client.tags }} + - name: GREMLIN_COLLECT_DNS + value: {{ $root.Values.gremlin.collect.dns | quote }} + - name: GREMLIN_SERVICE_URL + value: {{ include "gremlinServiceUrl" $root }} + {{- if not $root.Values.gremlin.features.pushCIDRTags.enabled }} + - name: GREMLIN_PUSH_POD_CIDR_TAGS + value: "false" + - name: GREMLIN_PUSH_ZONE_CIDR_TAGS + value: "false" + {{- end }} + {{- if $root.Values.gremlin.proxy.url }} + - name: https_proxy + value: {{ $root.Values.gremlin.proxy.url }} + {{- end }} + {{- if $root.Values.ssl.certFile }} + - name: SSL_CERT_FILE + value: /etc/gremlin/ssl/certfile.pem + {{- end }} + {{- if $root.Values.ssl.certDir }} + - name: SSL_CERT_DIR + value: {{ $root.Values.ssl.certDir }} + {{- end }} + {{- if include "gremlinTlsIdentityEnv" $root }} + {{- include "gremlinTlsIdentityEnv" $root | nindent 10 }} + {{- end }} + {{- with $gpu.env }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $root.Values.gremlin.extraEnv }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + - name: gremlin-state + mountPath: /var/lib/gremlin + readOnly: false + - name: gremlin-executions + mountPath: /var/lib/gremlin/executions + readOnly: false + - name: gremlin-logs + mountPath: /var/log/gremlin + readOnly: false + - name: cgroup-root + mountPath: /sys/fs/cgroup + readOnly: false + {{- if include "containerMounts" $root }} + {{- include "containerMounts" $root | nindent 10 }} + {{- end }} + {{- if (eq (include "gremlin.secretType" $root) "certificate") }} + - name: gremlin-cert + mountPath: /var/lib/gremlin/cert + readOnly: true + {{- end }} + {{- if $root.Values.ssl.certFile }} + - name: ssl-cert-file + mountPath: /etc/gremlin/ssl + readOnly: true + {{- end }} + {{- if include "gremlinTlsIdentityVolumeMounts" $root }} + {{- include "gremlinTlsIdentityVolumeMounts" $root | nindent 10 }} + {{- end }} + {{- with $gpu.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if include "gremlinGpuOpenclIcdActiveFromSpec" (dict "root" $root "gpu" $gpu) }} + - name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/{{ $gpu.openclIcd.filename }} + subPath: {{ $gpu.openclIcd.filename }} + readOnly: true + {{- end }} + volumes: + - name: cgroup-root + hostPath: + path: {{ $root.Values.gremlin.cgroup.root }} + {{- if include "containerVolumes" $root }} + {{- include "containerVolumes" $root | nindent 8 }} + {{- end }} + # The Gremlin daemon communicates with Gremlin sidecars via its state directory. + - name: gremlin-state + emptyDir: + medium: Memory + - name: gremlin-executions + hostPath: + path: /var/lib/gremlin/executions + # The Gremlin daemon forwards logs from the Gremlin sidecars to the Gremlin control plane + # These logs should be shared with the host + - name: gremlin-logs + hostPath: + path: /var/log/gremlin + {{- if (eq (include "gremlin.secretType" $root) "certificate") }} + - name: gremlin-cert + secret: + secretName: {{ include "gremlin.secretName" $root }} + {{- end }} + {{- if and $root.Values.gremlin.podSecurity.seccomp.enabled (eq "localhost/gremlin" $root.Values.gremlin.podSecurity.seccomp.profile) }} + - name: seccomp-root + hostPath: + path: {{ $root.Values.gremlin.podSecurity.seccomp.root }} + - name: seccomp-profile + configMap: + name: {{ template "gremlin.fullname" $root }}-seccomp + {{- end }} + {{- if $root.Values.ssl.certFile }} + - name: ssl-cert-file + secret: + secretName: ssl-cert-file + {{- end }} + {{- if include "gremlinTlsIdentityVolumes" $root }} + {{- include "gremlinTlsIdentityVolumes" $root | nindent 8 }} + {{- end }} + {{- with $gpu.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if include "gremlinGpuOpenclIcdActiveFromSpec" (dict "root" $root "gpu" $gpu) }} + - name: gremlin-opencl-icd + configMap: + name: {{ include "gremlin.fullname" $root }}-opencl-icd + {{- end }} +{{- if $root.Values.gremlin.priorityClassName }} + priorityClassName: {{ $root.Values.gremlin.priorityClassName }} +{{- end }} +{{- end -}} diff --git a/gremlin/templates/_helpers.tpl b/gremlin/templates/_helpers.tpl index 5aebf27..f088fa7 100644 --- a/gremlin/templates/_helpers.tpl +++ b/gremlin/templates/_helpers.tpl @@ -244,6 +244,49 @@ When createSecret or existingSecret are configured {{- end -}} {{- end -}} +{{/* +gremlinGpuOpenclIcdActiveFromSpec returns "true" when the OpenCL ICD file should be projected: +gremlin.gpu.projectOpenclIcd is set and the vendor block defines openclIcd (filename + library). +Context: dict "root" $ "gpu" +*/}} +{{- define "gremlinGpuOpenclIcdActiveFromSpec" -}} +{{- $icd := .gpu.openclIcd -}} +{{- if and .root.Values.gremlin.gpu.projectOpenclIcd $icd $icd.filename $icd.library -}}true{{- end -}} +{{- end -}} + +{{/* +gremlinGpuNodeAffinity returns the user's .Values.affinity (as YAML) with an added requirement that a +node carry ("In") or lack ("NotIn") the nodeSelector labels of the given vendor blocks. The +requirement is ANDed into every nodeSelectorTerm the user supplied (or a new term when they supplied +none), so GPU node targeting and a user's affinity are both honored. Renders nothing when those +vendor blocks define no nodeSelector, rather than an empty (API-invalid) nodeSelectorTerm. +Context: dict "root" $ "vendors" "operator" "In" | "NotIn" +*/}} +{{- define "gremlinGpuNodeAffinity" -}} +{{- $root := .root -}} +{{- $operator := .operator -}} +{{- $exprs := list -}} +{{- range $vendor := .vendors -}} +{{- range $k, $v := (default (dict) (index $root.Values.gremlin.gpu $vendor)).nodeSelector -}} +{{- $exprs = append $exprs (dict "key" $k "operator" $operator "values" (list (toString $v))) -}} +{{- end -}} +{{- end -}} +{{- if $exprs -}} +{{- $affinity := deepCopy (default (dict) $root.Values.affinity) -}} +{{- $nodeAffinity := default (dict) $affinity.nodeAffinity -}} +{{- $required := default (dict) $nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution -}} +{{- /* the terms are dicts, so `set` updates them in place inside $terms */ -}} +{{- $terms := default (list (dict)) $required.nodeSelectorTerms -}} +{{- range $term := $terms -}} +{{- $_ := set $term "matchExpressions" (concat (default (list) $term.matchExpressions) $exprs) -}} +{{- end -}} +{{- $_ := set $required "nodeSelectorTerms" $terms -}} +{{- $_ := set $nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" $required -}} +{{- $_ := set $affinity "nodeAffinity" $nodeAffinity -}} +{{- $affinity | toYaml -}} +{{- end -}} +{{- end -}} + {{/* chaoTlsIdentityArgs returns the chao cli arguments needed to configure TLS client identity When remoteSecret is configured diff --git a/gremlin/templates/daemonset.yaml b/gremlin/templates/daemonset.yaml index 6a3de46..c1def39 100644 --- a/gremlin/templates/daemonset.yaml +++ b/gremlin/templates/daemonset.yaml @@ -1,240 +1,30 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ include "gremlin.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "gremlin.name" . }} - helm.sh/chart: {{ include "gremlin.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - version: v1 - {{- if .Values.gremlin.podLabels }} - {{- toYaml .Values.gremlin.podLabels | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - app.kubernetes.io/name: {{ include "gremlin.name" . }} - {{- if .Values.gremlin.updateStrategy }} - updateStrategy: - {{- toYaml .Values.gremlin.updateStrategy | nindent 4 }} - {{- end }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ include "gremlin.name" . }} - helm.sh/chart: {{ include "gremlin.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - version: v1 - {{- if .Values.gremlin.podLabels }} - {{- toYaml .Values.gremlin.podLabels | nindent 8 }} - {{- end }} - {{- if or .Values.gremlin.apparmor .Values.gremlin.installApparmorProfile .Values.gremlin.podSecurity.seccomp.enabled .Values.gremlin.podSecurity.securityContextConstraints.create .Values.gremlin.podAnnotations }} - annotations: - {{- if .Values.gremlin.apparmor }} - container.apparmor.security.beta.kubernetes.io/{{ .Chart.Name }}: {{ .Values.gremlin.apparmor }} - {{- else if .Values.gremlin.installApparmorProfile }} - container.apparmor.security.beta.kubernetes.io/{{ .Chart.Name }}: {{ "localhost/gremlin-agent" }} - {{- end }} - {{- if .Values.gremlin.podSecurity.seccomp.enabled }} - container.seccomp.security.alpha.kubernetes.io/{{ .Chart.Name }}: {{ .Values.gremlin.podSecurity.seccomp.profile }} - {{- end }} - {{- if .Values.gremlin.podSecurity.securityContextConstraints.create }} - openshift.io/required-scc: "gremlin" - {{- end }} - {{- if .Values.gremlin.podAnnotations }} - {{- toYaml .Values.gremlin.podAnnotations | nindent 8 }} - {{- end }} - {{- end }} - spec: - serviceAccountName: gremlin - {{- if .Values.affinity }} - affinity: {{ toYaml .Values.affinity | trimSuffix "\n" | nindent 8 }} - {{- end }} - {{- if .Values.nodeSelector }} - nodeSelector: {{ toYaml .Values.nodeSelector | trimSuffix "\n" | nindent 8 }} - {{- end }} - {{- if .Values.tolerations }} - tolerations: {{ toYaml .Values.tolerations | trimSuffix "\n" | nindent 8 }} - {{- end }} - dnsPolicy: {{ .Values.gremlin.dnsPolicy }} - hostPID: {{ .Values.gremlin.hostPID }} - hostNetwork: {{ .Values.gremlin.hostNetwork }} - {{- if .Values.image.pullSecret }} - imagePullSecrets: - - name: {{ .Values.image.pullSecret }} - {{- end }} - {{- if and .Values.gremlin.podSecurity.seccomp.enabled (eq "localhost/gremlin" .Values.gremlin.podSecurity.seccomp.profile) }} - initContainers: - - name: seccomp-init - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - volumeMounts: - - mountPath: {{ .Values.gremlin.podSecurity.seccomp.root }} - name: seccomp-root - - mountPath: /gremlin - name: seccomp-profile - command: - - cp - - /gremlin/seccomp.json - - {{ .Values.gremlin.podSecurity.seccomp.root }}/gremlin - {{- end }} - containers: - - name: {{ .Chart.Name }} - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - args: [ "daemon" ] - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if .Values.gremlin.resources }} - resources: {{ toYaml .Values.gremlin.resources | nindent 10 }} - {{- end }} - securityContext: - privileged: {{ .Values.gremlin.podSecurity.privileged }} - allowPrivilegeEscalation: {{ .Values.gremlin.podSecurity.allowPrivilegeEscalation }} - capabilities: - add: {{ toYaml .Values.gremlin.podSecurity.capabilities | nindent 14 }} - {{- if .Values.gremlin.podSecurity.seLinuxOptions }} - seLinuxOptions: {{ toYaml .Values.gremlin.podSecurity.seLinuxOptions | nindent 12 }} - {{- end }} - readOnlyRootFilesystem: {{ .Values.gremlin.podSecurity.readOnlyRootFilesystem }} - env: - - name: GREMLIN_TEAM_ID - {{- /* If we aren't managing this secret and a teamID was supplied, assume teamID is not in the external secret */}} - {{- if (and (not .Values.gremlin.secret.managed) (default .Values.gremlin.teamID .Values.gremlin.secret.teamID)) }} - value: {{ default .Values.gremlin.teamID .Values.gremlin.secret.teamID | quote }} - {{- else }} - valueFrom: - secretKeyRef: - name: {{ include "gremlin.secretName" . }} - key: GREMLIN_TEAM_ID - {{- end }} +{{- /* +Decides which Gremlin DaemonSet(s) exist and resolves each one's name, pod-selector labels, GPU +vendor config and node affinity. The DaemonSet body itself is rendered by the "gremlin.daemonset" +template (see _daemonset.tpl), which knows nothing about GPUs or node mixes. - {{- if (eq (include "gremlin.secretType" .) "secret") }} - - name: GREMLIN_TEAM_SECRET - valueFrom: - secretKeyRef: - name: {{ include "gremlin.secretName" . }} - key: GREMLIN_TEAM_SECRET - {{- else }} - - name: GREMLIN_TEAM_CERTIFICATE_OR_FILE - {{- /* If managed outside of this chart, or if the value is a literal, reference the secret as a file */}} - {{- if or (not .Values.gremlin.secret.managed) (hasPrefix "-----BEGIN" .Values.gremlin.secret.certificate) }} - value: file:///var/lib/gremlin/cert/gremlin.cert - {{- else }} - value: {{ .Values.gremlin.secret.certificate }} - {{- end }} - - name: GREMLIN_TEAM_PRIVATE_KEY_OR_FILE - {{- /* If managed outside of this chart, or if the value is a literal, reference the secret as a file */}} - {{- if or (not .Values.gremlin.secret.managed) (hasPrefix "-----BEGIN" .Values.gremlin.secret.certificate) }} - value: file:///var/lib/gremlin/cert/gremlin.key - {{- else }} - value: {{ .Values.gremlin.secret.key }} - {{- end }} - {{- end }} - - name: GREMLIN_IDENTIFIER - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: GREMLIN_CLIENT_TAGS - value: {{ .Values.gremlin.client.tags }} - - name: GREMLIN_COLLECT_DNS - value: {{ .Values.gremlin.collect.dns | quote }} - - name: GREMLIN_SERVICE_URL - value: {{ include "gremlinServiceUrl" . }} - {{- if not .Values.gremlin.features.pushCIDRTags.enabled }} - - name: GREMLIN_PUSH_POD_CIDR_TAGS - value: "false" - - name: GREMLIN_PUSH_ZONE_CIDR_TAGS - value: "false" - {{- end }} - {{- if .Values.gremlin.proxy.url }} - - name: https_proxy - value: {{ .Values.gremlin.proxy.url }} - {{- end }} - {{- if .Values.ssl.certFile }} - - name: SSL_CERT_FILE - value: /etc/gremlin/ssl/certfile.pem - {{- end }} - {{- if .Values.ssl.certDir }} - - name: SSL_CERT_DIR - value: {{ .Values.ssl.certDir }} - {{- end }} - {{- if include "gremlinTlsIdentityEnv" . }} - {{- include "gremlinTlsIdentityEnv" . | nindent 10 }} - {{- end }} - {{- with .Values.gremlin.extraEnv }} - {{- toYaml . | nindent 10 }} - {{- end }} - volumeMounts: - - name: gremlin-state - mountPath: /var/lib/gremlin - readOnly: false - - name: gremlin-executions - mountPath: /var/lib/gremlin/executions - readOnly: false - - name: gremlin-logs - mountPath: /var/log/gremlin - readOnly: false - - name: cgroup-root - mountPath: /sys/fs/cgroup - readOnly: false - {{- if include "containerMounts" . }} - {{- include "containerMounts" . | nindent 10 }} - {{- end }} - {{- if (eq (include "gremlin.secretType" .) "certificate") }} - - name: gremlin-cert - mountPath: /var/lib/gremlin/cert - readOnly: true - {{- end }} - {{- if .Values.ssl.certFile }} - - name: ssl-cert-file - mountPath: /etc/gremlin/ssl - readOnly: true - {{- end }} - {{- if include "gremlinTlsIdentityVolumeMounts" . }} - {{- include "gremlinTlsIdentityVolumeMounts" . | nindent 10 }} - {{- end }} - volumes: - - name: cgroup-root - hostPath: - path: {{ .Values.gremlin.cgroup.root }} - {{- if include "containerVolumes" . }} - {{- include "containerVolumes" . | nindent 8 }} - {{- end }} - # The Gremlin daemon communicates with Gremlin sidecars via its state directory. - - name: gremlin-state - emptyDir: - medium: Memory - - name: gremlin-executions - hostPath: - path: /var/lib/gremlin/executions - # The Gremlin daemon forwards logs from the Gremlin sidecars to the Gremlin control plane - # These logs should be shared with the host - - name: gremlin-logs - hostPath: - path: /var/log/gremlin - {{- if (eq (include "gremlin.secretType" .) "certificate") }} - - name: gremlin-cert - secret: - secretName: {{ include "gremlin.secretName" . }} - {{- end }} - {{- if and .Values.gremlin.podSecurity.seccomp.enabled (eq "localhost/gremlin" .Values.gremlin.podSecurity.seccomp.profile) }} - - name: seccomp-root - hostPath: - path: {{ .Values.gremlin.podSecurity.seccomp.root }} - - name: seccomp-profile - configMap: - name: {{ template "gremlin.fullname" . }}-seccomp - {{- end }} - {{- if .Values.ssl.certFile }} - - name: ssl-cert-file - secret: - secretName: ssl-cert-file - {{- end }} - {{- if include "gremlinTlsIdentityVolumes" . }} - {{- include "gremlinTlsIdentityVolumes" . | nindent 8 }} - {{- end }} -{{- if .Values.gremlin.priorityClassName }} - priorityClassName: {{ .Values.gremlin.priorityClassName }} -{{- end }} + - GPU disabled -> a single DaemonSet, ``. + - GPU enabled -> one DaemonSet per gremlin.gpu.vendors entry, `-gpu-`, pinned to + that vendor's nodes and carrying that vendor's GPU config, plus + `-gpu-none` for every node that belongs to none of those vendors. +*/ -}} +{{- $fullname := include "gremlin.fullname" $ -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $vendors := default (list) .Values.gremlin.gpu.vendors -}} +{{- range $vendor := $vendors }} +{{- include "gremlin.daemonset" (dict + "root" $ + "name" (printf "%s-gpu-%s" $fullname $vendor) + "selectorLabels" (dict "gremlin.com/gpu" $vendor) + "gpu" (default (dict) (index $.Values.gremlin.gpu $vendor)) + "affinity" (include "gremlinGpuNodeAffinity" (dict "root" $ "vendors" (list $vendor) "operator" "In"))) }} +--- +{{ end }} +{{- include "gremlin.daemonset" (dict + "root" $ + "name" (printf "%s-gpu-none" $fullname) + "selectorLabels" (dict "gremlin.com/gpu" "none") + "affinity" (include "gremlinGpuNodeAffinity" (dict "root" $ "vendors" $vendors "operator" "NotIn"))) }} +{{- else -}} +{{- include "gremlin.daemonset" (dict "root" $ "name" $fullname) }} +{{- end -}} diff --git a/gremlin/templates/opencl-icd-configmap.yaml b/gremlin/templates/opencl-icd-configmap.yaml new file mode 100644 index 0000000..5e42a6b --- /dev/null +++ b/gremlin/templates/opencl-icd-configmap.yaml @@ -0,0 +1,39 @@ +{{- /* +Projects an OpenCL ICD registry file into the Gremlin container(s). + +Some NVIDIA container runtimes inject the OpenCL driver library (libnvidia-opencl.so.1) but do +NOT create the /etc/OpenCL/vendors/.icd registry file that the ICD loader reads to +discover the driver. Without it, the OpenCL platforms cannot be listed. + +One ConfigMap holds an entry per gremlin.gpu.vendors block that needs an ICD file (see +gremlinGpuOpenclIcdActiveFromSpec). Each vendor DaemonSet subPath-mounts its own entry from this +shared ConfigMap. The ICD filename and library come from the vendor block's `openclIcd` field in +values.yaml (gremlin.gpu..openclIcd). +*/ -}} +{{- $root := . -}} +{{- $entries := dict -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- range $vendor := (default (list) .Values.gremlin.gpu.vendors) -}} +{{- $spec := default (dict) (index $root.Values.gremlin.gpu $vendor) -}} +{{- if include "gremlinGpuOpenclIcdActiveFromSpec" (dict "root" $root "gpu" $spec) -}} +{{- $_ := set $entries $spec.openclIcd.filename $spec.openclIcd.library -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- if $entries }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "gremlin.fullname" . }}-opencl-icd + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ include "gremlin.chart" . }} + app.kubernetes.io/name: {{ include "gremlin.fullname" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: +{{- range $filename, $library := $entries }} + {{ $filename }}: | + {{ $library }} +{{- end }} +{{- end }} diff --git a/gremlin/tests/daemonset_gpu_test.yaml b/gremlin/tests/daemonset_gpu_test.yaml new file mode 100644 index 0000000..7624622 --- /dev/null +++ b/gremlin/tests/daemonset_gpu_test.yaml @@ -0,0 +1,457 @@ +suite: Test GPU / OpenCL access options +templates: + - daemonset.yaml +release: + name: my-release + namespace: my-namespace + revision: 1 + upgrade: true +tests: + # --- GPU disabled: one DaemonSet, keeping the original name ------------------------------ + - it: should render a single DaemonSet with no GPU configuration by default + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release-gremlin + - notExists: + path: metadata.labels["gremlin.com/gpu"] + - notExists: + path: spec.template.spec.runtimeClassName + - notContains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors + type: DirectoryOrCreate + + - it: should not add GPU configuration when vendors are set but gpu is disabled + set: + gremlin.gpu.enabled: false + gremlin.gpu.vendors: + - nvidia + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release-gremlin + - notExists: + path: spec.template.spec.runtimeClassName + - notContains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_VISIBLE_DEVICES + value: "all" + + # --- GPU enabled: one DaemonSet per vendor, plus gpu-none -------------------------------- + - it: should render one DaemonSet per vendor plus a gpu-none DaemonSet when enabled + set: + gremlin.gpu.enabled: true + asserts: + - hasDocuments: + count: 3 + - equal: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + documentIndex: 0 + - equal: + path: metadata.name + value: my-release-gremlin-gpu-amd + documentIndex: 1 + - equal: + path: metadata.name + value: my-release-gremlin-gpu-none + documentIndex: 2 + + - it: should render a DaemonSet per vendor even when only one vendor is listed + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendors: + - nvidia + asserts: + - hasDocuments: + count: 2 + - equal: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + documentIndex: 0 + - equal: + path: metadata.name + value: my-release-gremlin-gpu-none + documentIndex: 1 + + - it: should render only the gpu-none DaemonSet when no vendors are listed + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendors: [] + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release-gremlin-gpu-none + # with no vendor labels to exclude there is nothing to add to the user's affinity + - notExists: + path: spec.template.spec.affinity + + # --- Each DaemonSet carries its own vendor's configuration ------------------------------- + - it: should apply the nvidia block to the nvidia DaemonSet + set: + gremlin.gpu.enabled: true + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + asserts: + - equal: + path: metadata.labels["gremlin.com/gpu"] + value: nvidia + - equal: + path: spec.selector.matchLabels["gremlin.com/gpu"] + value: nvidia + - equal: + path: spec.template.spec.runtimeClassName + value: nvidia + # scheduled onto the nvidia nodes + - contains: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions + content: + key: nvidia.com/gpu.present + operator: In + values: + - "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_VISIBLE_DEVICES + value: "all" + - contains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_DRIVER_CAPABILITIES + value: "all" + # nvidia relies on the container toolkit, so no GPU hostPath volumes are added + - notContains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors + type: DirectoryOrCreate + # the OpenCL ICD registry file is projected so the injected driver is discoverable + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/nvidia.icd + subPath: nvidia.icd + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: gremlin-opencl-icd + configMap: + name: my-release-gremlin-opencl-icd + + - it: should apply the amd block to the amd DaemonSet + set: + gremlin.gpu.enabled: true + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-amd + asserts: + - equal: + path: metadata.labels["gremlin.com/gpu"] + value: amd + # amd does not use a RuntimeClass + - notExists: + path: spec.template.spec.runtimeClassName + # scheduled onto the amd nodes + - contains: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions + content: + key: amd.com/gpu.present + operator: In + values: + - "true" + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: kfd + mountPath: /dev/kfd + readOnly: false + - contains: + path: spec.template.spec.volumes + content: + name: kfd + hostPath: + path: /dev/kfd + type: CharDevice + - contains: + path: spec.template.spec.volumes + content: + name: dri + hostPath: + path: /dev/dri + type: Directory + - contains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors + type: DirectoryOrCreate + # amd already mounts /etc/OpenCL/vendors from the host and defines no openclIcd, so the + # chart does not project its own ICD file there + - notContains: + path: spec.template.spec.volumes + content: + name: gremlin-opencl-icd + configMap: + name: my-release-gremlin-opencl-icd + + - it: should keep the gpu-none DaemonSet off every vendor's nodes and free of GPU config + set: + gremlin.gpu.enabled: true + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-none + asserts: + - equal: + path: metadata.labels["gremlin.com/gpu"] + value: none + - equal: + path: spec.selector.matchLabels["gremlin.com/gpu"] + value: none + - notExists: + path: spec.template.spec.runtimeClassName + - notContains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_VISIBLE_DEVICES + value: "all" + - notContains: + path: spec.template.spec.volumes + content: + name: kfd + hostPath: + path: /dev/kfd + type: CharDevice + - contains: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions + content: + key: nvidia.com/gpu.present + operator: NotIn + values: + - "true" + - contains: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions + content: + key: amd.com/gpu.present + operator: NotIn + values: + - "true" + + - it: should AND the vendor node requirement into a user-supplied affinity + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendors: + - nvidia + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + asserts: + - equal: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms + value: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: nvidia.com/gpu.present + operator: In + values: + - "true" + + # --- Editing / overriding a vendor block ------------------------------------------------- + - it: should honor a runtimeClassName set on the vendor block + set: + gremlin.gpu.enabled: true + gremlin.gpu.nvidia.runtimeClassName: nvidia-custom + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + asserts: + - equal: + path: spec.template.spec.runtimeClassName + value: nvidia-custom + + - it: should honor env set on the vendor block + set: + gremlin.gpu.enabled: true + gremlin.gpu.nvidia.env: + - name: NVIDIA_VISIBLE_DEVICES + value: "0,1" + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_VISIBLE_DEVICES + value: "0,1" + # the block's env is the whole list, so the default NVIDIA_DRIVER_CAPABILITIES is gone + - notContains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_DRIVER_CAPABILITIES + value: "all" + + - it: should render native volumes and volumeMounts from a custom vendor block + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendors: + - custom + gremlin.gpu.custom.nodeSelector: + my.com/gpu.present: "true" + gremlin.gpu.custom.volumes: + - name: nvidia0 + hostPath: + path: /dev/nvidia0 + type: CharDevice + gremlin.gpu.custom.volumeMounts: + - name: nvidia0 + mountPath: /dev/nvidia0 + readOnly: false + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-custom + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nvidia0 + mountPath: /dev/nvidia0 + readOnly: false + - contains: + path: spec.template.spec.volumes + content: + name: nvidia0 + hostPath: + path: /dev/nvidia0 + type: CharDevice + - contains: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions + content: + key: my.com/gpu.present + operator: In + values: + - "true" + + # --- CDI (Container Device Interface) ---------------------------------------------------- + - it: should not add a CDI annotation by default + set: + gremlin.gpu.enabled: true + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + asserts: + - notExists: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + + - it: should add the CDI device annotation to the vendor DaemonSets when cdiDevice is set + set: + gremlin.gpu.enabled: true + gremlin.gpu.cdiDevice: nvidia.com/gpu=all + asserts: + - equal: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + value: nvidia.com/gpu=all + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + # the non-GPU nodes get no CDI device + - notExists: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-none + + - it: should let a vendor block override the CDI device + set: + gremlin.gpu.enabled: true + gremlin.gpu.cdiDevice: nvidia.com/gpu=all + gremlin.gpu.nvidia.cdiDevice: nvidia.com/gpu=0 + asserts: + - equal: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + value: nvidia.com/gpu=0 + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + - equal: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + value: nvidia.com/gpu=all + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-amd + + - it: should not add a CDI annotation when gpu is disabled + set: + gremlin.gpu.enabled: false + gremlin.gpu.cdiDevice: nvidia.com/gpu=all + asserts: + - notExists: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + + # --- ICD projection toggle --------------------------------------------------------------- + - it: should not project the OpenCL ICD when projectOpenclIcd is false + set: + gremlin.gpu.enabled: true + gremlin.gpu.projectOpenclIcd: false + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + asserts: + - notContains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/nvidia.icd + subPath: nvidia.icd + readOnly: true + - notContains: + path: spec.template.spec.volumes + content: + name: gremlin-opencl-icd + configMap: + name: my-release-gremlin-opencl-icd + + - it: should not project the OpenCL ICD when the vendor block defines none + set: + gremlin.gpu.enabled: true + gremlin.gpu.nvidia.openclIcd: null + documentSelector: + path: metadata.name + value: my-release-gremlin-gpu-nvidia + asserts: + - notContains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/nvidia.icd + subPath: nvidia.icd + readOnly: true diff --git a/gremlin/tests/opencl_icd_configmap_test.yaml b/gremlin/tests/opencl_icd_configmap_test.yaml new file mode 100644 index 0000000..9598ed7 --- /dev/null +++ b/gremlin/tests/opencl_icd_configmap_test.yaml @@ -0,0 +1,57 @@ +suite: Test OpenCL ICD registry ConfigMap +templates: + - opencl-icd-configmap.yaml +release: + name: my-release + namespace: my-namespace + revision: 1 + upgrade: true +tests: + - it: should not render when gpu is disabled + asserts: + - hasDocuments: + count: 0 + + - it: should render the nvidia ICD ConfigMap when gpu is enabled + set: + gremlin.gpu.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: my-release-gremlin-opencl-icd + - equal: + path: data["nvidia.icd"] + value: | + libnvidia-opencl.so.1 + - matchRegex: + path: data["nvidia.icd"] + pattern: libnvidia-opencl\.so\.1 + + - it: should not render when no vendor in use defines an ICD + set: + gremlin.gpu.enabled: true + gremlin.gpu.nvidia.openclIcd: null + asserts: + - hasDocuments: + count: 0 + + - it: should not render when projectOpenclIcd is false + set: + gremlin.gpu.enabled: true + gremlin.gpu.projectOpenclIcd: false + asserts: + - hasDocuments: + count: 0 + + - it: should not render for the amd preset alone (host registry is mounted instead) + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendors: + - amd + asserts: + - hasDocuments: + count: 0 diff --git a/gremlin/values.yaml b/gremlin/values.yaml index c6f7cf0..9fcfb6e 100644 --- a/gremlin/values.yaml +++ b/gremlin/values.yaml @@ -243,6 +243,129 @@ gremlin: # Attacks targeting other Kubernetes pods will use the resource specification of their target. resources: {} + # gremlin.gpu - + # Exposes the host's GPU / OpenCL drivers to the Gremlin agent so that the GPU attack can + # enumerate and target GPUs from inside the container. + # + # The easy path is to set `enabled: true` and list the vendors your cluster has in `vendors` + # (e.g. "nvidia" or "amd"). Each vendor's full configuration lives in a same-named block below. + # To customize, edit that vendor's block, or copy it to `custom` and add "custom" to `vendors`. + gpu: + # gremlin.gpu.enabled - + # When true, the chart renders one DaemonSet per `vendors` entry, each carrying that vendor's GPU + # configuration, plus a `-gpu-none` DaemonSet for the cluster's non-GPU nodes. When false, none + # of the GPU settings have any effect and a single DaemonSet is rendered. + enabled: false + + # NOTE: when a selected vendor sets runtimeClassName, that RuntimeClass must already exist on the + # cluster (typically created by the GPU Operator or your platform). This chart does NOT create + # RuntimeClass objects; a missing RuntimeClass will cause the Gremlin pod to fail to start. + + # gremlin.gpu.cdiDevice - + # For nodes using the Nvidia Container Device Interface (CDI) rather than the Nvidia container-toolkit + # runtime hook. When set, the chart adds a `cdi.k8s.io/gremlin-gpu: ` pod annotation + # so the CRI injects the named CDI device(s). Use this when NVIDIA_VISIBLE_DEVICES is NOT + # honored by your runtime (pure Kubernetes-native CDI). A vendor block may override this with + # its own `cdiDevice`. + # + # Requires containerd >= 1.7 or CRI-O with CDI enabled. + cdiDevice: "" + + # gremlin.gpu.projectOpenclIcd - + # Whether to project a vendor's `openclIcd` file (e.g. /etc/OpenCL/vendors/nvidia.icd) into the + # container, when that vendor block defines one. Defaults to true, which fixes runtimes + # that inject the OpenCL driver library but not the ICD file. Set to false when your CDI spec + # (or another mechanism) already creates the ICD file, to avoid a conflicting mount. + projectOpenclIcd: true + + # gremlin.gpu.vendors - + # The vendor blocks to target, which handles a hybrid / mixed cluster where only some nodes have + # GPUs and different nodes may have different vendors. Whenever GPU access is enabled the chart + # renders one DaemonSet per vendor listed here, scheduled only onto that vendor's nodes and + # carrying that vendor's GPU config, plus one `-gpu-none` DaemonSet for every remaining node, so + # each node runs exactly one agent matching its capabilities. Trim this list to the vendors your + # cluster actually has. + # + # Each name must reference a vendor block below defining a `nodeSelector`: those labels are what + # pin the vendor's DaemonSet to its nodes (and keep it off the `-gpu-none` DaemonSet's nodes), so + # GPU nodes must carry them (the NVIDIA GPU Operator / Node Feature Discovery and the AMD GPU + # labeller set them). + vendors: + - nvidia + - amd + + # --- Vendor configuration blocks -------------------------------------------------------- + # Each block is the full, native-shaped GPU configuration for one vendor. Fields: + # nodeSelector - node label(s) that identify AND schedule this vendor's nodes + # runtimeClassName - the RuntimeClass the Gremlin pods run under ("" for none) + # env - environment variables added to the Gremlin container (native env entries). + # For the NVIDIA container toolkit the capabilities value MUST include + # `compute` (or `all`) for OpenCL. + # volumes - pod volumes added to the Gremlin DaemonSet (native volume entries) + # volumeMounts - container volumeMounts added to the Gremlin container (native mount entries) + # openclIcd - optional {filename, library}. When set (and projectOpenclIcd is true) the + # chart projects an /etc/OpenCL/vendors/ file containing , + # for runtimes that inject the driver library but not its ICD registry file. + # cdiDevice - optional per-vendor override of gremlin.gpu.cdiDevice + # + # NOTE: GPU device nodes (/dev/nvidia*, /dev/kfd, /dev/dri) require the container to have device + # access. This generally means running privileged (gremlin.podSecurity.privileged: true) or + # otherwise granting the pod cgroup device permissions. Device access is not required when the + # NVIDIA container toolkit injects the devices for you. + + nvidia: + # Relies on the NVIDIA container toolkit: run under the "nvidia" RuntimeClass and set + # NVIDIA_VISIBLE_DEVICES / NVIDIA_DRIVER_CAPABILITIES so the runtime injects the driver + # libraries and device nodes (no host volumes required). Projects the nvidia OpenCL ICD + # registry file so the injected driver is discoverable. + nodeSelector: + nvidia.com/gpu.present: "true" + runtimeClassName: nvidia + env: + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_DRIVER_CAPABILITIES + value: "all" + volumes: [] + volumeMounts: [] + openclIcd: + filename: nvidia.icd + library: libnvidia-opencl.so.1 + + amd: + # Relies on the ROCm/amdgpu driver on the host: mount the /dev/kfd and /dev/dri device nodes + # plus the host's OpenCL ICD registry (/etc/OpenCL/vendors) so the container reaches the GPUs + # directly. + nodeSelector: + amd.com/gpu.present: "true" + runtimeClassName: "" + env: [] + volumes: + - name: kfd + hostPath: { path: /dev/kfd, type: CharDevice } + - name: dri + hostPath: { path: /dev/dri, type: Directory } + - name: opencl-vendors + hostPath: { path: /etc/OpenCL/vendors, type: DirectoryOrCreate } + volumeMounts: + - name: kfd + mountPath: /dev/kfd + readOnly: false + - name: dri + mountPath: /dev/dri + readOnly: false + - name: opencl-vendors + mountPath: /etc/OpenCL/vendors + readOnly: true + + custom: + # A blank block to configure a GPU vendor by hand. Add "custom" to gremlin.gpu.vendors, give it + # a nodeSelector, and fill in the fields. + runtimeClassName: "" + env: [] + volumes: [] + volumeMounts: [] + secret: # Gremlin supports both `certificate` and `secret` types # To manage secrets with helm, set `managed=true` and fill in either the certificate auth or secret auth sections