From 785228275571720762064a9c4f6101b66636188d Mon Sep 17 00:00:00 2001 From: Arnaud Lemaire Date: Sun, 20 Apr 2025 00:35:31 +0200 Subject: [PATCH] remove default affinity and toleration --- base/ci/advanced-result.yaml | 11 -- base/ci/basic-result.yaml | 11 -- base/ci/inferenceService-result.yaml | 11 -- base/ci/multiple-workloads-main-result.yaml | 22 ---- base/ci/multiple-workloads-result.yaml | 22 ---- base/ci/test-result.yaml | 11 -- .../features/workloads/_scheduling_infra.tpl | 36 +----- .../workloads/_scheduling_infra_test.yaml | 110 ------------------ .../pods/fields/field_affinity_test.yaml | 6 - .../pods/fields/field_tolerations_test.yaml | 7 +- 10 files changed, 2 insertions(+), 245 deletions(-) diff --git a/base/ci/advanced-result.yaml b/base/ci/advanced-result.yaml index 3baff61..e3675f0 100644 --- a/base/ci/advanced-result.yaml +++ b/base/ci/advanced-result.yaml @@ -133,12 +133,6 @@ spec: runAsNonRoot: true dnsPolicy: ClusterFirst affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: @@ -171,11 +165,6 @@ spec: maxSkew: 2 topologyKey: kubernetes.io/hostname whenUnsatisfiable: DoNotSchedule - tolerations: - - effect: NoSchedule - key: cloud.blabla.io/k8s-volatile - operator: Equal - value: "true" containers: - env: - name: HTTPS_PORT diff --git a/base/ci/basic-result.yaml b/base/ci/basic-result.yaml index 516e19c..7daf838 100644 --- a/base/ci/basic-result.yaml +++ b/base/ci/basic-result.yaml @@ -196,12 +196,6 @@ spec: runAsNonRoot: true dnsPolicy: ClusterFirst affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: @@ -234,11 +228,6 @@ spec: maxSkew: 2 topologyKey: kubernetes.io/hostname whenUnsatisfiable: DoNotSchedule - tolerations: - - effect: NoSchedule - key: cloud.blabla.io/k8s-volatile - operator: Equal - value: "true" containers: - env: - name: DD_SERVICE diff --git a/base/ci/inferenceService-result.yaml b/base/ci/inferenceService-result.yaml index 32fa545..b63fc05 100644 --- a/base/ci/inferenceService-result.yaml +++ b/base/ci/inferenceService-result.yaml @@ -86,12 +86,6 @@ spec: runAsUser: 1000 dnsPolicy: ClusterFirst affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: @@ -124,11 +118,6 @@ spec: maxSkew: 2 topologyKey: kubernetes.io/hostname whenUnsatisfiable: DoNotSchedule - tolerations: - - effect: NoSchedule - key: cloud.blabla.io/k8s-volatile - operator: Equal - value: "true" volumes: - hostPath: path: /var/run/datadog diff --git a/base/ci/multiple-workloads-main-result.yaml b/base/ci/multiple-workloads-main-result.yaml index 581c461..458911e 100644 --- a/base/ci/multiple-workloads-main-result.yaml +++ b/base/ci/multiple-workloads-main-result.yaml @@ -193,12 +193,6 @@ spec: runAsNonRoot: true dnsPolicy: ClusterFirst affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: @@ -231,11 +225,6 @@ spec: maxSkew: 2 topologyKey: kubernetes.io/hostname whenUnsatisfiable: DoNotSchedule - tolerations: - - effect: NoSchedule - key: cloud.blabla.io/k8s-volatile - operator: Equal - value: "true" containers: - image: there:42 imagePullPolicy: IfNotPresent @@ -319,12 +308,6 @@ spec: runAsNonRoot: true dnsPolicy: ClusterFirst affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: @@ -357,11 +340,6 @@ spec: maxSkew: 2 topologyKey: kubernetes.io/hostname whenUnsatisfiable: DoNotSchedule - tolerations: - - effect: NoSchedule - key: cloud.blabla.io/k8s-volatile - operator: Equal - value: "true" containers: - image: test:43 imagePullPolicy: IfNotPresent diff --git a/base/ci/multiple-workloads-result.yaml b/base/ci/multiple-workloads-result.yaml index 481d44f..e1cd8f2 100644 --- a/base/ci/multiple-workloads-result.yaml +++ b/base/ci/multiple-workloads-result.yaml @@ -193,12 +193,6 @@ spec: runAsNonRoot: true dnsPolicy: ClusterFirst affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: @@ -231,11 +225,6 @@ spec: maxSkew: 2 topologyKey: kubernetes.io/hostname whenUnsatisfiable: DoNotSchedule - tolerations: - - effect: NoSchedule - key: cloud.blabla.io/k8s-volatile - operator: Equal - value: "true" containers: - image: test:43 imagePullPolicy: IfNotPresent @@ -319,12 +308,6 @@ spec: runAsNonRoot: true dnsPolicy: ClusterFirst affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: @@ -357,11 +340,6 @@ spec: maxSkew: 2 topologyKey: kubernetes.io/hostname whenUnsatisfiable: DoNotSchedule - tolerations: - - effect: NoSchedule - key: cloud.blabla.io/k8s-volatile - operator: Equal - value: "true" containers: - image: there:42 imagePullPolicy: IfNotPresent diff --git a/base/ci/test-result.yaml b/base/ci/test-result.yaml index 2c327b0..efe0e5b 100644 --- a/base/ci/test-result.yaml +++ b/base/ci/test-result.yaml @@ -117,12 +117,6 @@ spec: runAsNonRoot: true dnsPolicy: ClusterFirst affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: @@ -155,11 +149,6 @@ spec: maxSkew: 2 topologyKey: kubernetes.io/hostname whenUnsatisfiable: DoNotSchedule - tolerations: - - effect: NoSchedule - key: cloud.blabla.io/k8s-volatile - operator: Equal - value: "true" containers: - image: ghcr.io/mendhak/http-https-echo:31 imagePullPolicy: IfNotPresent diff --git a/base/templates/features/workloads/_scheduling_infra.tpl b/base/templates/features/workloads/_scheduling_infra.tpl index d69b3e9..8523872 100644 --- a/base/templates/features/workloads/_scheduling_infra.tpl +++ b/base/templates/features/workloads/_scheduling_infra.tpl @@ -5,32 +5,12 @@ {{- if include "base.lib.utils.isEnabled" (dig "scheduling" dict $workload ) -}} -{{- $nodepools := (list "common" "bursty" "volatile") }} -{{- if not (has $workload.scheduling.nodepool $nodepools) }} - {{- fail (printf "workload scheduling must be one of %v(workload: %s)" $nodepools $workloadId) }} -{{- end }} resources: controllers: {{$workloadId}}: pod: - {{- if or (eq $workload.scheduling.nodepool "volatile") (eq $workload.scheduling.nodepool "bursty") (include "base.lib.utils.isEnabled" $workload.scheduling.podSpreadingForHA) }} + {{- if include "base.lib.utils.isEnabled" $workload.scheduling.podSpreadingForHA }} affinity: - {{- if or (eq $workload.scheduling.nodepool "volatile") (eq $workload.scheduling.nodepool "bursty") }} - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - {{- if eq $workload.scheduling.nodepool "volatile" }} - - key: cloud.blabla.io/k8s-volatile - operator: Exists - {{- else if eq $workload.scheduling.nodepool "bursty" }} - - key: dedicated - operator: In - values: - - bursty - {{- end }} - {{- end }} - {{- if include "base.lib.utils.isEnabled" $workload.scheduling.podSpreadingForHA }} podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -68,19 +48,5 @@ resources: {{- include "base.lib.metadata.selectorLabels" $rootContext | nindent 14 }} {{- end }} - {{- if or (eq $workload.scheduling.nodepool "volatile") (eq $workload.scheduling.nodepool "bursty") }} - tolerations: - - effect: NoSchedule - operator: Equal - {{- if eq $workload.scheduling.nodepool "volatile" }} - key: cloud.blabla.io/k8s-volatile - value: "true" - {{- else if eq $workload.scheduling.nodepool "bursty" }} - key: dedicated - value: "bursty" - {{- end }} - {{- end }} - - {{- end -}} {{- end }} diff --git a/base/templates/features/workloads/_scheduling_infra_test.yaml b/base/templates/features/workloads/_scheduling_infra_test.yaml index fd6d745..787716d 100644 --- a/base/templates/features/workloads/_scheduling_infra_test.yaml +++ b/base/templates/features/workloads/_scheduling_infra_test.yaml @@ -72,113 +72,3 @@ tests: path: spec.template.spec.topologySpreadConstraints - notExists: path: spec.template.spec.affinity.podAntiAffinity - - - it: nodepool is mandatory - set: - workloads.main.scheduling.nodepool: "" - asserts: - - failedTemplate: - errorMessage: "workload scheduling must be one of [common bursty volatile](workload: main)" - - - it: nodepool must be in enum - set: - workloads.main.scheduling.nodepool: toto - asserts: - - failedTemplate: - errorMessage: "workload scheduling must be one of [common bursty volatile](workload: main)" - - - it: default nodepool is volatile - documentSelector: - path: kind - value: Deployment - asserts: - - equal: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution - value: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists - - equal: - path: spec.template.spec.tolerations[0] - value: - effect: NoSchedule - key: cloud.blabla.io/k8s-volatile - operator: Equal - value: "true" - - - it: nodepool can be common - set: - workloads.main.scheduling.nodepool: common - asserts: - - notExists: - path: spec.template.spec.affinity.nodeAffinity - - notExists: - path: spec.template.spec.tolerations - - - it: nodepool can be bursty - set: - workloads.main.scheduling.nodepool: bursty - documentSelector: - path: kind - value: Deployment - asserts: - - equal: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution - value: - nodeSelectorTerms: - - matchExpressions: - - key: dedicated - operator: In - values: - - bursty - - equal: - path: spec.template.spec.tolerations[0] - value: - effect: NoSchedule - operator: Equal - key: dedicated - value: "bursty" - - - it: common nodepool with spread disabled - set: - workloads.main.scheduling: - nodepool: common - podSpreadingForHA: - enabled: false - documentSelector: - path: kind - value: Deployment - asserts: - - notExists: - path: spec.template.spec.affinity - - notExists: - path: spec.template.spec.tolerations - - notExists: - path: spec.template.spec.topologySpreadConstraints - - - it: override toleration from workload is working - set: - resources.controllers.main.pod.tolerations: - - effect: NoSchedule - key: test - operator: Equal - value: "true" - documentSelector: - path: kind - value: Deployment - asserts: - - equal: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution - value: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists - - equal: - path: spec.template.spec.tolerations[0] - value: - effect: NoSchedule - key: test - operator: Equal - value: "true" \ No newline at end of file diff --git a/base/templates/resources/pods/fields/field_affinity_test.yaml b/base/templates/resources/pods/fields/field_affinity_test.yaml index 45cf1b3..a51ce49 100644 --- a/base/templates/resources/pods/fields/field_affinity_test.yaml +++ b/base/templates/resources/pods/fields/field_affinity_test.yaml @@ -14,12 +14,6 @@ tests: - equal: path: spec.template.spec.affinity value: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.blabla.io/k8s-volatile - operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: diff --git a/base/templates/resources/pods/fields/field_tolerations_test.yaml b/base/templates/resources/pods/fields/field_tolerations_test.yaml index ceece1c..e51110b 100644 --- a/base/templates/resources/pods/fields/field_tolerations_test.yaml +++ b/base/templates/resources/pods/fields/field_tolerations_test.yaml @@ -11,13 +11,8 @@ tests: path: kind value: Deployment asserts: - - equal: + - isNull: path: spec.template.spec.tolerations - value: - - effect: NoSchedule - key: cloud.blabla.io/k8s-volatile - operator: Equal - value: "true" - it: resourcesDefault should pass set: