Define node selector and tolerations for prometheus (#625)

Prometheus spec allows for setting node selectors and tolerations

https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml,
but we couldn't override those as the operator spec didn't configure it.

- Added spec and new variables in prometheus-cloud.values.yaml
- Kept default values for the values-cloud.yaml empty
- Corresponding replaces in the cloud/prometheus-operator.yaml

Changed the yaml replace structure to accomodate iterating over
interface objects, open to suggestions for more robustness.

---------

Signed-off-by: Dharini Dutia <dharinidutia@intrinsic.ai>

Author: quarkytale

src/app_charts/prometheus/cloud/prometheus-operator.yaml

@@ -5,4 +5,28 @@
 # prometheus deployment. How could we let them override prometheus-cloud.values.yaml?
 # NOTE: The order here is important. The domain and project might be part of other values and
 #       need to be replaced last.
-{{ .Files.Get "files/prometheus-operator-chart.cloud.yaml" | replace "${CR_GF_SERVER_DOMAIN}" .Values.gf_server_domain | replace "${CR_GF_SERVER_ROOT_URL}" .Values.gf_server_root_url | replace "${CR_GF_CSRF_TRUSTED_ORIGINS}" .Values.gf_csrf_trusted_origins | replace "${CR_GF_SMTP_ENABLED}" .Values.gf_smtp_enabled | replace "${CR_GF_SMTP_HOST}" .Values.gf_smtp_host | replace "${CR_GF_SMTP_USER}" .Values.gf_smtp_user | replace "${CR_GF_SMTP_PASSWORD}" .Values.gf_smtp_password | replace "${CR_GF_SMTP_FROM_ADDRESS}" .Values.gf_smtp_from_address | replace "${CR_GF_SMTP_FROM_NAME}" .Values.gf_smtp_from_name | replace "${CR_GF_SMTP_SKIP_VERIFY}" .Values.gf_smtp_skip_verify | replace "${CR_GF_INGRESS_AUTH_URL}" .Values.gf_ingress_auth_url | replace "${CR_GF_INGRESS_AUTH_SIGNIN}" .Values.gf_ingress_auth_signin | replace "${CR_GF_INGRESS_ERROR_PAGE_403}" .Values.gf_ingress_error_page_403 | replace "${CR_PROM_INGRESS_AUTH_URL}" .Values.prom_ingress_auth_url | replace "${CR_PROM_INGRESS_AUTH_SIGNIN}" .Values.prom_ingress_auth_signin | replace "${CR_PROM_INGRESS_ERROR_PAGE_403}" .Values.prom_ingress_error_page_403 | replace "HELM-NAMESPACE" .Release.Namespace | replace "${LIMITS_MEMORY}" .Values.limits.memory | replace "${LIMITS_CPU}" .Values.limits.cpu | replace "${REQUESTS_STORAGE}" .Values.requests.storage | replace "${RETENTION_TIME}" .Values.retention.time | replace "${RETENTION_SIZE}" .Values.retention.size | replace "${EXTERNAL_URL}" .Values.prom_external_url | replace "${CLOUD_ROBOTICS_DOMAIN}" .Values.domain | replace "${GCP_PROJECT_ID}" .Values.project }}
+{{- $data := .Files.Get "files/prometheus-operator-chart.cloud.yaml" -}}
+# Pre-process variables
+{{- $data = $data | replace "${CR_GF_SERVER_DOMAIN}" .Values.gf_server_domain | replace "${CR_GF_SERVER_ROOT_URL}" .Values.gf_server_root_url | replace "${CR_GF_CSRF_TRUSTED_ORIGINS}" .Values.gf_csrf_trusted_origins | replace "${CR_GF_SMTP_ENABLED}" .Values.gf_smtp_enabled | replace "${CR_GF_SMTP_HOST}" .Values.gf_smtp_host | replace "${CR_GF_SMTP_USER}" .Values.gf_smtp_user | replace "${CR_GF_SMTP_PASSWORD}" .Values.gf_smtp_password | replace "${CR_GF_SMTP_FROM_ADDRESS}" .Values.gf_smtp_from_address | replace "${CR_GF_SMTP_FROM_NAME}" .Values.gf_smtp_from_name | replace "${CR_GF_SMTP_SKIP_VERIFY}" .Values.gf_smtp_skip_verify | replace "${CR_GF_INGRESS_AUTH_URL}" .Values.gf_ingress_auth_url | replace "${CR_GF_INGRESS_AUTH_SIGNIN}" .Values.gf_ingress_auth_signin | replace "${CR_GF_INGRESS_ERROR_PAGE_403}" .Values.gf_ingress_error_page_403 | replace "${CR_PROM_INGRESS_AUTH_URL}" .Values.prom_ingress_auth_url | replace "${CR_PROM_INGRESS_AUTH_SIGNIN}" .Values.prom_ingress_auth_signin | replace "${CR_PROM_INGRESS_ERROR_PAGE_403}" .Values.prom_ingress_error_page_403 | replace "HELM-NAMESPACE" .Release.Namespace | replace "${LIMITS_MEMORY}" .Values.limits.memory | replace "${LIMITS_CPU}" .Values.limits.cpu | replace "${REQUESTS_STORAGE}" .Values.requests.storage | replace "${RETENTION_TIME}" .Values.retention.time | replace "${RETENTION_SIZE}" .Values.retention.size | replace "${EXTERNAL_URL}" .Values.prom_external_url | replace "${CLOUD_ROBOTICS_DOMAIN}" .Values.domain | replace "${GCP_PROJECT_ID}" .Values.project -}}
+
+# Inject the nodeSelector as a pre-formatted YAML block to allow users to define multiple selectors in a dict within values-cloud.yaml while maintaining valid indentation in the output.
+{{- $prometheusNodeSelectorMap := .Values.prometheus.prometheusSpec.nodeSelector -}}
+{{- if $prometheusNodeSelectorMap }}
+  {{- $prometheusNodeSelectorRawYaml := toYaml $prometheusNodeSelectorMap -}}
+  {{- $formattedPrometheusNodeSelectorObject := $prometheusNodeSelectorRawYaml | nindent 6 -}}
+  {{- $data = $data | replace "${CR_PROMETHEUS_NODE_SELECTOR_OBJECT}" $formattedPrometheusNodeSelectorObject -}}
+{{- else }}
+  {{- $data = $data | replace "${CR_PROMETHEUS_NODE_SELECTOR_OBJECT}" " {}" -}}
+{{- end }}
+
+# Inject the tolerations as a pre-formatted YAML block to support multiple key-value pairs in a list within values-cloud.yaml
+{{- $prometheusTolerationsList := .Values.prometheus.prometheusSpec.tolerations -}}
+{{- if $prometheusTolerationsList }}
+  {{- $prometheusTolerationsRawYaml := toYaml $prometheusTolerationsList -}}
+  {{- $formattedTolerationsObject := $prometheusTolerationsRawYaml | nindent 6 -}}
+  {{- $data = $data | replace "${CR_PROMETHEUS_TOLERATIONS_OBJECT}" $formattedTolerationsObject -}}
+{{- else }}
+  {{- $data = $data | replace "${CR_PROMETHEUS_TOLERATIONS_OBJECT}" " []" -}}
+{{- end }}
+
+{{ $data }}

src/app_charts/prometheus/prometheus-cloud.values.yaml

@@ -63,6 +63,8 @@ prometheus:
     retention: "${RETENTION_TIME}"
     retentionSize: "${RETENTION_SIZE}"
     walCompression: true
+    nodeSelector: "${CR_PROMETHEUS_NODE_SELECTOR_OBJECT}"
+    tolerations: "${CR_PROMETHEUS_TOLERATIONS_OBJECT}"
     resources:
       requests:
         cpu: "${LIMITS_CPU}"

src/app_charts/prometheus/values-cloud.yaml

@@ -33,6 +33,9 @@ retention:
 
 # MetricRelabelConfigs to apply to samples after scraping, but before ingestion.
 prometheus:
+  prometheusSpec:
+    nodeSelector: {}
+    tolerations: []
   serviceMonitor:
     metricRelabelings: []