grafana/rollout-operator
+```
+
+The Grafana rollout-operator should be installed in the same namespace as the statefulsets it is operating upon.
+It is not a highly available application and runs as a single pod.
+
+{{ template "chart.valuesSection" . }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/NOTES.txt b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/NOTES.txt
new file mode 100644
index 0000000..a76e5ba
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/NOTES.txt
@@ -0,0 +1,10 @@
+Repo : {{ .Chart.Home }}
+
+Validation:
+
+Check the logs of the pod and ensure messages for reconcilliation of the statefulsets are present.
+```
+kubectl logs -n {{ .Release.Namespace }} -l {{ include "cli.labels" . }}
+```
+Example log line:
+level=debug ts=2022-04-20T13:59:52.783051541Z msg="reconciling StatefulSet" statefulset=mimir-store-gateway-zone-a
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/_helpers.tpl b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/_helpers.tpl
new file mode 100644
index 0000000..bf3553a
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/_helpers.tpl
@@ -0,0 +1,79 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "rollout-operator.name" -}}
+{{- default (include "rollout-operator.chartName" .) .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "rollout-operator.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default (include "rollout-operator.chartName" .) .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Recalculate the chart name, because it may be sub-chart included as rollout_operator,
+and _ is not valid in resource names.
+*/}}
+{{- define "rollout-operator.chartName" -}}
+{{- print .Chart.Name | replace "_" "-" -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "rollout-operator.chart" -}}
+{{- printf "%s-%s" (include "rollout-operator.chartName" .) .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "rollout-operator.labels" -}}
+helm.sh/chart: {{ include "rollout-operator.chart" . }}
+{{ include "rollout-operator.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "rollout-operator.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "rollout-operator.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "rollout-operator.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "rollout-operator.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+
+{{- define "cli.labels" -}}
+{{- $list := list -}}
+{{- range $k, $v := ( include "rollout-operator.selectorLabels" . | fromYaml ) -}}
+{{- $list = append $list (printf "%s=%s" $k $v) -}}
+{{- end -}}
+{{ join "," $list }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/deployment.yaml b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/deployment.yaml
new file mode 100644
index 0000000..d35b866
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/deployment.yaml
@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "rollout-operator.fullname" . }}
+ labels:
+ {{- include "rollout-operator.labels" . | nindent 4 }}
+spec:
+ replicas: 1
+ minReadySeconds: {{ .Values.minReadySeconds }}
+ selector:
+ matchLabels:
+ {{- include "rollout-operator.selectorLabels" . | nindent 6 }}
+ strategy:
+ rollingUpdate:
+ maxSurge: 0
+ maxUnavailable: 1
+ template:
+ metadata:
+ {{- with .Values.podAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ labels:
+ {{- include "rollout-operator.selectorLabels" . | nindent 8 }}
+ {{- with .Values.podLabels }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ spec:
+ {{- with .Values.priorityClassName }}
+ priorityClassName: {{ . }}
+ {{- end }}
+ {{- with .Values.imagePullSecrets }}
+ imagePullSecrets:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.hostAliases }}
+ hostAliases:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ serviceAccountName: {{ include "rollout-operator.serviceAccountName" . }}
+ securityContext:
+ {{- toYaml .Values.podSecurityContext | nindent 8 }}
+ containers:
+ - name: rollout-operator
+ securityContext:
+ {{- toYaml .Values.securityContext | nindent 12 }}
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - -kubernetes.namespace={{ .Release.Namespace }}
+ ports:
+ - name: http-metrics
+ containerPort: 8001
+ protocol: TCP
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 5
+ timeoutSeconds: 1
+ resources:
+ {{- toYaml .Values.resources | nindent 12 }}
+ {{- with .Values.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/role.yaml b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/role.yaml
new file mode 100644
index 0000000..210c456
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/role.yaml
@@ -0,0 +1,28 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ include "rollout-operator.fullname" . }}
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - pods
+ verbs:
+ - list
+ - get
+ - watch
+ - delete
+- apiGroups:
+ - apps
+ resources:
+ - statefulsets
+ verbs:
+ - list
+ - get
+ - watch
+- apiGroups:
+ - apps
+ resources:
+ - statefulsets/status
+ verbs:
+ - update
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/rolebinding.yaml b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/rolebinding.yaml
new file mode 100644
index 0000000..24fcd72
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/rolebinding.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ include "rollout-operator.fullname" . }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ include "rollout-operator.fullname" . }}
+subjects:
+- kind: ServiceAccount
+ name: {{ include "rollout-operator.serviceAccountName" . }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/service.yaml b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/service.yaml
new file mode 100644
index 0000000..60ce5b1
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/service.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.serviceMonitor.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "rollout-operator.fullname" . }}
+ labels:
+ {{- include "rollout-operator.labels" . | nindent 4 }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ ports:
+ - port: 8001
+ targetPort: http-metrics
+ protocol: TCP
+ name: http-metrics
+ selector:
+ {{- include "rollout-operator.selectorLabels" . | nindent 4 }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/serviceaccount.yaml b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/serviceaccount.yaml
new file mode 100644
index 0000000..37698a4
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/serviceaccount.yaml
@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ include "rollout-operator.serviceAccountName" . }}
+ labels:
+ {{- include "rollout-operator.labels" . | nindent 4 }}
+ {{- with .Values.serviceAccount.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/servicemonitor.yaml b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/servicemonitor.yaml
new file mode 100644
index 0000000..7810843
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/templates/servicemonitor.yaml
@@ -0,0 +1,36 @@
+{{- if .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ name: {{ include "rollout-operator.fullname" . }}
+ {{- with .Values.serviceMonitor.namespace }}
+ namespace: {{ . }}
+ {{- end }}
+ labels:
+ {{- include "rollout-operator.labels" . | nindent 4 }}
+ {{- with .Values.serviceMonitor.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ {{- with .Values.serviceMonitor.namespaceSelector }}
+ namespaceSelector:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ selector:
+ matchLabels:
+ {{- include "rollout-operator.selectorLabels" . | nindent 6 }}
+ endpoints:
+ - port: http-metrics
+ {{- with .Values.serviceMonitor.interval }}
+ interval: {{ . }}
+ {{- end }}
+ {{- with .Values.serviceMonitor.scrapeTimeout }}
+ scrapeTimeout: {{ . }}
+ {{- end }}
+ relabelings:
+ {{- with .Values.serviceMonitor.relabelings }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ scheme: http
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/values.yaml b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/values.yaml
new file mode 100644
index 0000000..66f9486
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/charts/rollout-operator/values.yaml
@@ -0,0 +1,83 @@
+# Default values for rollout-operator.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+image:
+ repository: grafana/rollout-operator
+ pullPolicy: IfNotPresent
+ # -- Overrides the image tag whose default is the chart appVersion.
+ tag: ""
+
+imagePullSecrets: []
+
+# -- hostAliases to add
+hostAliases: []
+# - ip: 1.2.3.4
+# hostnames:
+# - domain.tld
+
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+ # -- Specifies whether a service account should be created
+ create: true
+ # -- Annotations to add to the service account
+ annotations: {}
+ # -- The name of the service account to use.
+ # If not set and create is true, a name is generated using the fullname template
+ name: ""
+
+# -- Pod Annotations
+podAnnotations: {}
+
+# -- Pod (extra) Labels
+podLabels: {}
+
+podSecurityContext: {}
+ # fsGroup: 2000
+
+securityContext: {}
+ # capabilities:
+ # drop:
+ # - ALL
+ # readOnlyRootFilesystem: true
+ # runAsNonRoot: true
+ # runAsUser: 1000
+
+resources:
+ limits:
+ # cpu: "1"
+ memory: 200Mi
+ requests:
+ cpu: 100m
+ memory: 100Mi
+
+minReadySeconds: 10
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+priorityClassName: ""
+
+serviceMonitor:
+ # -- Create ServiceMonitor to scrape metrics for Prometheus
+ enabled: false
+ # -- Alternative namespace for ServiceMonitor resources
+ namespace: null
+ # -- Namespace selector for ServiceMonitor resources
+ namespaceSelector: {}
+ # -- ServiceMonitor annotations
+ annotations: {}
+ # -- Additional ServiceMonitor labels
+ labels: {}
+ # -- ServiceMonitor scrape interval
+ interval: null
+ # -- ServiceMonitor scrape timeout in Go duration format (e.g. 15s)
+ scrapeTimeout: null
+ # -- ServiceMonitor relabel configs to apply to samples before scraping
+ # https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig
+ relabelings: []
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/large.yaml b/observability/lgtm-distributed/charts/mimir-distributed/large.yaml
new file mode 100644
index 0000000..ac9ba73
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/large.yaml
@@ -0,0 +1,209 @@
+# These values configure the Grafana Mimir or Grafana Enterprise Metrics cluster
+# for a more production-ready setup. The setup targets 70% CPU and memory utilization
+# so that the cluster has room to grow. The resource requests reflect 70% utilization
+# and the limits reflect 100% utilization. The values do not set CPU limits,
+# because CPU limits have caused severe issues elsewhere, so we don't apply any in our helm chart:
+# see https://engineering.indeedblog.com/blog/2019/12/unthrottled-fixing-cpu-limits-in-the-cloud/
+# If you require CPU limits for billing purposes see capped-large.yaml
+#
+# These values are suitable for ingestion of ~10M series and scrape interval of 15s.
+# This implies ingestion rate of around 660000 samples per second.
+#
+# Query requirements can vary dramatically depending on query rate and query
+# ranges. The values here satisfy a "usual" query load of around 50 queries per second
+# as seen from our production clusters at this scale.
+#
+# The values in this file also add podAntiAffinity rules for ingesters and store-gateways.
+# The rules ensure that the replicas of the same component are not scheduled on the same
+# Kubernetes Node. Zone-aware replication is enabled by default on new installation.
+# Refer to [Migrate from single zone to zone-aware replication with Helm](https://grafana.com/docs/mimir/latest/migration-guide/migrating-from-single-zone-with-helm) and
+# [Zone-Aware Replication](https://grafana.com/docs/mimir/latest/configure/configure-zone-aware-replication/)
+# for more information.
+#
+# MinIO is no longer enabled, and you are encouraged to use your cloud providers
+# object storage service such as S3 or GCS.
+
+alertmanager:
+ persistentVolume:
+ enabled: true
+ replicas: 3
+ resources:
+ limits:
+ memory: 1.4Gi
+ requests:
+ cpu: 1
+ memory: 1Gi
+ statefulSet:
+ enabled: true
+
+compactor:
+ persistentVolume:
+ size: 50Gi
+ resources:
+ limits:
+ memory: 2.8Gi
+ requests:
+ cpu: 1
+ memory: 2Gi
+
+distributor:
+ replicas: 12
+ resources:
+ limits:
+ memory: 5.7Gi
+ requests:
+ cpu: 2
+ memory: 4Gi
+
+ingester:
+ persistentVolume:
+ size: 50Gi
+ replicas: 27
+ resources:
+ limits:
+ memory: 12Gi
+ requests:
+ cpu: 3.5
+ memory: 8Gi
+ topologySpreadConstraints: {}
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: target # support for enterprise.legacyLabels
+ operator: In
+ values:
+ - ingester
+ topologyKey: 'kubernetes.io/hostname'
+
+ - labelSelector:
+ matchExpressions:
+ - key: app.kubernetes.io/component
+ operator: In
+ values:
+ - ingester
+ topologyKey: 'kubernetes.io/hostname'
+
+ zoneAwareReplication:
+ topologyKey: 'kubernetes.io/hostname'
+
+admin-cache:
+ enabled: true
+ replicas: 2
+
+chunks-cache:
+ enabled: true
+ replicas: 4
+
+index-cache:
+ enabled: true
+ replicas: 4
+
+metadata-cache:
+ enabled: true
+
+results-cache:
+ enabled: true
+ replicas: 4
+ allocatedMemory: 1024
+
+minio:
+ enabled: false
+
+overrides_exporter:
+ replicas: 1
+ resources:
+ limits:
+ memory: 128Mi
+ requests:
+ cpu: 100m
+ memory: 128Mi
+
+querier:
+ replicas: 4
+ resources:
+ limits:
+ memory: 8.5Gi
+ requests:
+ cpu: 2
+ memory: 6Gi
+
+query_frontend:
+ replicas: 3
+ resources:
+ limits:
+ memory: 2.8Gi
+ requests:
+ cpu: 2
+ memory: 2Gi
+
+ruler:
+ replicas: 3
+ resources:
+ limits:
+ memory: 5.7Gi
+ requests:
+ cpu: 1
+ memory: 4Gi
+
+store_gateway:
+ persistentVolume:
+ size: 50Gi
+ replicas: 6
+ resources:
+ limits:
+ memory: 8.5Gi
+ requests:
+ cpu: 1
+ memory: 6Gi
+ topologySpreadConstraints: {}
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: target # support for enterprise.legacyLabels
+ operator: In
+ values:
+ - store-gateway
+ topologyKey: 'kubernetes.io/hostname'
+
+ - labelSelector:
+ matchExpressions:
+ - key: app.kubernetes.io/component
+ operator: In
+ values:
+ - store-gateway
+ topologyKey: 'kubernetes.io/hostname'
+
+ zoneAwareReplication:
+ topologyKey: 'kubernetes.io/hostname'
+
+nginx:
+ replicas: 3
+ resources:
+ limits:
+ memory: 731Mi
+ requests:
+ cpu: 1
+ memory: 512Mi
+
+# Grafana Enterprise Metrics feature related
+admin_api:
+ replicas: 2
+ resources:
+ limits:
+ memory: 128Mi
+ requests:
+ cpu: 100m
+ memory: 64Mi
+
+gateway:
+ replicas: 3
+ resources:
+ limits:
+ memory: 731Mi
+ requests:
+ cpu: 1
+ memory: 512Mi
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/alerts.yaml b/observability/lgtm-distributed/charts/mimir-distributed/mixins/alerts.yaml
new file mode 100644
index 0000000..ef05886
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/alerts.yaml
@@ -0,0 +1,1061 @@
+groups:
+- name: mimir_alerts
+ rules:
+ - alert: MimirIngesterUnhealthy
+ annotations:
+ message: Mimir cluster {{ $labels.cluster }}/{{ $labels.namespace }} has {{
+ printf "%f" $value }} unhealthy ingester(s).
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterunhealthy
+ expr: |
+ min by (cluster, namespace) (cortex_ring_members{state="Unhealthy", name="ingester"}) > 0
+ for: 15m
+ labels:
+ severity: critical
+ - alert: MimirRequestErrors
+ annotations:
+ message: |
+ The route {{ $labels.route }} in {{ $labels.cluster }}/{{ $labels.namespace }} is experiencing {{ printf "%.2f" $value }}% errors.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrequesterrors
+ expr: |
+ 100 * sum by (cluster, namespace, job, route) (rate(cortex_request_duration_seconds_count{status_code=~"5..",route!~"ready|debug_pprof"}[1m]))
+ /
+ sum by (cluster, namespace, job, route) (rate(cortex_request_duration_seconds_count{route!~"ready|debug_pprof"}[1m]))
+ > 1
+ for: 15m
+ labels:
+ severity: critical
+ - alert: MimirRequestLatency
+ annotations:
+ message: |
+ {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrequestlatency
+ expr: |
+ cluster_namespace_job_route:cortex_request_duration_seconds:99quantile{route!~"metrics|/frontend.Frontend/Process|ready|/schedulerpb.SchedulerForFrontend/FrontendLoop|/schedulerpb.SchedulerForQuerier/QuerierLoop|debug_pprof"}
+ >
+ 2.5
+ for: 15m
+ labels:
+ severity: warning
+ - alert: MimirInconsistentRuntimeConfig
+ annotations:
+ message: |
+ An inconsistent runtime config file is used across cluster {{ $labels.cluster }}/{{ $labels.namespace }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirinconsistentruntimeconfig
+ expr: |
+ count(count by(cluster, namespace, job, sha256) (cortex_runtime_config_hash)) without(sha256) > 1
+ for: 1h
+ labels:
+ severity: critical
+ - alert: MimirBadRuntimeConfig
+ annotations:
+ message: |
+ {{ $labels.job }} failed to reload runtime config.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirbadruntimeconfig
+ expr: |
+ # The metric value is reset to 0 on error while reloading the config at runtime.
+ cortex_runtime_config_last_reload_successful == 0
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirFrontendQueriesStuck
+ annotations:
+ message: |
+ There are {{ $value }} queued up queries in {{ $labels.cluster }}/{{ $labels.namespace }} {{ $labels.job }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirfrontendqueriesstuck
+ expr: |
+ sum by (cluster, namespace, job) (min_over_time(cortex_query_frontend_queue_length[1m])) > 0
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirSchedulerQueriesStuck
+ annotations:
+ message: |
+ There are {{ $value }} queued up queries in {{ $labels.cluster }}/{{ $labels.namespace }} {{ $labels.job }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirschedulerqueriesstuck
+ expr: |
+ sum by (cluster, namespace, job) (min_over_time(cortex_query_scheduler_queue_length[1m])) > 0
+ for: 7m
+ labels:
+ severity: critical
+ - alert: MimirCacheRequestErrors
+ annotations:
+ message: |
+ The cache {{ $labels.name }} used by Mimir {{ $labels.cluster }}/{{ $labels.namespace }} is experiencing {{ printf "%.2f" $value }}% errors for {{ $labels.operation }} operation.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircacherequesterrors
+ expr: |
+ (
+ sum by(cluster, namespace, name, operation) (
+ rate(thanos_memcached_operation_failures_total[1m])
+ or
+ rate(thanos_cache_operation_failures_total[1m])
+ )
+ /
+ sum by(cluster, namespace, name, operation) (
+ rate(thanos_memcached_operations_total[1m])
+ or
+ rate(thanos_cache_operations_total[1m])
+ )
+ ) * 100 > 5
+ for: 5m
+ labels:
+ severity: warning
+ - alert: MimirIngesterRestarts
+ annotations:
+ message: Mimir {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has restarted {{ printf "%.2f" $value }} times in the last 30 mins.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterrestarts
+ expr: |
+ (
+ sum by(cluster, namespace, pod) (
+ increase(kube_pod_container_status_restarts_total{container=~"(ingester|mimir-write)"}[30m])
+ )
+ >= 2
+ )
+ and
+ (
+ count by(cluster, namespace, pod) (cortex_build_info) > 0
+ )
+ labels:
+ severity: warning
+ - alert: MimirKVStoreFailure
+ annotations:
+ message: |
+ Mimir {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} is failing to talk to the KV store {{ $labels.kv_name }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirkvstorefailure
+ expr: |
+ (
+ sum by(cluster, namespace, pod, status_code, kv_name) (rate(cortex_kv_request_duration_seconds_count{status_code!~"2.+"}[1m]))
+ /
+ sum by(cluster, namespace, pod, status_code, kv_name) (rate(cortex_kv_request_duration_seconds_count[1m]))
+ )
+ # We want to get alerted only in case there's a constant failure.
+ == 1
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirMemoryMapAreasTooHigh
+ annotations:
+ message: '{{ $labels.job }}/{{ $labels.pod }} has a number of mmap-ed areas
+ close to the limit.'
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirmemorymapareastoohigh
+ expr: |
+ process_memory_map_areas{job=~".*/(ingester.*|cortex|mimir|mimir-write.*|store-gateway.*|cortex|mimir|mimir-backend.*)"} / process_memory_map_areas_limit{job=~".*/(ingester.*|cortex|mimir|mimir-write.*|store-gateway.*|cortex|mimir|mimir-backend.*)"} > 0.8
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirIngesterInstanceHasNoTenants
+ annotations:
+ message: Mimir ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has no tenants assigned.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterinstancehasnotenants
+ expr: |
+ (min by(cluster, namespace, pod) (cortex_ingester_memory_users) == 0)
+ and on (cluster, namespace)
+ # Only if there are more time-series than would be expected due to continuous testing load
+ (
+ sum by(cluster, namespace) (cortex_ingester_memory_series)
+ /
+ max by(cluster, namespace) (cortex_distributor_replication_factor)
+ ) > 100000
+ for: 1h
+ labels:
+ severity: warning
+ - alert: MimirRulerInstanceHasNoRuleGroups
+ annotations:
+ message: Mimir ruler {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has no rule groups assigned.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrulerinstancehasnorulegroups
+ expr: |
+ # Alert on ruler instances in microservices mode that have no rule groups assigned,
+ min by(cluster, namespace, pod) (cortex_ruler_managers_total{pod=~"(.*mimir-)?ruler.*"}) == 0
+ # but only if other ruler instances of the same cell do have rule groups assigned
+ and on (cluster, namespace)
+ (max by(cluster, namespace) (cortex_ruler_managers_total) > 0)
+ # and there are more than two instances overall
+ and on (cluster, namespace)
+ (count by (cluster, namespace) (cortex_ruler_managers_total) > 2)
+ for: 1h
+ labels:
+ severity: warning
+ - alert: MimirIngestedDataTooFarInTheFuture
+ annotations:
+ message: Mimir ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has ingested samples with timestamps more than 1h in the future.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesteddatatoofarinthefuture
+ expr: |
+ max by(cluster, namespace, pod) (
+ cortex_ingester_tsdb_head_max_timestamp_seconds - time()
+ and
+ cortex_ingester_tsdb_head_max_timestamp_seconds > 0
+ ) > 60*60
+ for: 5m
+ labels:
+ severity: warning
+ - alert: MimirStoreGatewayTooManyFailedOperations
+ annotations:
+ message: Mimir store-gateway {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is experiencing {{ $value | humanizePercentage }} errors while doing {{
+ $labels.operation }} on the object storage.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirstoregatewaytoomanyfailedoperations
+ expr: |
+ sum by(cluster, namespace, operation) (rate(thanos_objstore_bucket_operation_failures_total{component="store-gateway"}[1m])) > 0
+ for: 5m
+ labels:
+ severity: warning
+ - alert: MimirRingMembersMismatch
+ annotations:
+ message: |
+ Number of members in Mimir ingester hash ring does not match the expected number in {{ $labels.cluster }}/{{ $labels.namespace }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirringmembersmismatch
+ expr: |
+ (
+ avg by(cluster, namespace) (sum by(cluster, namespace, pod) (cortex_ring_members{name="ingester",job=~".*/(ingester.*|cortex|mimir|mimir-write.*)"}))
+ != sum by(cluster, namespace) (up{job=~".*/(ingester.*|cortex|mimir|mimir-write.*)"})
+ )
+ and
+ (
+ count by(cluster, namespace) (cortex_build_info) > 0
+ )
+ for: 15m
+ labels:
+ component: ingester
+ severity: warning
+- name: mimir_instance_limits_alerts
+ rules:
+ - alert: MimirIngesterReachingSeriesLimit
+ annotations:
+ message: |
+ Ingester {{ $labels.job }}/{{ $labels.pod }} has reached {{ $value | humanizePercentage }} of its series limit.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterreachingserieslimit
+ expr: |
+ (
+ (cortex_ingester_memory_series / ignoring(limit) cortex_ingester_instance_limits{limit="max_series"})
+ and ignoring (limit)
+ (cortex_ingester_instance_limits{limit="max_series"} > 0)
+ ) > 0.8
+ for: 3h
+ labels:
+ severity: warning
+ - alert: MimirIngesterReachingSeriesLimit
+ annotations:
+ message: |
+ Ingester {{ $labels.job }}/{{ $labels.pod }} has reached {{ $value | humanizePercentage }} of its series limit.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterreachingserieslimit
+ expr: |
+ (
+ (cortex_ingester_memory_series / ignoring(limit) cortex_ingester_instance_limits{limit="max_series"})
+ and ignoring (limit)
+ (cortex_ingester_instance_limits{limit="max_series"} > 0)
+ ) > 0.9
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirIngesterReachingTenantsLimit
+ annotations:
+ message: |
+ Ingester {{ $labels.job }}/{{ $labels.pod }} has reached {{ $value | humanizePercentage }} of its tenant limit.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterreachingtenantslimit
+ expr: |
+ (
+ (cortex_ingester_memory_users / ignoring(limit) cortex_ingester_instance_limits{limit="max_tenants"})
+ and ignoring (limit)
+ (cortex_ingester_instance_limits{limit="max_tenants"} > 0)
+ ) > 0.7
+ for: 5m
+ labels:
+ severity: warning
+ - alert: MimirIngesterReachingTenantsLimit
+ annotations:
+ message: |
+ Ingester {{ $labels.job }}/{{ $labels.pod }} has reached {{ $value | humanizePercentage }} of its tenant limit.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterreachingtenantslimit
+ expr: |
+ (
+ (cortex_ingester_memory_users / ignoring(limit) cortex_ingester_instance_limits{limit="max_tenants"})
+ and ignoring (limit)
+ (cortex_ingester_instance_limits{limit="max_tenants"} > 0)
+ ) > 0.8
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirReachingTCPConnectionsLimit
+ annotations:
+ message: |
+ Mimir instance {{ $labels.job }}/{{ $labels.pod }} has reached {{ $value | humanizePercentage }} of its TCP connections limit for {{ $labels.protocol }} protocol.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirreachingtcpconnectionslimit
+ expr: |
+ cortex_tcp_connections / cortex_tcp_connections_limit > 0.8 and
+ cortex_tcp_connections_limit > 0
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirDistributorReachingInflightPushRequestLimit
+ annotations:
+ message: |
+ Distributor {{ $labels.job }}/{{ $labels.pod }} has reached {{ $value | humanizePercentage }} of its inflight push request limit.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirdistributorreachinginflightpushrequestlimit
+ expr: |
+ (
+ (cortex_distributor_inflight_push_requests / ignoring(limit) cortex_distributor_instance_limits{limit="max_inflight_push_requests"})
+ and ignoring (limit)
+ (cortex_distributor_instance_limits{limit="max_inflight_push_requests"} > 0)
+ ) > 0.8
+ for: 5m
+ labels:
+ severity: critical
+- name: mimir-rollout-alerts
+ rules:
+ - alert: MimirRolloutStuck
+ annotations:
+ message: |
+ The {{ $labels.rollout_group }} rollout is stuck in {{ $labels.cluster }}/{{ $labels.namespace }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrolloutstuck
+ expr: |
+ (
+ max without (revision) (
+ sum without(statefulset) (label_replace(kube_statefulset_status_current_revision, "rollout_group", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?"))
+ unless
+ sum without(statefulset) (label_replace(kube_statefulset_status_update_revision, "rollout_group", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?"))
+ )
+ *
+ (
+ sum without(statefulset) (label_replace(kube_statefulset_replicas, "rollout_group", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?"))
+ !=
+ sum without(statefulset) (label_replace(kube_statefulset_status_replicas_updated, "rollout_group", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?"))
+ )
+ ) and (
+ changes(sum without(statefulset) (label_replace(kube_statefulset_status_replicas_updated, "rollout_group", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?"))[15m:1m])
+ ==
+ 0
+ )
+ * on(cluster, namespace) group_left max by(cluster, namespace) (cortex_build_info)
+ for: 30m
+ labels:
+ severity: warning
+ workload_type: statefulset
+ - alert: MimirRolloutStuck
+ annotations:
+ message: |
+ The {{ $labels.rollout_group }} rollout is stuck in {{ $labels.cluster }}/{{ $labels.namespace }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrolloutstuck
+ expr: |
+ (
+ sum without(deployment) (label_replace(kube_deployment_spec_replicas, "rollout_group", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"))
+ !=
+ sum without(deployment) (label_replace(kube_deployment_status_replicas_updated, "rollout_group", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"))
+ ) and (
+ changes(sum without(deployment) (label_replace(kube_deployment_status_replicas_updated, "rollout_group", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"))[15m:1m])
+ ==
+ 0
+ )
+ * on(cluster, namespace) group_left max by(cluster, namespace) (cortex_build_info)
+ for: 30m
+ labels:
+ severity: warning
+ workload_type: deployment
+ - alert: RolloutOperatorNotReconciling
+ annotations:
+ message: |
+ Rollout operator is not reconciling the rollout group {{ $labels.rollout_group }} in {{ $labels.cluster }}/{{ $labels.namespace }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#rolloutoperatornotreconciling
+ expr: |
+ max by(cluster, namespace, rollout_group) (time() - rollout_operator_last_successful_group_reconcile_timestamp_seconds) > 600
+ for: 5m
+ labels:
+ severity: critical
+- name: mimir-provisioning
+ rules:
+ - alert: MimirAllocatingTooMuchMemory
+ annotations:
+ message: |
+ Instance {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} is using too much memory.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirallocatingtoomuchmemory
+ expr: |
+ (
+ # We use RSS instead of working set memory because of the ingester's extensive usage of mmap.
+ # See: https://github.com/grafana/mimir/issues/2466
+ container_memory_rss{container=~"(ingester|mimir-write|mimir-backend)"}
+ /
+ ( container_spec_memory_limit_bytes{container=~"(ingester|mimir-write|mimir-backend)"} > 0 )
+ )
+ # Match only Mimir namespaces.
+ * on(cluster, namespace) group_left max by(cluster, namespace) (cortex_build_info)
+ > 0.65
+ for: 15m
+ labels:
+ severity: warning
+ - alert: MimirAllocatingTooMuchMemory
+ annotations:
+ message: |
+ Instance {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} is using too much memory.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirallocatingtoomuchmemory
+ expr: |
+ (
+ # We use RSS instead of working set memory because of the ingester's extensive usage of mmap.
+ # See: https://github.com/grafana/mimir/issues/2466
+ container_memory_rss{container=~"(ingester|mimir-write|mimir-backend)"}
+ /
+ ( container_spec_memory_limit_bytes{container=~"(ingester|mimir-write|mimir-backend)"} > 0 )
+ )
+ # Match only Mimir namespaces.
+ * on(cluster, namespace) group_left max by(cluster, namespace) (cortex_build_info)
+ > 0.8
+ for: 15m
+ labels:
+ severity: critical
+- name: ruler_alerts
+ rules:
+ - alert: MimirRulerTooManyFailedPushes
+ annotations:
+ message: |
+ Mimir Ruler {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} is experiencing {{ printf "%.2f" $value }}% write (push) errors.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrulertoomanyfailedpushes
+ expr: |
+ 100 * (
+ sum by (cluster, namespace, pod) (rate(cortex_ruler_write_requests_failed_total[1m]))
+ /
+ sum by (cluster, namespace, pod) (rate(cortex_ruler_write_requests_total[1m]))
+ ) > 1
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirRulerTooManyFailedQueries
+ annotations:
+ message: |
+ Mimir Ruler {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} is experiencing {{ printf "%.2f" $value }}% errors while evaluating rules.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrulertoomanyfailedqueries
+ expr: |
+ 100 * (
+ sum by (cluster, namespace, pod) (rate(cortex_ruler_queries_failed_total[1m]))
+ /
+ sum by (cluster, namespace, pod) (rate(cortex_ruler_queries_total[1m]))
+ ) > 1
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirRulerMissedEvaluations
+ annotations:
+ message: |
+ Mimir Ruler {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} is experiencing {{ printf "%.2f" $value }}% missed iterations for the rule group {{ $labels.rule_group }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrulermissedevaluations
+ expr: |
+ 100 * (
+ sum by (cluster, namespace, pod, rule_group) (rate(cortex_prometheus_rule_group_iterations_missed_total[1m]))
+ /
+ sum by (cluster, namespace, pod, rule_group) (rate(cortex_prometheus_rule_group_iterations_total[1m]))
+ ) > 1
+ for: 5m
+ labels:
+ severity: warning
+ - alert: MimirRulerFailedRingCheck
+ annotations:
+ message: |
+ Mimir Rulers in {{ $labels.cluster }}/{{ $labels.namespace }} are experiencing errors when checking the ring for rule group ownership.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrulerfailedringcheck
+ expr: |
+ sum by (cluster, namespace, job) (rate(cortex_ruler_ring_check_errors_total[1m]))
+ > 0
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirRulerRemoteEvaluationFailing
+ annotations:
+ message: |
+ Mimir rulers in {{ $labels.cluster }}/{{ $labels.namespace }} are failing to perform {{ printf "%.2f" $value }}% of remote evaluations through the ruler-query-frontend.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrulerremoteevaluationfailing
+ expr: |
+ 100 * (
+ sum by (cluster, namespace) (rate(cortex_request_duration_seconds_count{route="/httpgrpc.HTTP/Handle", status_code=~"5..", job=~".*/(ruler-query-frontend.*)"}[5m]))
+ /
+ sum by (cluster, namespace) (rate(cortex_request_duration_seconds_count{route="/httpgrpc.HTTP/Handle", job=~".*/(ruler-query-frontend.*)"}[5m]))
+ ) > 1
+ for: 5m
+ labels:
+ severity: warning
+- name: gossip_alerts
+ rules:
+ - alert: MimirGossipMembersTooHigh
+ annotations:
+ message: One or more Mimir instances in {{ $labels.cluster }}/{{ $labels.namespace
+ }} consistently sees a higher than expected number of gossip members.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirgossipmemberstoohigh
+ expr: |
+ max by (cluster, namespace) (memberlist_client_cluster_members_count)
+ >
+ (sum by (cluster, namespace) (up{job=~".+/(admin-api|alertmanager|compactor.*|distributor.*|ingester.*|querier.*|ruler|ruler-querier.*|store-gateway.*|cortex|mimir|mimir-write.*|mimir-read.*|mimir-backend.*)"}) + 10)
+ for: 20m
+ labels:
+ severity: warning
+ - alert: MimirGossipMembersTooLow
+ annotations:
+ message: One or more Mimir instances in {{ $labels.cluster }}/{{ $labels.namespace
+ }} consistently sees a lower than expected number of gossip members.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirgossipmemberstoolow
+ expr: |
+ min by (cluster, namespace) (memberlist_client_cluster_members_count)
+ <
+ (sum by (cluster, namespace) (up{job=~".+/(admin-api|alertmanager|compactor.*|distributor.*|ingester.*|querier.*|ruler|ruler-querier.*|store-gateway.*|cortex|mimir|mimir-write.*|mimir-read.*|mimir-backend.*)"}) * 0.5)
+ for: 20m
+ labels:
+ severity: warning
+- name: etcd_alerts
+ rules:
+ - alert: EtcdAllocatingTooMuchMemory
+ annotations:
+ message: |
+ Too much memory being used by {{ $labels.namespace }}/{{ $labels.pod }} - bump memory limit.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#etcdallocatingtoomuchmemory
+ expr: |
+ (
+ container_memory_working_set_bytes{container="etcd"}
+ /
+ ( container_spec_memory_limit_bytes{container="etcd"} > 0 )
+ ) > 0.65
+ for: 15m
+ labels:
+ severity: warning
+ - alert: EtcdAllocatingTooMuchMemory
+ annotations:
+ message: |
+ Too much memory being used by {{ $labels.namespace }}/{{ $labels.pod }} - bump memory limit.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#etcdallocatingtoomuchmemory
+ expr: |
+ (
+ container_memory_working_set_bytes{container="etcd"}
+ /
+ ( container_spec_memory_limit_bytes{container="etcd"} > 0 )
+ ) > 0.8
+ for: 15m
+ labels:
+ severity: critical
+- name: alertmanager_alerts
+ rules:
+ - alert: MimirAlertmanagerSyncConfigsFailing
+ annotations:
+ message: |
+ Mimir Alertmanager {{ $labels.job }}/{{ $labels.pod }} is failing to read tenant configurations from storage.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiralertmanagersyncconfigsfailing
+ expr: |
+ rate(cortex_alertmanager_sync_configs_failed_total[5m]) > 0
+ for: 30m
+ labels:
+ severity: critical
+ - alert: MimirAlertmanagerRingCheckFailing
+ annotations:
+ message: |
+ Mimir Alertmanager {{ $labels.job }}/{{ $labels.pod }} is unable to check tenants ownership via the ring.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiralertmanagerringcheckfailing
+ expr: |
+ rate(cortex_alertmanager_ring_check_errors_total[2m]) > 0
+ for: 10m
+ labels:
+ severity: critical
+ - alert: MimirAlertmanagerPartialStateMergeFailing
+ annotations:
+ message: |
+ Mimir Alertmanager {{ $labels.job }}/{{ $labels.pod }} is failing to merge partial state changes received from a replica.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiralertmanagerpartialstatemergefailing
+ expr: |
+ rate(cortex_alertmanager_partial_state_merges_failed_total[2m]) > 0
+ for: 10m
+ labels:
+ severity: critical
+ - alert: MimirAlertmanagerReplicationFailing
+ annotations:
+ message: |
+ Mimir Alertmanager {{ $labels.job }}/{{ $labels.pod }} is failing to replicating partial state to its replicas.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiralertmanagerreplicationfailing
+ expr: |
+ rate(cortex_alertmanager_state_replication_failed_total[2m]) > 0
+ for: 10m
+ labels:
+ severity: critical
+ - alert: MimirAlertmanagerPersistStateFailing
+ annotations:
+ message: |
+ Mimir Alertmanager {{ $labels.job }}/{{ $labels.pod }} is unable to persist full state snaphots to remote storage.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiralertmanagerpersiststatefailing
+ expr: |
+ rate(cortex_alertmanager_state_persist_failed_total[15m]) > 0
+ for: 1h
+ labels:
+ severity: critical
+ - alert: MimirAlertmanagerInitialSyncFailed
+ annotations:
+ message: |
+ Mimir Alertmanager {{ $labels.job }}/{{ $labels.pod }} was unable to obtain some initial state when starting up.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiralertmanagerinitialsyncfailed
+ expr: |
+ increase(cortex_alertmanager_state_initial_sync_completed_total{outcome="failed"}[1m]) > 0
+ labels:
+ severity: critical
+ - alert: MimirAlertmanagerAllocatingTooMuchMemory
+ annotations:
+ message: |
+ Alertmanager {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} is using too much memory.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiralertmanagerallocatingtoomuchmemory
+ expr: |
+ (container_memory_working_set_bytes{container="alertmanager"} / container_spec_memory_limit_bytes{container="alertmanager"}) > 0.80
+ and
+ (container_spec_memory_limit_bytes{container="alertmanager"} > 0)
+ for: 15m
+ labels:
+ severity: warning
+ - alert: MimirAlertmanagerAllocatingTooMuchMemory
+ annotations:
+ message: |
+ Alertmanager {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} is using too much memory.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiralertmanagerallocatingtoomuchmemory
+ expr: |
+ (container_memory_working_set_bytes{container="alertmanager"} / container_spec_memory_limit_bytes{container="alertmanager"}) > 0.90
+ and
+ (container_spec_memory_limit_bytes{container="alertmanager"} > 0)
+ for: 15m
+ labels:
+ severity: critical
+ - alert: MimirAlertmanagerInstanceHasNoTenants
+ annotations:
+ message: Mimir alertmanager {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} owns no tenants.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiralertmanagerinstancehasnotenants
+ expr: |
+ # Alert on alertmanager instances in microservices mode that own no tenants,
+ min by(cluster, namespace, pod) (cortex_alertmanager_tenants_owned{pod=~"(.*mimir-)?alertmanager.*"}) == 0
+ # but only if other instances of the same cell do have tenants assigned.
+ and on (cluster, namespace)
+ max by(cluster, namespace) (cortex_alertmanager_tenants_owned) > 0
+ for: 1h
+ labels:
+ severity: warning
+- name: mimir_blocks_alerts
+ rules:
+ - alert: MimirIngesterHasNotShippedBlocks
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has not shipped any block in the last 4 hours.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterhasnotshippedblocks
+ expr: |
+ (min by(cluster, namespace, pod) (time() - cortex_ingester_shipper_last_successful_upload_timestamp_seconds) > 60 * 60 * 4)
+ and
+ (max by(cluster, namespace, pod) (cortex_ingester_shipper_last_successful_upload_timestamp_seconds) > 0)
+ and
+ # Only if the ingester has ingested samples over the last 4h.
+ (max by(cluster, namespace, pod) (max_over_time(cluster_namespace_pod:cortex_ingester_ingested_samples_total:rate1m[4h])) > 0)
+ and
+ # Only if the ingester was ingesting samples 4h ago. This protects against the case where the ingester replica
+ # had ingested samples in the past, then no traffic was received for a long period and then it starts
+ # receiving samples again. Without this check, the alert would fire as soon as it gets back receiving
+ # samples, while the a block shipping is expected within the next 4h.
+ (max by(cluster, namespace, pod) (max_over_time(cluster_namespace_pod:cortex_ingester_ingested_samples_total:rate1m[1h] offset 4h)) > 0)
+ for: 15m
+ labels:
+ severity: critical
+ - alert: MimirIngesterHasNotShippedBlocksSinceStart
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has not shipped any block in the last 4 hours.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterhasnotshippedblockssincestart
+ expr: |
+ (max by(cluster, namespace, pod) (cortex_ingester_shipper_last_successful_upload_timestamp_seconds) == 0)
+ and
+ (max by(cluster, namespace, pod) (max_over_time(cluster_namespace_pod:cortex_ingester_ingested_samples_total:rate1m[4h])) > 0)
+ for: 4h
+ labels:
+ severity: critical
+ - alert: MimirIngesterHasUnshippedBlocks
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has compacted a block {{ $value | humanizeDuration }} ago but it hasn't
+ been successfully uploaded to the storage yet.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterhasunshippedblocks
+ expr: |
+ (time() - cortex_ingester_oldest_unshipped_block_timestamp_seconds > 3600)
+ and
+ (cortex_ingester_oldest_unshipped_block_timestamp_seconds > 0)
+ for: 15m
+ labels:
+ severity: critical
+ - alert: MimirIngesterTSDBHeadCompactionFailed
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is failing to compact TSDB head.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringestertsdbheadcompactionfailed
+ expr: |
+ rate(cortex_ingester_tsdb_compactions_failed_total[5m]) > 0
+ for: 15m
+ labels:
+ severity: critical
+ - alert: MimirIngesterTSDBHeadTruncationFailed
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is failing to truncate TSDB head.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringestertsdbheadtruncationfailed
+ expr: |
+ rate(cortex_ingester_tsdb_head_truncations_failed_total[5m]) > 0
+ labels:
+ severity: critical
+ - alert: MimirIngesterTSDBCheckpointCreationFailed
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is failing to create TSDB checkpoint.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringestertsdbcheckpointcreationfailed
+ expr: |
+ rate(cortex_ingester_tsdb_checkpoint_creations_failed_total[5m]) > 0
+ labels:
+ severity: critical
+ - alert: MimirIngesterTSDBCheckpointDeletionFailed
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is failing to delete TSDB checkpoint.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringestertsdbcheckpointdeletionfailed
+ expr: |
+ rate(cortex_ingester_tsdb_checkpoint_deletions_failed_total[5m]) > 0
+ labels:
+ severity: critical
+ - alert: MimirIngesterTSDBWALTruncationFailed
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is failing to truncate TSDB WAL.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringestertsdbwaltruncationfailed
+ expr: |
+ rate(cortex_ingester_tsdb_wal_truncations_failed_total[5m]) > 0
+ labels:
+ severity: warning
+ - alert: MimirIngesterTSDBWALCorrupted
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} got a corrupted TSDB WAL.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringestertsdbwalcorrupted
+ expr: |
+ # alert when there are more than one corruptions
+ count by (cluster, namespace) (rate(cortex_ingester_tsdb_wal_corruptions_total[5m]) > 0) > 1
+ and
+ # and there is only one zone
+ count by (cluster, namespace) (group by (cluster, namespace, job) (cortex_ingester_tsdb_wal_corruptions_total)) == 1
+ labels:
+ deployment: single-zone
+ severity: critical
+ - alert: MimirIngesterTSDBWALCorrupted
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} got a corrupted TSDB WAL.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringestertsdbwalcorrupted
+ expr: |
+ # alert when there are more than one corruptions
+ count by (cluster, namespace) (sum by (cluster, namespace, job) (rate(cortex_ingester_tsdb_wal_corruptions_total[5m]) > 0)) > 1
+ and
+ # and there are multiple zones
+ count by (cluster, namespace) (group by (cluster, namespace, job) (cortex_ingester_tsdb_wal_corruptions_total)) > 1
+ labels:
+ deployment: multi-zone
+ severity: critical
+ - alert: MimirIngesterTSDBWALWritesFailed
+ annotations:
+ message: Mimir Ingester {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is failing to write to TSDB WAL.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringestertsdbwalwritesfailed
+ expr: |
+ rate(cortex_ingester_tsdb_wal_writes_failed_total[1m]) > 0
+ for: 3m
+ labels:
+ severity: critical
+ - alert: MimirStoreGatewayHasNotSyncTheBucket
+ annotations:
+ message: Mimir store-gateway {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has not successfully synched the bucket since {{ $value | humanizeDuration
+ }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirstoregatewayhasnotsyncthebucket
+ expr: |
+ (time() - cortex_bucket_stores_blocks_last_successful_sync_timestamp_seconds{component="store-gateway"} > 60 * 30)
+ and
+ cortex_bucket_stores_blocks_last_successful_sync_timestamp_seconds{component="store-gateway"} > 0
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirStoreGatewayNoSyncedTenants
+ annotations:
+ message: Mimir store-gateway {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is not syncing any blocks for any tenant.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirstoregatewaynosyncedtenants
+ expr: |
+ min by(cluster, namespace, pod) (cortex_bucket_stores_tenants_synced{component="store-gateway"}) == 0
+ for: 1h
+ labels:
+ severity: warning
+ - alert: MimirBucketIndexNotUpdated
+ annotations:
+ message: Mimir bucket index for tenant {{ $labels.user }} in {{ $labels.cluster
+ }}/{{ $labels.namespace }} has not been updated since {{ $value | humanizeDuration
+ }}.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirbucketindexnotupdated
+ expr: |
+ min by(cluster, namespace, user) (time() - cortex_bucket_index_last_successful_update_timestamp_seconds) > 7200
+ labels:
+ severity: critical
+- name: mimir_compactor_alerts
+ rules:
+ - alert: MimirCompactorHasNotSuccessfullyCleanedUpBlocks
+ annotations:
+ message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has not successfully cleaned up blocks in the last 6 hours.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircompactorhasnotsuccessfullycleanedupblocks
+ expr: |
+ # The "last successful run" metric is updated even if the compactor owns no tenants,
+ # so this alert correctly doesn't fire if compactor has nothing to do.
+ (time() - cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds > 60 * 60 * 6)
+ for: 1h
+ labels:
+ severity: critical
+ - alert: MimirCompactorHasNotSuccessfullyRunCompaction
+ annotations:
+ message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has not run compaction in the last 24 hours.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircompactorhasnotsuccessfullyruncompaction
+ expr: |
+ # The "last successful run" metric is updated even if the compactor owns no tenants,
+ # so this alert correctly doesn't fire if compactor has nothing to do.
+ (time() - cortex_compactor_last_successful_run_timestamp_seconds > 60 * 60 * 24)
+ and
+ (cortex_compactor_last_successful_run_timestamp_seconds > 0)
+ for: 1h
+ labels:
+ reason: in-last-24h
+ severity: critical
+ - alert: MimirCompactorHasNotSuccessfullyRunCompaction
+ annotations:
+ message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has not run compaction in the last 24 hours.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircompactorhasnotsuccessfullyruncompaction
+ expr: |
+ # The "last successful run" metric is updated even if the compactor owns no tenants,
+ # so this alert correctly doesn't fire if compactor has nothing to do.
+ cortex_compactor_last_successful_run_timestamp_seconds == 0
+ for: 24h
+ labels:
+ reason: since-startup
+ severity: critical
+ - alert: MimirCompactorHasNotSuccessfullyRunCompaction
+ annotations:
+ message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} failed to run 2 consecutive compactions.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircompactorhasnotsuccessfullyruncompaction
+ expr: |
+ increase(cortex_compactor_runs_failed_total{reason!="shutdown"}[2h]) >= 2
+ labels:
+ reason: consecutive-failures
+ severity: critical
+ - alert: MimirCompactorHasNotUploadedBlocks
+ annotations:
+ message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has not uploaded any block in the last 24 hours.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircompactorhasnotuploadedblocks
+ expr: |
+ (time() - (max by(cluster, namespace, pod) (thanos_objstore_bucket_last_successful_upload_time{component="compactor"})) > 60 * 60 * 24)
+ and
+ (max by(cluster, namespace, pod) (thanos_objstore_bucket_last_successful_upload_time{component="compactor"}) > 0)
+ and
+ # Only if some compactions have started. We don't want to fire this alert if the compactor has nothing to do
+ # (e.g. there are more replicas than required because running as part of mimir-backend).
+ (sum by(cluster, namespace, pod) (rate(cortex_compactor_group_compaction_runs_started_total[24h])) > 0)
+ for: 15m
+ labels:
+ severity: critical
+ time_period: 24h
+ - alert: MimirCompactorHasNotUploadedBlocks
+ annotations:
+ message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has not uploaded any block since its start.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircompactorhasnotuploadedblocks
+ expr: |
+ (max by(cluster, namespace, pod) (thanos_objstore_bucket_last_successful_upload_time{component="compactor"}) == 0)
+ and
+ # Only if some compactions have started. We don't want to fire this alert if the compactor has nothing to do
+ # (e.g. there are more replicas than required because running as part of mimir-backend).
+ (sum by(cluster, namespace, pod) (rate(cortex_compactor_group_compaction_runs_started_total[24h])) > 0)
+ for: 24h
+ labels:
+ severity: critical
+ time_period: since-start
+ - alert: MimirCompactorSkippedUnhealthyBlocks
+ annotations:
+ message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has found and ignored unhealthy blocks.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircompactorskippedunhealthyblocks
+ expr: |
+ increase(cortex_compactor_blocks_marked_for_no_compaction_total[5m]) > 0
+ for: 1m
+ labels:
+ severity: warning
+ - alert: MimirCompactorSkippedUnhealthyBlocks
+ annotations:
+ message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} has found and ignored unhealthy blocks.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircompactorskippedunhealthyblocks
+ expr: |
+ increase(cortex_compactor_blocks_marked_for_no_compaction_total[5m]) > 1
+ for: 30m
+ labels:
+ severity: critical
+- name: mimir_autoscaling
+ rules:
+ - alert: MimirAutoscalerNotActive
+ annotations:
+ message: The Horizontal Pod Autoscaler (HPA) {{ $labels.horizontalpodautoscaler
+ }} in {{ $labels.namespace }} is not active.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirautoscalernotactive
+ expr: |
+ (
+ label_replace((
+ kube_horizontalpodautoscaler_status_condition{condition="ScalingActive",status="false"}
+ # Match only Mimir namespaces.
+ * on(cluster, namespace) group_left max by(cluster, namespace) (cortex_build_info)
+ # Add "metric" label.
+ + on(cluster, namespace, horizontalpodautoscaler) group_right label_replace(kube_horizontalpodautoscaler_spec_target_metric*0, "metric", "$1", "metric_name", "(.+)")
+ > 0),
+ "scaledObject", "$1", "horizontalpodautoscaler", "keda-hpa-(.*)"
+ )
+ )
+ # Alert only if the scaling metric exists and is > 0. If the KEDA ScaledObject is configured to scale down 0,
+ # then HPA ScalingActive may be false when expected to run 0 replicas. In this case, the scaling metric exported
+ # by KEDA could not exist at all or being exposed with a value of 0.
+ and on (cluster, namespace, metric, scaledObject)
+ (label_replace(keda_scaler_metrics_value, "namespace", "$0", "exported_namespace", ".+") > 0)
+ for: 1h
+ labels:
+ severity: critical
+ - alert: MimirAutoscalerKedaFailing
+ annotations:
+ message: The Keda ScaledObject {{ $labels.scaledObject }} in {{ $labels.namespace
+ }} is experiencing errors.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirautoscalerkedafailing
+ expr: |
+ (
+ # Find KEDA scalers reporting errors.
+ label_replace(rate(keda_scaler_errors[5m]), "namespace", "$1", "exported_namespace", "(.*)")
+ # Match only Mimir namespaces.
+ * on(cluster, namespace) group_left max by(cluster, namespace) (cortex_build_info)
+ )
+ > 0
+ for: 1h
+ labels:
+ severity: critical
+- name: mimir_ingest_storage_alerts
+ rules:
+ - alert: MimirIngesterLastConsumedOffsetCommitFailed
+ annotations:
+ message: Mimir {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is failing to commit the last consumed offset.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterlastconsumedoffsetcommitfailed
+ expr: |
+ sum by(cluster, namespace, pod) (rate(cortex_ingest_storage_reader_offset_commit_failures_total[5m]))
+ /
+ sum by(cluster, namespace, pod) (rate(cortex_ingest_storage_reader_offset_commit_requests_total[5m]))
+ > 0.2
+ for: 15m
+ labels:
+ severity: critical
+ - alert: MimirIngesterFailedToReadRecordsFromKafka
+ annotations:
+ message: Mimir {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is failing to read records from Kafka.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterfailedtoreadrecordsfromkafka
+ expr: |
+ sum by(cluster, namespace, pod, node_id) (rate(cortex_ingest_storage_reader_read_errors_total[1m]))
+ > 0
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirIngesterKafkaFetchErrorsRateTooHigh
+ annotations:
+ message: Mimir {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} is receiving fetch errors when reading records from Kafka.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterkafkafetcherrorsratetoohigh
+ expr: |
+ sum by (cluster, namespace, pod) (rate (cortex_ingest_storage_reader_fetch_errors_total[5m]))
+ /
+ sum by (cluster, namespace, pod) (rate (cortex_ingest_storage_reader_fetches_total[5m]))
+ > 0.1
+ for: 15m
+ labels:
+ severity: critical
+ - alert: MimirStartingIngesterKafkaReceiveDelayIncreasing
+ annotations:
+ message: Mimir {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} in "starting" phase is not reducing consumption lag of write requests read
+ from Kafka.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirstartingingesterkafkareceivedelayincreasing
+ expr: |
+ deriv((
+ sum by (cluster, namespace, pod) (rate(cortex_ingest_storage_reader_receive_delay_seconds_sum{phase="starting"}[1m]))
+ /
+ sum by (cluster, namespace, pod) (rate(cortex_ingest_storage_reader_receive_delay_seconds_count{phase="starting"}[1m]))
+ )[5m:1m]) > 0
+ for: 5m
+ labels:
+ severity: warning
+ - alert: MimirRunningIngesterReceiveDelayTooHigh
+ annotations:
+ message: Mimir {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} in "running" phase is too far behind in its consumption of write requests
+ from Kafka.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirrunningingesterreceivedelaytoohigh
+ expr: |
+ (
+ sum by (cluster, namespace, pod) (rate(cortex_ingest_storage_reader_receive_delay_seconds_sum{phase="running"}[1m]))
+ /
+ sum by (cluster, namespace, pod) (rate(cortex_ingest_storage_reader_receive_delay_seconds_count{phase="running"}[1m]))
+ ) > (10 * 60)
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirIngesterFailsToProcessRecordsFromKafka
+ annotations:
+ message: Mimir {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} fails to consume write requests read from Kafka due to internal errors.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterfailstoprocessrecordsfromkafka
+ expr: |
+ sum by (cluster, namespace, pod) (rate(cortex_ingest_storage_reader_records_failed_total{cause="server"}[1m])) > 0
+ for: 5m
+ labels:
+ severity: critical
+ - alert: MimirIngesterFailsEnforceStrongConsistencyOnReadPath
+ annotations:
+ message: Mimir {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
+ }} fails to enforce strong-consistency on read-path.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimiringesterfailsenforcestrongconsistencyonreadpath
+ expr: |
+ sum by (cluster, namespace, pod) (rate(cortex_ingest_storage_strong_consistency_failures_total[1m])) > 0
+ for: 5m
+ labels:
+ severity: critical
+- name: mimir_continuous_test
+ rules:
+ - alert: MimirContinuousTestNotRunningOnWrites
+ annotations:
+ message: Mimir continuous test {{ $labels.test }} in {{ $labels.cluster }}/{{
+ $labels.namespace }} is not effectively running because writes are failing.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircontinuoustestnotrunningonwrites
+ expr: |
+ sum by(cluster, namespace, test) (rate(mimir_continuous_test_writes_failed_total[5m])) > 0
+ for: 1h
+ labels:
+ severity: warning
+ - alert: MimirContinuousTestNotRunningOnReads
+ annotations:
+ message: Mimir continuous test {{ $labels.test }} in {{ $labels.cluster }}/{{
+ $labels.namespace }} is not effectively running because queries are failing.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircontinuoustestnotrunningonreads
+ expr: |
+ sum by(cluster, namespace, test) (rate(mimir_continuous_test_queries_failed_total[5m])) > 0
+ for: 1h
+ labels:
+ severity: warning
+ - alert: MimirContinuousTestFailed
+ annotations:
+ message: Mimir continuous test {{ $labels.test }} in {{ $labels.cluster }}/{{
+ $labels.namespace }} failed when asserting query results.
+ runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimircontinuoustestfailed
+ expr: |
+ sum by(cluster, namespace, test) (rate(mimir_continuous_test_query_result_checks_failed_total[10m])) > 0
+ labels:
+ severity: warning
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-alertmanager-resources.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-alertmanager-resources.json
new file mode 100644
index 0000000..18d924e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-alertmanager-resources.json
@@ -0,0 +1,697 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Alertmanager",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?alertmanager.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?alertmanager.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Network",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"alertmanager\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk writes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"alertmanager\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk reads",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Disk",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 12,
+ "targets": [
+ {
+ "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(alertmanager).*\"\n }\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{persistentvolumeclaim}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk space utilization",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Alertmanager resources",
+ "uid": "a6883fb22799ac74479c7db872451092",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-alertmanager.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-alertmanager.json
new file mode 100644
index 0000000..92596ba
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-alertmanager.json
@@ -0,0 +1,2444 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "100px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "short",
+ "id": 1,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(cluster_job_pod:cortex_alertmanager_alerts:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Total alerts",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "short",
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(cluster_job_pod:cortex_alertmanager_silences:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Total silences",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "short",
+ "id": 3,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "max(cortex_alertmanager_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Tenants",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Headlines",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "QPS",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Alertmanager Distributor",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 12,
+ "targets": [
+ {
+ "expr": "sum(cluster_job:cortex_alertmanager_alerts_received_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "APS",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Alerts received",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 12,
+ "targets": [
+ {
+ "expr": "cortex_alertmanager_dispatcher_aggregation_groups{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "per pod Active Aggregation Groups",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Alerts grouping",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "NPS",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "(\nsum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) by(integration)\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) by(integration)\n) > 0\nor on () vector(0)\n",
+ "format": "time_series",
+ "legendFormat": "success - {{ integration }}",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) by(integration)",
+ "format": "time_series",
+ "legendFormat": "failed - {{ integration }}",
+ "legendLink": null
+ }
+ ],
+ "title": "NPS by integration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Alert notifications",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Operations / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "max": 1,
+ "min": 0,
+ "noValue": "0",
+ "unit": "percentunit"
+ }
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval])) >= 0",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Error rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Attributes",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Exists",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Alertmanager Configuration Object Store (Alertmanager accesses)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Get",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: GetRange",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Upload",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Delete",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (cortex_alertmanager_tenants_owned{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod tenants",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 20,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_alerts:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod alerts",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 21,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_silences:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod silences",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Replication",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 22,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_alertmanager_sync_configs_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_sync_configs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_alertmanager_sync_configs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Syncs/sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 23,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(reason) (rate(cortex_alertmanager_sync_configs_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{reason}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Syncs/sec (by reason)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 24,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum (rate(cortex_alertmanager_ring_check_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "errors",
+ "legendLink": null
+ }
+ ],
+ "title": "Ring check errors/sec",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Tenant configuration sync",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 25,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(outcome) (rate(cortex_alertmanager_state_initial_sync_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "{{outcome}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Initial syncs /sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 26,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Initial sync duration",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 27,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Fetch state from other alertmanagers /sec",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Sharding initial state sync",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 28,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(cluster_job:cortex_alertmanager_state_replication_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Replicate state to other alertmanagers /sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 29,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Merge state from other alertmanagers /sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 30,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_alertmanager_state_persist_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_persist_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_alertmanager_state_persist_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Persist state to remote storage /sec",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Sharding runtime state sync",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Alertmanager",
+ "uid": "b0d38d318bbddd80476246d4930f9e55",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-compactor-resources.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-compactor-resources.json
new file mode 100644
index 0000000..3ca6721
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-compactor-resources.json
@@ -0,0 +1,810 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "CPU and memory",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (RSS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?compactor.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?compactor.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Network",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk writes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk reads",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(compactor).*\"\n }\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{persistentvolumeclaim}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk space utilization",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Disk",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Compactor resources",
+ "uid": "09a5c49e9cdb2f2b24c6d184574a07fd",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-compactor.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-compactor.json
new file mode 100644
index 0000000..7a80a90
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-compactor.json
@@ -0,0 +1,2161 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Per-instance runs\nNumber of times a compactor instance triggers a compaction across all tenants that it manages.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "completed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "started"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#34CCEB",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_compactor_runs_started_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "started",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_compactor_runs_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "completed",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_compactor_runs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Per-instance runs / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Tenants compaction progress\nIn a multi-tenant cluster, display the progress of tenants that are compacted while compaction is running.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "max": 1,
+ "noValue": 1,
+ "unit": "percentunit"
+ }
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "(\n cortex_compactor_tenants_processing_succeeded{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} +\n cortex_compactor_tenants_processing_failed{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} +\n cortex_compactor_tenants_skipped{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}\n)\n/\ncortex_compactor_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} > 0\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Tenants compaction progress",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### Longest time since last successful run\nDisplays the amount of time since the most recent successful execution\nof the compactor.\nThe value shown will be for the compactor replica that has the longest time since its\nlast successful run.\nThe table to the right shows a summary for all compactor replicas.\n\nIf there is no time value, one of the following messages might appear:\n\n- If you see \"No compactor data\" in this panel, that means that no compactors are active yet.\n\n- If you see \"No successful runs\" in this panel, that means that compactors are active, but none\n of them were successfully executed yet.\n\nThese might be expected - for example, if you just recently restarted your compactors,\nthey might not have had a chance to complete their first compaction run.\nHowever, if these messages persist, you should check the health of your compactors.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 1,
+ "noValue": "No compactor data",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Last run"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 74
+ },
+ {
+ "id": "mappings",
+ "value": [
+ {
+ "options": {
+ "from": "-Infinity",
+ "result": {
+ "color": "text",
+ "text": "No successful runs since startup yet"
+ },
+ "to": 0
+ },
+ "type": "range"
+ }
+ ]
+ },
+ {
+ "id": "color",
+ "value": {
+ "mode": "thresholds"
+ }
+ },
+ {
+ "id": "thresholds",
+ "value": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "yellow",
+ "value": 7200
+ },
+ {
+ "color": "orange",
+ "value": 21600
+ },
+ {
+ "color": "red",
+ "value": 43200
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "fill": 1,
+ "id": 3,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "reduceOptions": {
+ "calcs": [
+ "first"
+ ],
+ "fields": "/^Last run$/",
+ "values": false
+ },
+ "textMode": "value"
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 3,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "max by(pod)\n(\n (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h]) !=bool 0))\n -\n max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h])\n)\n",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Longest time since last successful run",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transformations": [
+ {
+ "id": "organize",
+ "options": {
+ "renameByName": {
+ "Value": "Last run",
+ "pod": "Compactor"
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "sort": [
+ {
+ "desc": true,
+ "field": "Last run"
+ }
+ ]
+ }
+ }
+ ],
+ "type": "stat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Last successful run per-compactor replica\nDisplays the compactor replicas, and for each, shows how long it has been since\nits last successful compaction run.\n\nThe value in the status column is based on how long it has been since the last successful compaction.\n\n- Okay: less than 2 hours\n- Delayed: more than 2 hours\n- Late: more than 6 hours\n- Very late: more than 12 hours\n\nIf the status of any compactor replicas are *Late* or *Very late*, check their health.\n\n",
+ "fieldConfig": {
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Status"
+ },
+ "properties": [
+ {
+ "id": "custom.displayMode",
+ "value": "color-background"
+ },
+ {
+ "id": "mappings",
+ "value": [
+ {
+ "options": {
+ "from": "-Infinity",
+ "result": {
+ "color": "transparent",
+ "text": "N/A"
+ },
+ "to": 0
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "color": "green",
+ "text": "Ok"
+ },
+ "to": 7200
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 7200,
+ "result": {
+ "color": "yellow",
+ "text": "Delayed"
+ },
+ "to": 21600
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 21600,
+ "result": {
+ "color": "orange",
+ "text": "Late"
+ },
+ "to": 43200
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 43200,
+ "result": {
+ "color": "red",
+ "text": "Very late"
+ },
+ "to": "Infinity"
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "match": "null+nan",
+ "result": {
+ "color": "transparent",
+ "text": "Unknown"
+ }
+ },
+ "type": "special"
+ }
+ ]
+ },
+ {
+ "id": "custom.width",
+ "value": 86
+ },
+ {
+ "id": "custom.align",
+ "value": "center"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Last run"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "s"
+ },
+ {
+ "id": "custom.width",
+ "value": 74
+ },
+ {
+ "id": "mappings",
+ "value": [
+ {
+ "options": {
+ "from": "-Infinity",
+ "result": {
+ "text": "Never"
+ },
+ "to": 0
+ },
+ "type": "range"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "max by(pod)\n(\n (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h]) !=bool 0))\n -\n max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h])\n)\n",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "Last run",
+ "legendLink": null
+ }
+ ],
+ "title": "Last successful run per-compactor replica",
+ "transformations": [
+ {
+ "id": "organize",
+ "options": {
+ "renameByName": {
+ "Value": "Last run",
+ "pod": "Compactor"
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "sort": [
+ {
+ "desc": true,
+ "field": "Last run"
+ }
+ ]
+ }
+ },
+ {
+ "id": "calculateField",
+ "options": {
+ "alias": "One",
+ "binary": {
+ "left": "Last run",
+ "operator": "/",
+ "right": "Last run"
+ },
+ "mode": "binary",
+ "replaceFields": false
+ }
+ },
+ {
+ "id": "calculateField",
+ "options": {
+ "alias": "Status",
+ "binary": {
+ "left": "Last run",
+ "operator": "*",
+ "right": "One"
+ },
+ "mode": "binary",
+ "replaceFields": false
+ }
+ },
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "names": [
+ "Compactor",
+ "Last run",
+ "Status"
+ ]
+ }
+ }
+ }
+ ],
+ "type": "table"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Summary",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Estimated Compaction Jobs\nEstimated number of compaction jobs based on latest version of bucket index. Ingesters upload new blocks every 2 hours (shortly after 01:00 UTC, 03:00 UTC, 05:00 UTC, etc.),\nand compactors should process all of them within 2h interval. If this graph regularly goes to zero (or close to zero) in 2 hour intervals, then compaction works as designed.\n\nMetric with number of compaction jobs is computed from blocks in bucket index, which is updated regularly. Metric doesn't change between bucket index updates, even if\nthere were compaction jobs finished in this time. When computing compaction jobs, only jobs that can be executed at given moment are counted. There can be more\njobs, but if they are blocked, they are not counted in the metric. For example if there is a split compaction job pending for some time range, no merge job\ncovering the same time range can run. In this case only split compaction job is counted toward the metric, but merge job isn't.\n\nIn other words, computed number of compaction jobs is the minimum number of compaction jobs based on latest version of bucket index.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(cortex_bucket_index_estimated_compaction_jobs{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}) and (sum(rate(cortex_bucket_index_estimated_compaction_jobs_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) == 0)",
+ "format": "time_series",
+ "legendFormat": "Jobs",
+ "legendLink": null
+ }
+ ],
+ "title": "Estimated Compaction Jobs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### TSDB compactions / sec\nRate of TSDB compactions. Single TSDB compaction takes one or more input blocks and produces one or more (during \"split\" phase) output blocks.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(prometheus_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "compactions",
+ "legendLink": null
+ }
+ ],
+ "title": "TSDB compactions / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### TSDB compaction duration\nDisplay the amount of time that it has taken to run a single TSDB compaction.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "TSDB compaction duration",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "avg(max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ }
+ ],
+ "title": "Average blocks / tenant",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Tenants with largest number of blocks\nThe 10 tenants with the largest number of blocks.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "topk(10, max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}))",
+ "format": "time_series",
+ "legendFormat": "{{user}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Tenants with largest number of blocks",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "blocks",
+ "legendLink": null
+ }
+ ],
+ "title": "Blocks marked for deletion / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_compactor_blocks_cleaned_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Blocks deletions / sec",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Garbage collector",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_compactor_meta_syncs_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Metadata syncs / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Metadata sync duration",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Metadata sync",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Operations / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "max": 1,
+ "min": 0,
+ "noValue": "0",
+ "unit": "percentunit"
+ }
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) >= 0",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Error rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Attributes",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Exists",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Object Store",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Get",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: GetRange",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 20,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Upload",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 21,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Delete",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 22,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 23,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Key-value store for compactors ring",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Compactor",
+ "uid": "1b3443aea86db629e6efdb7d05c53823",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-config.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-config.json
new file mode 100644
index 0000000..069ce8d
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-config.json
@@ -0,0 +1,258 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "instances"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 12,
+ "targets": [
+ {
+ "expr": "count(cortex_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)",
+ "format": "time_series",
+ "legendFormat": "sha256:{{sha256}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Startup config file hashes",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Startup config file",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "instances"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 12,
+ "targets": [
+ {
+ "expr": "count(cortex_runtime_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)",
+ "format": "time_series",
+ "legendFormat": "sha256:{{sha256}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Runtime config file hashes",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Runtime config file",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Config",
+ "uid": "5d9d0b4724c0f80d68467088ec61e003",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-object-store.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-object-store.json
new file mode 100644
index 0000000..456b849
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-object-store.json
@@ -0,0 +1,822 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{component}}",
+ "legendLink": null
+ }
+ ],
+ "title": "RPS / component",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "max": 1,
+ "min": 0,
+ "noValue": "0",
+ "unit": "percentunit"
+ }
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >= 0",
+ "format": "time_series",
+ "legendFormat": "{{component}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Error rate / component",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Components",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "RPS / operation",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "max": 1,
+ "min": 0,
+ "noValue": "0",
+ "unit": "percentunit"
+ }
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >= 0",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Error rate / operation",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Operations",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Op: Get",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Op: GetRange",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Op: Exists",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Op: Attributes",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Op: Upload",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Op: Delete",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Object Store",
+ "uid": "e1324ee2a434f4158c00a9ee279d3292",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overrides.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overrides.json
new file mode 100644
index 0000000..708e1b2
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overrides.json
@@ -0,0 +1,266 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "${datasource}",
+ "id": 1,
+ "span": 12,
+ "targets": [
+ {
+ "expr": "max by(limit_name) (cortex_limits_defaults{cluster=~\"$cluster\",namespace=~\"$namespace\"})",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Defaults",
+ "transformations": [
+ {
+ "id": "labelsToFields",
+ "options": { }
+ },
+ {
+ "id": "merge",
+ "options": { }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": true
+ },
+ "indexByName": {
+ "Value": 1,
+ "limit_name": 0
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "fields": { },
+ "sort": [
+ {
+ "field": "limit_name"
+ }
+ ]
+ }
+ }
+ ],
+ "type": "table"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "${datasource}",
+ "id": 2,
+ "span": 12,
+ "targets": [
+ {
+ "expr": "max by(user, limit_name) (cortex_limits_overrides{cluster=~\"$cluster\",namespace=~\"$namespace\",user=~\"${tenant_id}\"})",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Per-tenant overrides",
+ "transformations": [
+ {
+ "id": "labelsToFields",
+ "options": {
+ "mode": "columns",
+ "valueLabel": "limit_name"
+ }
+ },
+ {
+ "id": "merge",
+ "options": { }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": true
+ },
+ "indexByName": {
+ "user": 0
+ }
+ }
+ }
+ ],
+ "type": "table"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": ".*",
+ "value": ".*"
+ },
+ "hide": 0,
+ "label": "Tenant ID",
+ "name": "tenant_id",
+ "options": [
+ {
+ "selected": true,
+ "text": ".*",
+ "value": ".*"
+ }
+ ],
+ "query": ".*",
+ "type": "textbox"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Overrides",
+ "uid": "1e2c358600ac53f09faea133f811b5bb",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overview-networking.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overview-networking.json
new file mode 100644
index 0000000..451d487
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overview-networking.json
@@ -0,0 +1,820 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Writes",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Reads",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Backend",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Overview networking",
+ "uid": "e15c71d372cc541367a088f10d9fcd92",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overview-resources.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overview-resources.json
new file mode 100644
index 0000000..b5b33b8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overview-resources.json
@@ -0,0 +1,922 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Writes",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"distributor|ingester|mimir-write\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk writes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"distributor|ingester|mimir-write\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk reads",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(distributor|ingester|mimir-write).*\"\n }\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{persistentvolumeclaim}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk space utilization",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Reads",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Backend",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk writes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk reads",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"\n }\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{persistentvolumeclaim}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk space utilization",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Overview resources",
+ "uid": "a9b92d3c4d1af325d872a9e9a7083d71",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overview.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overview.json
new file mode 100644
index 0000000..1f16a96
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-overview.json
@@ -0,0 +1,1600 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "content": "The 'Status' panel shows an overview on the cluster health over the time.\nTo investigate failures, see a specific dashboard:\n\n- Writes\n- Reads\n- Rule evaluations\n- Alerting notifications\n- Object storage\n",
+ "datasource": null,
+ "description": "",
+ "id": 1,
+ "mode": "markdown",
+ "span": 3,
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#7EB26D",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 0.01
+ },
+ {
+ "color": "#E24D42",
+ "value": 0.050000000000000003
+ }
+ ]
+ }
+ }
+ },
+ "id": 2,
+ "options": {
+ "showValue": "never"
+ },
+ "span": 6,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "exemplar": false,
+ "expr": "(\n # gRPC errors are not tracked as 5xx but \"error\".\n sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"5.*|error\"}[$__rate_interval]))\n or\n # Handle the case no failure has been tracked yet.\n vector(0)\n)\n/\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
+ "instant": false,
+ "legendFormat": "Writes",
+ "range": true
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "exemplar": false,
+ "expr": "(\n sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.*\"}[$__rate_interval]))\n or\n # Handle the case no failure has been tracked yet.\n vector(0)\n)\n/\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
+ "instant": false,
+ "legendFormat": "Reads",
+ "range": true
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "exemplar": false,
+ "expr": "(\n (\n sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n +\n # Consider missed evaluations as failures.\n sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n )\n or\n # Handle the case no failure has been tracked yet.\n vector(0)\n)\n/\nsum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "instant": false,
+ "legendFormat": "Rule evaluations",
+ "range": true
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "exemplar": false,
+ "expr": "(\n # Failed notifications from ruler to Alertmanager (handling the case the ruler metrics are missing).\n ((sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n) or vector(0))\n +\n # Failed notifications from Alertmanager to receivers (handling the case the alertmanager metrics are missing).\n ((sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n) or vector(0))\n)\n/\n(\n # Total notifications from ruler to Alertmanager (handling the case the ruler metrics are missing).\n ((sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n) or vector(0))\n +\n # Total notifications from Alertmanager to receivers (handling the case the alertmanager metrics are missing).\n ((sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n) or vector(0))\n)\n",
+ "instant": false,
+ "legendFormat": "Alerting notifications",
+ "range": true
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "exemplar": false,
+ "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n",
+ "instant": false,
+ "legendFormat": "Object storage",
+ "range": true
+ }
+ ],
+ "title": "Status",
+ "type": "state-timeline"
+ },
+ {
+ "id": 3,
+ "options": {
+ "alertInstanceLabelFilter": "cluster=~\"$cluster\", namespace=~\"$namespace\"",
+ "alertName": "Mimir",
+ "dashboardAlerts": false,
+ "maxItems": 100,
+ "sortOrder": 3,
+ "stateFilter": {
+ "error": true,
+ "firing": true,
+ "noData": false,
+ "normal": false,
+ "pending": false
+ }
+ },
+ "span": 3,
+ "title": "Firing alerts",
+ "type": "alertlist"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Mimir cluster health",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "content": "These panels show an overview on the write path. \nTo examine the write path in detail, see a specific dashboard:\n\n- Writes\n- Writes resources\n- Writes networking\n- Overview resources\n- Overview networking\n",
+ "datasource": null,
+ "description": "",
+ "id": 4,
+ "mode": "markdown",
+ "span": 3,
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Write requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Write latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "cps"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(cluster_namespace_job:cortex_distributor_received_samples:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
+ "format": "time_series",
+ "legendFormat": "samples / sec",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(cluster_namespace_job:cortex_distributor_received_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
+ "format": "time_series",
+ "legendFormat": "exemplars / sec",
+ "legendLink": null
+ }
+ ],
+ "title": "Ingestion / sec",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Writes",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "content": "These panels show an overview on the read path. \nTo examine the read path in detail, see a specific dashboard:\n\n- Reads\n- Reads resources\n- Reads networking\n- Overview resources\n- Overview networking\n- Queries\n- Compactor\n",
+ "datasource": null,
+ "description": "",
+ "id": 8,
+ "mode": "markdown",
+ "span": 3,
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Read requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Read latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_query($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "instant queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#429D48",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_query_range($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "range queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#F1C731",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_labels($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "\"label names\" queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#2A66CF",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_label_name_values($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "\"label values\" queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#9E44C1",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_series($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "series queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFAB57",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_read($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "remote read queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#C79424",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_metadata($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "metadata queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#84D586",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_query_exemplars($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "exemplar queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A1C4FC",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_cardinality_active_series($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "\"active series\" queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#C788DE",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_cardinality_label_names($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "\"label name cardinality\" queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#3F6833",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/.*_api_v1_cardinality_label_values($|[^_])/"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "\"label value cardinality\" queries"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#447EBC",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by (route) (rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)(_api_v1_query|_api_v1_query_range|_api_v1_labels|_api_v1_label_name_values|_api_v1_series|_api_v1_read|_api_v1_metadata|_api_v1_query_exemplars|_api_v1_cardinality_active_series|_api_v1_cardinality_label_names|_api_v1_cardinality_label_values)\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_.*\",route!~\"(prometheus|api_prom)(_api_v1_query|_api_v1_query_range|_api_v1_labels|_api_v1_label_name_values|_api_v1_series|_api_v1_read|_api_v1_metadata|_api_v1_query_exemplars|_api_v1_cardinality_active_series|_api_v1_cardinality_label_names|_api_v1_cardinality_label_values)\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "other",
+ "legendLink": null
+ }
+ ],
+ "title": "Queries / sec",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Reads",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "content": "These panels show an overview on the recording and alerting rules evaluation.\nTo examine the rules evaluation and alerts notifications in detail, see a specific dashboard:\n\n- Ruler\n- Alertmanager\n- Alertmanager resources\n- Overview resources\n- Overview networking\n",
+ "datasource": null,
+ "description": "",
+ "id": 12,
+ "mode": "markdown",
+ "span": 3,
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "success",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "missed",
+ "legendLink": null
+ }
+ ],
+ "title": "Rule evaluations / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "average",
+ "legendLink": null
+ }
+ ],
+ "title": "Rule evaluations latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n -\nsum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Alerting notifications sent to Alertmanager / sec",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Recording and alerting rules",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "content": "These panels show an overview on the long-term storage (object storage).\nTo examine the storage in detail, see a specific dashboard:\n\n- Object store\n- Compactor\n",
+ "datasource": null,
+ "description": "",
+ "id": 16,
+ "mode": "markdown",
+ "span": 3,
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n-\nsum(rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "seriesOverrides": [
+ {
+ "alias": "attributes",
+ "color": "#429D48"
+ },
+ {
+ "alias": "delete",
+ "color": "#F1C731"
+ },
+ {
+ "alias": "exists",
+ "color": "#2A66CF"
+ },
+ {
+ "alias": "get",
+ "color": "#9E44C1"
+ },
+ {
+ "alias": "get_range",
+ "color": "#FFAB57"
+ },
+ {
+ "alias": "iter",
+ "color": "#C79424"
+ },
+ {
+ "alias": "upload",
+ "color": "#84D586"
+ }
+ ],
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Operations / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(max by(user) (max_over_time(cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[15m])))",
+ "format": "time_series",
+ "legendFormat": "blocks",
+ "legendLink": null
+ }
+ ],
+ "title": "Total number of blocks in the storage",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Long-term storage (object storage)",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Overview",
+ "uid": "ffcd83628d7d4b5a03d1cafd159e6c9c",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-queries.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-queries.json
new file mode 100644
index 0000000..940ef79
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-queries.json
@@ -0,0 +1,2535 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Queue duration",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_query_frontend_retries_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Retries",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (cortex_query_frontend_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Queue length (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(user) (cortex_query_frontend_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}) > 0",
+ "format": "time_series",
+ "legendFormat": "{{user}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Queue length (per user)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-frontend",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Queue duration",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Queue length (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(user) (cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}) > 0",
+ "format": "time_series",
+ "legendFormat": "{{user}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Queue length (per user)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-scheduler",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Intervals per query\nThe average number of split queries (partitioned by time) executed a single input query.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_frontend_split_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", method=\"split_by_interval_and_results_cache\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "splitting rate",
+ "legendLink": null
+ }
+ ],
+ "title": "Intervals per query",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "# Query the new metric introduced in Mimir 2.10.\n(\n sum by(request_type) (rate(cortex_frontend_query_result_cache_hits_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n /\n sum by(request_type) (rate(cortex_frontend_query_result_cache_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n)\n# Otherwise fallback to the previous general-purpose metrics.\nor\n(\n label_replace(\n # Query metrics before and after dskit cache refactor.\n sum (\n rate(thanos_cache_memcached_hits_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_hits_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n )\n /\n sum (\n rate(thanos_cache_memcached_requests_total{name=~\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_requests_total{name=~\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n ),\n \"request_type\", \"query_range\", \"\", \"\")\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{request_type}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Query results cache hit ratio",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Query results cache skipped\nThe % of queries whose results could not be cached.\nIt is tracked for each split query when the splitting by interval is enabled.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_frontend_query_result_cache_skipped_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (reason) /\nignoring (reason) group_left sum(rate(cortex_frontend_query_result_cache_attempted_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "{{reason}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Query results cache skipped",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-frontend - query splitting and results cache",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Sharded queries ratio\nThe % of queries that have been successfully rewritten and executed in a shardable way.\nThis panel only takes into account the type of queries that are supported by query sharding (eg. range queries).\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_frontend_query_sharding_rewrites_succeeded_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) /\nsum(rate(cortex_frontend_query_sharding_rewrites_attempted_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "sharded queries ratio",
+ "legendLink": null
+ }
+ ],
+ "title": "Sharded queries ratio",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Number of sharded queries per query\nThe number of sharded queries that have been executed for a single input query. It only tracks queries that\nhave been successfully rewritten in a shardable way.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_frontend_sharded_queries_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_frontend_sharded_queries_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Number of sharded queries per query",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-frontend - query sharding",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1 * sum(cluster_job:cortex_ingester_queried_series_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}) / sum(cluster_job:cortex_ingester_queried_series_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Series per query",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1 * sum(cluster_job:cortex_ingester_queried_samples_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}) / sum(cluster_job:cortex_ingester_queried_samples_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Samples per query",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1 * sum(cluster_job:cortex_ingester_queried_exemplars_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}) / sum(cluster_job:cortex_ingester_queried_exemplars_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Exemplars per query",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingester",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Number of store-gateways hit per query",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Refetches of missing blocks per query",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Consistency checks failed\nRate of queries that had to run with consistency checks and those checks failed. A failed consistency check means that some of at least one block which had to be queried wasn't present in any of the store-gateways.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Failure Rate"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Failure Rate",
+ "legendLink": null
+ }
+ ],
+ "title": "Consistency checks failed",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Rejected queries\nThe proportion of all queries received by queriers that were rejected for some reason.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by (reason) (rate(cortex_querier_queries_rejected_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) / ignoring (reason) group_left sum(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_query(_range)?\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{reason}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Rejected queries",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Querier",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 20,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"})",
+ "format": "time_series",
+ "legendFormat": "Max",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"})",
+ "format": "time_series",
+ "legendFormat": "Min",
+ "legendLink": null
+ },
+ {
+ "expr": "avg(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "legendLink": null
+ }
+ ],
+ "title": "Bucket indexes loaded (per querier)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 21,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_bucket_index_loads_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) - sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Bucket indexes load / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 22,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Bucket indexes load latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
+ "id": 23,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "blocks",
+ "legendLink": null
+ }
+ ],
+ "title": "Blocks queried / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": [ ]
+ },
+ "id": 24,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(data_type) (\n # Exclude \"chunks refetched\".\n rate(cortex_bucket_store_series_data_size_fetched_bytes_sum{component=\"store-gateway\", stage!=\"refetched\", cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{data_type}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Data fetched / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": [ ]
+ },
+ "id": 25,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(data_type) (\n # Exclude \"chunks processed\" to only count \"chunks returned\", other than postings and series.\n rate(cortex_bucket_store_series_data_size_touched_bytes_sum{component=\"store-gateway\", stage!=\"processed\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{data_type}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Data touched / sec",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Store-gateway",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 26,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(stage) (rate(cortex_bucket_store_series_request_stage_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum by(stage) (rate(cortex_bucket_store_series_request_stage_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "{{stage}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Series request average latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 27,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by(stage, le) (rate(cortex_bucket_store_series_request_stage_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])))\n",
+ "format": "time_series",
+ "legendFormat": "{{stage}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Series request 99th percentile latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Series batch preloading efficiency\nThis panel shows the % of time reduced by preloading, for Series() requests which have been\nsplit to 2+ batches. If a Series() request is served within a single batch, then preloading\nis not triggered, and thus not counted in this measurement.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 28,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "# Clamping min to 0 because if preloading not useful at all, then the actual value we get is\n# slightly negative because of the small overhead introduced by preloading.\nclamp_min(1 - (\n sum(rate(cortex_bucket_store_series_batch_preloading_wait_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\n sum(rate(cortex_bucket_store_series_batch_preloading_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n), 0)\n",
+ "format": "time_series",
+ "legendFormat": "% of time reduced by preloading",
+ "legendLink": null
+ }
+ ],
+ "title": "Series batch preloading efficiency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Blocks currently owned\nThis panel shows the number of blocks owned by each store-gateway replica.\nFor each owned block, the store-gateway keeps its index-header on disk, and\neventually loaded in memory (if index-header lazy loading is disabled, or lazy loading\nis enabled and the index-header was loaded).\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 29,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "cortex_bucket_store_blocks_loaded{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Blocks currently owned",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 30,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_bucket_store_block_loads_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Blocks loaded / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 31,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_bucket_store_block_drops_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Blocks dropped / sec",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 32,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "cortex_bucket_store_indexheader_lazy_load_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"} - cortex_bucket_store_indexheader_lazy_unload_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Lazy loaded index-headers",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 33,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Index-header lazy load duration",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Index-header lazy load gate latency\nTime spent waiting for a turn to load an index header. This time is not included in \"Index-header lazy load duration.\"\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 34,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_stores_gate_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_stores_gate_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_bucket_stores_gate_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_stores_gate_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Index-header lazy load gate latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 35,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_bucket_store_series_hash_cache_hits_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum(rate(cortex_bucket_store_series_hash_cache_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "hit ratio",
+ "legendLink": null
+ }
+ ],
+ "title": "Series hash cache hit ratio",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 36,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(thanos_store_index_cache_hits_total{item_type=\"ExpandedPostings\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum(rate(thanos_store_index_cache_requests_total{item_type=\"ExpandedPostings\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "hit ratio",
+ "legendLink": null
+ }
+ ],
+ "title": "ExpandedPostings cache hit ratio",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 37,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_cache_memory_hits_total{name=\"chunks-attributes-cache\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum(rate(cortex_cache_memory_requests_total{name=\"chunks-attributes-cache\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "hit ratio",
+ "legendLink": null
+ }
+ ],
+ "title": "Chunks attributes in-memory cache hit ratio",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Queries",
+ "uid": "b3abe8d5c040395cc36615cb4334c92d",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-reads-networking.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-reads-networking.json
new file mode 100644
index 0000000..1bab7d0
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-reads-networking.json
@@ -0,0 +1,1504 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Summary",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-frontend",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-scheduler",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Querier",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 20,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Store-gateway",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 21,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 22,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 23,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 24,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Reads networking",
+ "uid": "54b2a0a4748b3bd1aefa92ce5559a1c2",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-reads-resources.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-reads-resources.json
new file mode 100644
index 0000000..2e8f82f
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-reads-resources.json
@@ -0,0 +1,2449 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Summary",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-frontend",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-scheduler",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Querier",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingester",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (RSS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Rules",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 20,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 21,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 22,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Store-gateway",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 23,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (RSS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 24,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 25,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"store-gateway\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk writes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 26,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"store-gateway\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk reads",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 27,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(store-gateway).*\"\n }\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{persistentvolumeclaim}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk space utilization",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Reads resources",
+ "uid": "cc86fd5aa9301c6528986572ad974db9",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-reads.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-reads.json
new file mode 100644
index 0000000..ddc9af1
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-reads.json
@@ -0,0 +1,4737 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "175px",
+ "panels": [
+ {
+ "content": "\n This dashboard shows health metrics for the read path.\n It is broken into sections for each service on the read path, and organized by the order in which the read request flows.\n
\n Incoming queries travel from the gateway → query frontend → query scheduler → querier → ingester and/or store-gateway (depending on the time range of the query).\n
\n For each service, there are 3 panels showing (1) requests per second to that service, (2) average, median, and p99 latency of requests to that service, and (3) p99 latency of requests to each instance of that service.\n
\n\n The dashboard also shows metrics for the 4 optional caches that can be deployed:\n the query results cache, the metadata cache, the chunks cache, and the index cache.\n
\n These panels will show “no data” if the caches are not deployed.\n
\n\n Lastly, it also includes metrics for how the ingester and store-gateway interact with object storage.\n
\n",
+ "datasource": null,
+ "description": "",
+ "id": 1,
+ "mode": "markdown",
+ "span": 12,
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Reads dashboard description",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "100px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### Instant queries per second\nRate of instant queries per second being made to the system.\nIncludes both queries made to the /prometheus API as\nwell as queries from the ruler.\n\n",
+ "fill": 1,
+ "format": "reqps",
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",\n route=~\"(prometheus|api_prom)_api_v1_query\"\n }[$__rate_interval]\n )\n or\n rate(\n cortex_prometheus_rule_evaluations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Instant queries / sec",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### Range queries per second\nRate of range queries per second being made to\nMimir via the /prometheus API.\n\n",
+ "fill": 1,
+ "format": "reqps",
+ "id": 3,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_query_range\"}[$__rate_interval]))",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Range queries / sec",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### \"Label names\" queries per second\nRate of \"label names\" endpoint queries per second being made to\nMimir via the /prometheus API.\n\n",
+ "fill": 1,
+ "format": "reqps",
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_labels\"}[$__rate_interval]))",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Label names queries / sec",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### \"Label values\" queries per second\nRate of specific \"label values\" endpoint queries per second being made to\nMimir via the /prometheus API.\n\n",
+ "fill": 1,
+ "format": "reqps",
+ "id": 5,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_label_name_values\"}[$__rate_interval]))",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Label values queries / sec",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### Series queries per second\nRate of series queries per second being made to\nMimir via the /prometheus API.\n\n",
+ "fill": 1,
+ "format": "reqps",
+ "id": 6,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_series\"}[$__rate_interval]))",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Series queries / sec",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Headlines",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "displayMode": "hidden",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
+ "format": "time_series",
+ "legendFormat": "",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod p99 latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-frontend",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Requests / sec\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Latency (Time in Queue)\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency (Time in Queue)",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Queue length\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "queries"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "displayMode": "hidden",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(min_over_time(cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__interval]))",
+ "format": "time_series",
+ "legendFormat": "Queue length",
+ "legendLink": null
+ }
+ ],
+ "title": "Queue length",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-scheduler",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### 99th Percentile Latency by Queue Dimension\n\n The query scheduler can optionally create subqueues\n in order to enforce round-robin query queuing fairness\n across additional queue dimensions beyond the default.\n\n By default, query queuing fairness is only applied by tenant ID.\n Queries without additional queue dimensions are labeled 'none'.\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "label_replace(histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
+ "format": "time_series",
+ "legendFormat": "99th Percentile: {{ additional_queue_dimensions }}",
+ "refId": "A"
+ }
+ ],
+ "title": "99th Percentile Latency by Queue Dimension",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### 50th Percentile Latency by Queue Dimension\n\n The query scheduler can optionally create subqueues\n in order to enforce round-robin query queuing fairness\n across additional queue dimensions beyond the default.\n\n By default, query queuing fairness is only applied by tenant ID.\n Queries without additional queue dimensions are labeled 'none'.\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "label_replace(histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
+ "format": "time_series",
+ "legendFormat": "50th Percentile: {{ additional_queue_dimensions }}",
+ "refId": "A"
+ }
+ ],
+ "title": "50th Percentile Latency by Queue Dimension",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Average Latency by Queue Dimension\n\n The query scheduler can optionally create subqueues\n in order to enforce round-robin query queuing fairness\n across additional queue dimensions beyond the default.\n\n By default, query queuing fairness is only applied by tenant ID.\n Queries without additional queue dimensions are labeled 'none'.\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "label_replace(sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (additional_queue_dimensions) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (additional_queue_dimensions), \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
+ "format": "time_series",
+ "legendFormat": "Average: {{ additional_queue_dimensions }}",
+ "refId": "C"
+ }
+ ],
+ "title": "Average Latency by Queue Dimension",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Query-scheduler Latency (Time in Queue) Breakout by Additional Queue Dimensions",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum (\n rate(thanos_memcached_operations_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "legendFormat": "Requests/s",
+ "legendLink": null
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) by (le)) * 1e3\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) by (le)) * 1e3\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Cache – query results",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 20,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "displayMode": "hidden",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
+ "format": "time_series",
+ "legendFormat": "",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod p99 latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Querier",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 21,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 22,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 23,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "displayMode": "hidden",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/(QueryStream|QueryExemplars|LabelValues|LabelNames|UserStats|AllUserStats|MetricsForLabelMatchers|MetricsMetadata|LabelNamesAndValues|LabelValuesCardinality|ActiveSeries)\"}[$__rate_interval])))",
+ "format": "time_series",
+ "legendFormat": "",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod p99 latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingester",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 24,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 25,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 26,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "displayMode": "hidden",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval])))",
+ "format": "time_series",
+ "legendFormat": "",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod p99 latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Store-gateway",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Replicas\nThe maximum, and current number of querier replicas.\nPlease note that the current number of replicas can still show 1 replica even when scaled to 0.\nSince HPA never reports 0 replicas, the query will report 0 only if the HPA is not active.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/Max .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/Current .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/Min .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 27,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
+ "format": "time_series",
+ "legendFormat": "Max {{ scaletargetref_name }}",
+ "legendLink": null
+ },
+ {
+ "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n # HPA doesn't go to 0 replicas, so we multiply by 0 if the HPA is not active.\n * on (cluster, namespace, horizontalpodautoscaler)\n kube_horizontalpodautoscaler_status_condition{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\", condition=\"ScalingActive\", status=\"true\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
+ "format": "time_series",
+ "legendFormat": "Current {{ scaletargetref_name }}",
+ "legendLink": null
+ },
+ {
+ "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
+ "format": "time_series",
+ "legendFormat": "Min {{ scaletargetref_name }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Replicas",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Scaling metric (desired replicas)\nThis panel shows the result scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints which are applied later.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 28,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left\n label_replace(label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ), \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\")\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{ scaler }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Scaling metric (desired replicas)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 29,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{scaler}} failures",
+ "legendLink": null
+ }
+ ],
+ "title": "Autoscaler failures rate",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Querier - autoscaling",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 30,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 31,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Store-gateway – key-value store for store-gateways ring",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
+ "id": 32,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(operation) (\n # Backwards compatibility\n rate(\n thanos_memcached_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n or ignoring(backend)\n rate(\n thanos_cache_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 33,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency (getmulti)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Hit ratio\nEven if you do not set up memcached for the blocks index cache, you will still see data in this panel because the store-gateway by default has an\nin-memory blocks index cache.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 34,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(item_type) (\n rate(\n thanos_store_index_cache_hits_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n/\nsum by(item_type) (\n rate(\n thanos_store_index_cache_requests_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{item_type}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Hit ratio",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Memcached – block index cache (store-gateway accesses)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
+ "id": 35,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 36,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency (getmulti)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 37,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "legendFormat": "items",
+ "legendLink": null
+ }
+ ],
+ "title": "Hit ratio",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Memcached – chunks cache (store-gateway accesses)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
+ "id": 38,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 39,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency (getmulti)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 40,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "legendFormat": "items",
+ "legendLink": null
+ }
+ ],
+ "title": "Hit ratio",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Memcached – metadata cache (store-gateway accesses)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
+ "id": 41,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 42,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency (getmulti)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 43,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
+ "format": "time_series",
+ "legendFormat": "items",
+ "legendLink": null
+ }
+ ],
+ "title": "Hit ratio",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Memcached – metadata cache (querier accesses)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [ ]
+ },
+ "id": 44,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Operations / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "max": 1,
+ "min": 0,
+ "noValue": "0",
+ "unit": "percentunit"
+ }
+ },
+ "id": 45,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) >= 0",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Error rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 46,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Attributes",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 47,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Exists",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Blocks object store (store-gateway accesses)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 48,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Get",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 49,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: GetRange",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 50,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Upload",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 51,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Delete",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [ ]
+ },
+ "id": 52,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Operations / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "max": 1,
+ "min": 0,
+ "noValue": "0",
+ "unit": "percentunit"
+ }
+ },
+ "id": 53,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) >= 0",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Error rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 54,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Attributes",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 55,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Exists",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Blocks object store (querier accesses)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 56,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Get",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 57,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: GetRange",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 58,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Upload",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 59,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Delete",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Reads",
+ "uid": "e327503188913dc38ad571c647eef643",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-remote-ruler-reads-networking.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-remote-ruler-reads-networking.json
new file mode 100644
index 0000000..28c919f
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-remote-ruler-reads-networking.json
@@ -0,0 +1,1048 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(ruler-query-frontend|ruler-query-scheduler|ruler-querier).*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Summary",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-frontend.*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-query-frontend",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-query-scheduler.*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-query-scheduler",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler-querier.*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-querier",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Remote ruler reads networking",
+ "uid": "9e8cfff65f91632f8a25981c6fe44bc9",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-remote-ruler-reads-resources.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-remote-ruler-reads-resources.json
new file mode 100644
index 0000000..c079baa
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-remote-ruler-reads-resources.json
@@ -0,0 +1,982 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-query-frontend",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-query-scheduler",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-querier",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Remote ruler reads resources",
+ "uid": "1940f6ef765a506a171faa2056c956c3",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-remote-ruler-reads.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-remote-ruler-reads.json
new file mode 100644
index 0000000..3b509ef
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-remote-ruler-reads.json
@@ -0,0 +1,1683 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "175px",
+ "panels": [
+ {
+ "content": "\n This dashboard shows health metrics for the ruler read path when remote operational mode is enabled.\n It is broken into sections for each service on the ruler read path, and organized by the order in which the read request flows.\n
\n For each service, there are three panels showing (1) requests per second to that service, (2) average, median, and p99 latency of requests to that service, and (3) p99 latency of requests to each instance of that service.\n
\n",
+ "datasource": null,
+ "description": "",
+ "id": 1,
+ "mode": "markdown",
+ "span": 12,
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Remote ruler reads dashboard description",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "100px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### Evaluations per second\nRate of rule expressions evaluated per second.\n\n",
+ "fill": 1,
+ "format": "reqps",
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\",\n route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"\n }[$__rate_interval]\n )\n)\n",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Evaluations / sec",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Headlines",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "displayMode": "hidden",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"}[$__rate_interval])))",
+ "format": "time_series",
+ "legendFormat": "",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod p99 latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-query-frontend",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Requests / sec\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Latency (Time in Queue)\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency (Time in Queue)",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Queue length\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "queries"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "displayMode": "hidden",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(min_over_time(cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__interval]))",
+ "format": "time_series",
+ "legendFormat": "Queue length",
+ "legendLink": null
+ }
+ ],
+ "title": "Queue length",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-query-scheduler",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### 99th Percentile Latency by Queue Dimension\n\n The query scheduler can optionally create subqueues\n in order to enforce round-robin query queuing fairness\n across additional queue dimensions beyond the default.\n\n By default, query queuing fairness is only applied by tenant ID.\n Queries without additional queue dimensions are labeled 'none'.\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "label_replace(histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
+ "format": "time_series",
+ "legendFormat": "99th Percentile: {{ additional_queue_dimensions }}",
+ "refId": "A"
+ }
+ ],
+ "title": "99th Percentile Latency by Queue Dimension",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### 50th Percentile Latency by Queue Dimension\n\n The query scheduler can optionally create subqueues\n in order to enforce round-robin query queuing fairness\n across additional queue dimensions beyond the default.\n\n By default, query queuing fairness is only applied by tenant ID.\n Queries without additional queue dimensions are labeled 'none'.\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "label_replace(histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
+ "format": "time_series",
+ "legendFormat": "50th Percentile: {{ additional_queue_dimensions }}",
+ "refId": "A"
+ }
+ ],
+ "title": "50th Percentile Latency by Queue Dimension",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Average Latency by Queue Dimension\n\n The query scheduler can optionally create subqueues\n in order to enforce round-robin query queuing fairness\n across additional queue dimensions beyond the default.\n\n By default, query queuing fairness is only applied by tenant ID.\n Queries without additional queue dimensions are labeled 'none'.\n
\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "label_replace(sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (additional_queue_dimensions) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (additional_queue_dimensions), \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
+ "format": "time_series",
+ "legendFormat": "Average: {{ additional_queue_dimensions }}",
+ "refId": "C"
+ }
+ ],
+ "title": "Average Latency by Queue Dimension",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-query-scheduler Latency (Time in Queue) Breakout by Additional Queue Dimensions",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "displayMode": "hidden",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
+ "format": "time_series",
+ "legendFormat": "",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod p99 latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-querier",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Replicas\nThe maximum and current number of ruler-querier replicas.\nNote: The current number of replicas can still show 1 replica even when scaled to 0.\nBecause HPA never reports 0 replicas, the query will report 0 only if the HPA is not active.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/Max .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/Current .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/Min .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n)\n",
+ "format": "time_series",
+ "legendFormat": "Max {{ scaletargetref_name }}",
+ "legendLink": null
+ },
+ {
+ "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n # HPA doesn't go to 0 replicas, so we multiply by 0 if the HPA is not active\n * on (cluster, namespace, horizontalpodautoscaler)\n kube_horizontalpodautoscaler_status_condition{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\", condition=\"ScalingActive\", status=\"true\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n)\n",
+ "format": "time_series",
+ "legendFormat": "Current {{ scaletargetref_name }}",
+ "legendLink": null
+ },
+ {
+ "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n)\n",
+ "format": "time_series",
+ "legendFormat": "Min {{ scaletargetref_name }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Replicas",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{scaler}} failures",
+ "legendLink": null
+ }
+ ],
+ "title": "Autoscaler failures rate",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler-querier - autoscaling",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Scaling metric (CPU): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{ scaler }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Scaling metric (CPU): Desired replicas",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Scaling metric (memory): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{ scaler }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Scaling metric (memory): Desired replicas",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Scaling metric (in-flight queries): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*queries.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{ scaler }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Scaling metric (in-flight queries): Desired replicas",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Remote ruler reads",
+ "uid": "f103238f7f5ab2f1345ce650cbfbfe2f",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-rollout-progress.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-rollout-progress.json
new file mode 100644
index 0000000..734a630
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-rollout-progress.json
@@ -0,0 +1,1404 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "fillOpacity": 80,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineWidth": 1,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [ ],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Ready"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "green",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Updated"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "blue",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 13,
+ "w": 10,
+ "x": 0,
+ "y": 0
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "barRadius": 0,
+ "barWidth": 0.96999999999999997,
+ "fullHighlight": false,
+ "groupWidth": 0.69999999999999996,
+ "legend": {
+ "calcs": [ ],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "orientation": "horizontal",
+ "showValue": "auto",
+ "stacking": "none",
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ },
+ "xField": "Workload",
+ "xTickLabelRotation": 0,
+ "xTickLabelSpacing": 0
+ },
+ "targets": [
+ {
+ "expr": "(\n sum by (workload) (\n label_replace(label_replace(label_replace(\n kube_deployment_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n or\n kube_statefulset_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n , \"workload\", \"$1\", \"deployment\", \"(.+)\"), \"workload\", \"$1\", \"statefulset\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.*?)(?:-zone-[a-z])?\")\n )\n /\n sum by (workload) (\n label_replace(label_replace(label_replace(\n kube_deployment_status_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n or\n kube_statefulset_status_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n , \"workload\", \"$1\", \"deployment\", \"(.+)\"), \"workload\", \"$1\", \"statefulset\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.*?)(?:-zone-[a-z])?\")\n )\n) and (\n sum by (workload) (\n label_replace(label_replace(label_replace(\n kube_deployment_status_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n or\n kube_statefulset_status_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n , \"workload\", \"$1\", \"deployment\", \"(.+)\"), \"workload\", \"$1\", \"statefulset\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.*?)(?:-zone-[a-z])?\")\n )\n > 0\n)\n",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": null,
+ "legendFormat": "__auto",
+ "legendLink": null,
+ "step": null
+ },
+ {
+ "expr": "(\n sum by (workload) (\n label_replace(label_replace(label_replace(\n kube_deployment_status_replicas_ready{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n or\n kube_statefulset_status_replicas_ready{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n , \"workload\", \"$1\", \"deployment\", \"(.+)\"), \"workload\", \"$1\", \"statefulset\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.*?)(?:-zone-[a-z])?\")\n )\n /\n sum by (workload) (\n label_replace(label_replace(label_replace(\n kube_deployment_status_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n or\n kube_statefulset_status_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n , \"workload\", \"$1\", \"deployment\", \"(.+)\"), \"workload\", \"$1\", \"statefulset\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.*?)(?:-zone-[a-z])?\")\n )\n) and (\n sum by (workload) (\n label_replace(label_replace(label_replace(\n kube_deployment_status_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n or\n kube_statefulset_status_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n , \"workload\", \"$1\", \"deployment\", \"(.+)\"), \"workload\", \"$1\", \"statefulset\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.*?)(?:-zone-[a-z])?\")\n )\n > 0\n)\n",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": null,
+ "legendFormat": "__auto",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "title": "Rollout progress",
+ "transformations": [
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "workload",
+ "mode": "outer"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time 1": true,
+ "Time 2": true
+ },
+ "renameByName": {
+ "Value #A": "Updated",
+ "Value #B": "Ready",
+ "workload": "Workload"
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "sort": [
+ {
+ "field": "Workload"
+ }
+ ]
+ }
+ }
+ ],
+ "type": "barchart"
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 1,
+ "noValue": "",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 2,
+ "x": 10,
+ "y": 0
+ },
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
+ "format": null,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Writes - 2xx",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "stat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 1,
+ "noValue": "",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 0.20000000000000001
+ },
+ {
+ "color": "red",
+ "value": 0.5
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 2,
+ "x": 12,
+ "y": 0
+ },
+ "id": 3,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
+ "format": null,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Writes - 4xx",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "stat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 1,
+ "noValue": "",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 0.01
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 2,
+ "x": 14,
+ "y": 0
+ },
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
+ "format": null,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Writes - 5xx",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "stat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 1,
+ "noValue": "",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 0.20000000000000001
+ },
+ {
+ "color": "red",
+ "value": 0.5
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 8,
+ "x": 16,
+ "y": 0
+ },
+ "id": 5,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}))\n",
+ "format": null,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Writes 99th latency",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "stat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 1,
+ "noValue": "",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 2,
+ "x": 10,
+ "y": 4
+ },
+ "id": 6,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
+ "format": null,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Reads - 2xx",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "stat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 1,
+ "noValue": "",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 0.01
+ },
+ {
+ "color": "red",
+ "value": 0.050000000000000003
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 2,
+ "x": 12,
+ "y": 4
+ },
+ "id": 7,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
+ "format": null,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Reads - 4xx",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "stat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 1,
+ "noValue": "",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 0.01
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 2,
+ "x": 14,
+ "y": 4
+ },
+ "id": 8,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
+ "format": null,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Reads - 5xx",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "stat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 1,
+ "noValue": "",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 2.5
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 8,
+ "x": 16,
+ "y": 4
+ },
+ "id": 9,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))\n",
+ "format": null,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Reads 99th latency",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "stat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 0,
+ "noValue": "All healthy",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 2
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "fill": 1,
+ "gridPos": {
+ "h": 3,
+ "w": 10,
+ "x": 0,
+ "y": 13
+ },
+ "id": 10,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "text": {
+ "titleSize": 14,
+ "valueSize": 14
+ },
+ "textMode": "value_and_name"
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n> 0\n",
+ "format": null,
+ "instant": true,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "{{deployment}}",
+ "legendLink": null,
+ "step": null
+ },
+ {
+ "expr": "kube_statefulset_status_replicas_current{cluster=~\"$cluster\", namespace=~\"$namespace\"} -\nkube_statefulset_status_replicas_ready {cluster=~\"$cluster\", namespace=~\"$namespace\"}\n> 0\n",
+ "format": null,
+ "instant": true,
+ "interval": "",
+ "intervalFactor": null,
+ "legendFormat": "{{statefulset}}",
+ "legendLink": null,
+ "step": null
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Unhealthy pods",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "stat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "r.*"
+ },
+ "properties": [
+ {
+ "id": "custom.align",
+ "value": "center"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 6,
+ "x": 10,
+ "y": 8
+ },
+ "id": 11,
+ "targets": [
+ {
+ "expr": "count by(container, version) (\n label_replace(\n kube_pod_container_info{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"version\", \"$1\", \"image\", \".*:(.*)\"\n )\n)\n",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Pods count per version",
+ "transformations": [
+ {
+ "id": "labelsToFields",
+ "options": {
+ "valueLabel": "version"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": true
+ },
+ "indexByName": {
+ "Time": 0,
+ "container": 1
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "fields": { },
+ "sort": [
+ {
+ "field": "container"
+ }
+ ]
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "fillOpacity": 10
+ },
+ "unit": "percentunit"
+ }
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 8
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}))[1h:])\n)\n",
+ "format": "time_series",
+ "legendFormat": "writes",
+ "legendLink": null
+ },
+ {
+ "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))[1h:])\n)\n",
+ "format": "time_series",
+ "legendFormat": "reads",
+ "legendLink": null
+ }
+ ],
+ "title": "Latency vs 24h ago",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "10s",
+ "rows": null,
+ "schemaVersion": 27,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Rollout progress",
+ "uid": "7f0b5567d543a1698e695b530eb7f5de",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-ruler.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-ruler.json
new file mode 100644
index 0000000..ea91503
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-ruler.json
@@ -0,0 +1,2617 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "100px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "short",
+ "id": 1,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 3,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(cortex_ruler_managers_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Active configurations",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "short",
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 3,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Total rules",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### Read from ingesters - QPS\nNote: Even while operating in Remote ruler mode you will still see values for this panel.\n\nThis is because the metrics are inclusive of intermediate services and are showing the requests that ultimately reach the ingesters.\n\nFor a more detailed view of the read path when using remote ruler mode, see the Remote ruler reads dashboard.\n\n",
+ "fill": 1,
+ "format": "reqps",
+ "id": 3,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 3,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Read from ingesters - QPS",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "reqps",
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 3,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Write to ingesters - QPS",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Headlines",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "success",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "missed",
+ "legendLink": null
+ }
+ ],
+ "title": "Evaluations per second",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "average",
+ "legendLink": null
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Rule evaluations global",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "QPS",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Writes (ingesters)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "QPS",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Reads (ingesters)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler - key-value store for rulers ring",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Number of store-gateways hit per query",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Refetches of missing blocks per query",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Consistency checks failed\nRate of queries that had to run with consistency checks and those checks failed. A failed consistency check means that some of at least one block which had to be queried wasn't present in any of the store-gateways.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Failures / sec"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Failures / sec",
+ "legendLink": null
+ }
+ ],
+ "title": "Consistency checks failed",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler - blocks storage",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(user) (rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]) > 0)\n> 0\n",
+ "format": "time_series",
+ "legendFormat": "{{ user }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Delivery errors",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0\n",
+ "format": "time_series",
+ "legendFormat": "{{ user }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Queue length",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (user) (increase(cortex_prometheus_notifications_dropped_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0\n",
+ "format": "time_series",
+ "legendFormat": "{{ user }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Dropped",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Notifications",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(user) (rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0",
+ "format": "time_series",
+ "legendFormat": "{{ user }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Missed iterations",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 20,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "rate(cortex_prometheus_rule_group_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n /\nrate(cortex_prometheus_rule_group_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n",
+ "format": "time_series",
+ "legendFormat": "{{ user }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 21,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0",
+ "format": "time_series",
+ "legendFormat": "{{ rule_group }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Failures",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Group evaluations",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 22,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 12,
+ "targets": [
+ {
+ "expr": "sum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
+ "format": "time_series",
+ "legendFormat": "{{ user }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Rule evaluation per user",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [ ]
+ },
+ "id": 23,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Operations / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "max": 1,
+ "min": 0,
+ "noValue": "0",
+ "unit": "percentunit"
+ }
+ },
+ "id": 24,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval])) >= 0",
+ "format": "time_series",
+ "legendFormat": "{{operation}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Error rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 25,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Attributes",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 26,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Exists",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ruler configuration object store (ruler accesses)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 27,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Get",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 28,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: GetRange",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 29,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Upload",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 30,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency of op: Delete",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Ruler",
+ "uid": "631e15d5d85afb2ca8e35d62984eeaa0",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-scaling.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-scaling.json
new file mode 100644
index 0000000..b7b6182
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-scaling.json
@@ -0,0 +1,361 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "200px",
+ "panels": [
+ {
+ "id": 1,
+ "options": {
+ "content": "This dashboard identifies scaling-related issues by suggesting services that you might want to scale up.\nThe table that follows contains a suggested number of replicas and the reason why.\nIf the system is failing and depending on the reason, try scaling up to the specified number.\nThe specified numbers are intended as helpful guidelines when things go wrong, rather than prescriptive guidelines.\n\nReasons:\n- **sample_rate**: There are not enough replicas to handle the\n sample rate. Applies to distributor and ingesters.\n- **active_series**: There are not enough replicas\n to handle the number of active series. Applies to ingesters.\n- **cpu_usage**: There are not enough replicas\n based on the CPU usage of the jobs vs the resource requests.\n Applies to all jobs.\n- **memory_usage**: There are not enough replicas based on the memory\n usage vs the resource requests. Applies to all jobs.\n- **active_series_limits**: There are not enough replicas to hold 60% of the\n sum of all the per tenant series limits.\n- **sample_rate_limits**: There are not enough replicas to handle 60% of the\n sum of all the per tenant rate limits.\n",
+ "mode": "markdown"
+ },
+ "span": 12,
+ "title": "",
+ "type": "text"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Service scaling",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "400px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 0,
+ "desc": false
+ },
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "Required Replicas",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 0,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "Cluster",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "__name__",
+ "thresholds": [ ],
+ "type": "hidden",
+ "unit": "short"
+ },
+ {
+ "alias": "Cluster",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "cluster",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "Service",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "deployment",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "Namespace",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "namespace",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "Reason",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "reason",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "sort_desc(\n cluster_namespace_deployment_reason:required_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n > ignoring(reason) group_left\n cluster_namespace_deployment:actual_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\n",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Workload-based scaling",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Scaling",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Scaling",
+ "uid": "64bbad83507b7289b514725658e10352",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-slow-queries.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-slow-queries.json
new file mode 100644
index 0000000..240205b
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-slow-queries.json
@@ -0,0 +1,1463 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p99",
+ "legendLink": null
+ },
+ {
+ "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p50",
+ "legendLink": null
+ }
+ ],
+ "title": "Response time",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p99",
+ "legendLink": null
+ },
+ {
+ "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p50",
+ "legendLink": null
+ }
+ ],
+ "title": "Fetched series",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p99",
+ "legendLink": null
+ },
+ {
+ "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p50",
+ "legendLink": null
+ }
+ ],
+ "title": "Fetched chunks",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p99",
+ "legendLink": null
+ },
+ {
+ "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p50",
+ "legendLink": null
+ }
+ ],
+ "title": "Response size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p99",
+ "legendLink": null
+ },
+ {
+ "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p50",
+ "legendLink": null
+ }
+ ],
+ "title": "Time span",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "description": "### Query wall time\nSeconds per second spent by queriers evaluating queries.\nThis is roughly the product of the number of subqueries for a query and how long they took.\nIn increase in this metric means that queries take more resources from the query path to evaluate.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap query_wall_time_seconds [$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p99",
+ "legendLink": null
+ },
+ {
+ "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap query_wall_time_seconds [$__auto]) by ()",
+ "format": "time_series",
+ "legendFormat": "p50",
+ "legendLink": null
+ }
+ ],
+ "title": "Query wall time",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Across tenants",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by (user))",
+ "format": "time_series",
+ "legendFormat": "{{user}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 response time",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by (user))",
+ "format": "time_series",
+ "legendFormat": "{{user}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 fetched series",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by (user))",
+ "format": "time_series",
+ "legendFormat": "{{user}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 fetched chunks",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by (user))",
+ "format": "time_series",
+ "legendFormat": "{{user}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 response size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by (user))",
+ "format": "time_series",
+ "legendFormat": "{{user}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 time span",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "description": "### Query wall time\nSeconds per second spent by queriers evaluating queries.\nThis is roughly the product of the number of subqueries for a query and how long they took.\nIn increase in this metric means that queries take more resources from the query path to evaluate.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap query_wall_time_seconds [$__auto]) by (user))",
+ "format": "time_series",
+ "legendFormat": "{{user}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 query wall time",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Top 10 tenants",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by (user_agent))",
+ "format": "time_series",
+ "legendFormat": "{{user_agent}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 response time",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by (user_agent))",
+ "format": "time_series",
+ "legendFormat": "{{user_agent}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 fetched series",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by (user_agent))",
+ "format": "time_series",
+ "legendFormat": "{{user_agent}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 fetched chunks",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by (user_agent))",
+ "format": "time_series",
+ "legendFormat": "{{user_agent}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 response size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by (user_agent))",
+ "format": "time_series",
+ "legendFormat": "{{user_agent}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 time span",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${loki_datasource}",
+ "description": "### Query wall time\nSeconds per second spent by queriers evaluating queries.\nThis is roughly the product of the number of subqueries for a query and how long they took.\nIn increase in this metric means that queries take more resources from the query path to evaluate.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 2,
+ "targets": [
+ {
+ "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap query_wall_time_seconds [$__auto]) by (user_agent))",
+ "format": "time_series",
+ "legendFormat": "{{user_agent}}",
+ "legendLink": null
+ }
+ ],
+ "title": "P99 query wall time",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Top 10 User-Agents",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "${loki_datasource}",
+ "fieldConfig": {
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "fetched_chunk_bytes"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "fetched_index_bytes"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "response_size_bytes"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "results_cache_hit_bytes"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "results_cache_miss_bytes"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "estimated_series_count"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "short"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "fetched_chunks_count"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "short"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "fetched_series_count"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "short"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time span"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "s"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Duration"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "s"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Step"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "s"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "queue_time_seconds"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "s"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "query_wall_time_seconds"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "s"
+ }
+ ]
+ }
+ ]
+ },
+ "height": "500px",
+ "id": 19,
+ "span": 12,
+ "targets": [
+ {
+ "expr": "{cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | label_format response_time_seconds=\"{{ if .response_time }} {{ duration .response_time }} {{ end }}\",param_step_seconds=\"{{ if .param_step }} {{ div .param_step 1000 }} {{ end }}\",length_seconds=\"{{ if .length }} {{ duration .length }} {{ end }}\"",
+ "instant": false,
+ "legendFormat": "",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Slow queries",
+ "transformations": [
+ {
+ "id": "extractFields",
+ "options": {
+ "source": "labels"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Line": true,
+ "Time": true,
+ "caller": true,
+ "cluster": true,
+ "component": true,
+ "container": true,
+ "gossip_ring_member": true,
+ "host": true,
+ "id": true,
+ "job": true,
+ "labels": true,
+ "length": true,
+ "level": true,
+ "line": true,
+ "method": true,
+ "msg": true,
+ "name": true,
+ "namespace": true,
+ "param_step": true,
+ "path": true,
+ "pod": true,
+ "pod_template_hash": true,
+ "response_time": true,
+ "stream": true,
+ "traceID": true,
+ "tsNs": true
+ },
+ "indexByName": {
+ "err": 10,
+ "length_seconds": 3,
+ "param_end": 5,
+ "param_query": 8,
+ "param_start": 4,
+ "param_step_seconds": 7,
+ "param_time": 6,
+ "response_time_seconds": 9,
+ "status": 1,
+ "ts": 0,
+ "user": 2
+ },
+ "renameByName": {
+ "err": "Error",
+ "length_seconds": "Time span",
+ "param_end": "End",
+ "param_query": "Query",
+ "param_start": "Start",
+ "param_step_seconds": "Step",
+ "param_time": "Time (instant query)",
+ "response_time_seconds": "Duration",
+ "ts": "Completion date",
+ "user": "Tenant ID"
+ }
+ }
+ },
+ {
+ "id": "convertFieldType",
+ "options": {
+ "conversions": [
+ {
+ "destinationType": "number",
+ "targetField": "sharded_queries"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "split_queries"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "fetched_chunk_bytes"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "fetched_index_bytes"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "response_size_bytes"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "results_cache_hit_bytes"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "results_cache_miss_bytes"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "estimated_series_count"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "fetched_chunks_count"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "fetched_series_count"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "Time span"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "Duration"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "Step"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "queue_time_seconds"
+ },
+ {
+ "destinationType": "number",
+ "targetField": "query_wall_time_seconds"
+ }
+ ]
+ }
+ }
+ ],
+ "type": "table"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "hide": 0,
+ "includeAll": false,
+ "label": "Loki data source",
+ "multi": false,
+ "name": "loki_datasource",
+ "query": "loki",
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": "5s",
+ "value": "5s"
+ },
+ "hide": 0,
+ "label": "Min duration",
+ "name": "min_duration",
+ "options": [
+ {
+ "selected": true,
+ "text": "5s",
+ "value": "5s"
+ }
+ ],
+ "query": "5s",
+ "type": "textbox"
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": ".*",
+ "value": ".*"
+ },
+ "hide": 0,
+ "label": "Tenant ID",
+ "name": "tenant_id",
+ "options": [
+ {
+ "selected": true,
+ "text": ".*",
+ "value": ".*"
+ }
+ ],
+ "query": ".*",
+ "type": "textbox"
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": ".*",
+ "value": ".*"
+ },
+ "hide": 0,
+ "label": "User-Agent HTTP Header",
+ "name": "user_agent",
+ "options": [
+ {
+ "selected": true,
+ "text": ".*",
+ "value": ".*"
+ }
+ ],
+ "query": ".*",
+ "type": "textbox"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Slow queries",
+ "uid": "6089e1ce1e678788f46312a0a1e647e6",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-tenants.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-tenants.json
new file mode 100644
index 0000000..257327d
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-tenants.json
@@ -0,0 +1,2661 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "25px",
+ "panels": [
+ {
+ "content": "\n This dashboard shows various metrics detailed by tenant (user) selected above.\n
\n",
+ "datasource": null,
+ "description": "",
+ "id": 1,
+ "mode": "markdown",
+ "span": 12,
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Tenants dashboard description",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### All series\nNumber of active, in-memory, and owned series per user, and active series matching custom trackers (in parenthesis).\nNote that these counts include all series regardless of the type of data (counter, gauge, native histogram, etc.).\nNote that active series matching custom trackers are included in the total active series count.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum(\n (\n cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n - cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n )\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
+ "format": "time_series",
+ "legendFormat": "in-memory",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"max_global_series_per_user\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"max_global_series_per_user\"})\n",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(\n cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
+ "format": "time_series",
+ "legendFormat": "active",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(\n cortex_ingester_owned_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
+ "format": "time_series",
+ "legendFormat": "owned",
+ "legendLink": null
+ },
+ {
+ "expr": "sum by (name) (\n cortex_ingester_active_series_custom_tracker{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n) > 0\n",
+ "format": "time_series",
+ "legendFormat": "active ({{ name }})",
+ "legendLink": null
+ }
+ ],
+ "title": "All series",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### In-memory series per ingester\nLocal tenant series limit and number of in-memory series per ingester.\nBecause series can be unevenly distributed across ingesters, ingesters may hit the local limit at different times.\nNote that in-memory series may exceed the local limit if limiting based on owned series is enabled.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/local limit .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "yellow",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "min by (job) (cortex_ingester_local_limits{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", limit=\"max_global_series_per_user\", user=\"$user\"})\n",
+ "format": "time_series",
+ "legendFormat": "local limit ({{job}})",
+ "legendLink": null
+ },
+ {
+ "expr": "cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n- cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "In-memory series per ingester",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Owned series per ingester\nLocal tenant series limit and number of owned series per ingester.\nBecause series can be unevenly distributed across ingesters, ingesters may hit the local limit at different times.\nOwned series are the subset of an ingester's in-memory series that currently map to it in the ring\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/local limit .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "yellow",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "min by (job) (cortex_ingester_local_limits{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", limit=\"max_global_series_per_user\", user=\"$user\"})\n",
+ "format": "time_series",
+ "legendFormat": "local limit ({{job}})",
+ "legendLink": null
+ },
+ {
+ "expr": "cortex_ingester_owned_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Owned series per ingester",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Tenant series counts",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Series with exemplars\nNumber of series with exemplars currently in storage.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(\n cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
+ "format": "time_series",
+ "legendFormat": "series",
+ "legendLink": null
+ }
+ ],
+ "title": "Series with exemplars",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Oldest exemplar age\nThe age of the oldest exemplar stored in circular storage.\nUseful to check for what time range the current exemplar buffer limit allows.\nThis usually means the max age for all exemplars for a typical setup.\nThis is not true though if one of the series timestamp is in future compared to rest series.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "time() - min(cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "age",
+ "legendLink": null
+ }
+ ],
+ "title": "Oldest exemplar age",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Native histogram series\nNumber of active native histogram series per user, and active native histogram series matching custom trackers (in parenthesis).\nNote that active series matching custom trackers are included in the total active series count.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(\n cortex_ingester_active_native_histogram_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
+ "format": "time_series",
+ "legendFormat": "active",
+ "legendLink": null
+ },
+ {
+ "expr": "sum by (name) (\n cortex_ingester_active_native_histogram_series_custom_tracker{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n) > 0\n",
+ "format": "time_series",
+ "legendFormat": "active ({{ name }})",
+ "legendLink": null
+ }
+ ],
+ "title": "Native histogram series",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Total number of buckets used by native histogram series\nTotal number of buckets in active native histogram series per user, and total active native histogram buckets matching custom trackers (in parenthesis).\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(\n cortex_ingester_active_native_histogram_buckets{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
+ "format": "time_series",
+ "legendFormat": "buckets",
+ "legendLink": null
+ },
+ {
+ "expr": "sum by (name) (\n cortex_ingester_active_native_histogram_buckets_custom_tracker{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n) > 0\n",
+ "format": "time_series",
+ "legendFormat": "buckets ({{ name }})",
+ "legendLink": null
+ }
+ ],
+ "title": "Total number of buckets used by native histogram series",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Exemplars and native histograms",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor requests incoming rate\nThe rate of requests that have come in to the distributor, including rejected requests.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_distributor_requests_in_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "rate",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor requests incoming rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor requests received (accepted) rate\nThe rate of received requests, excluding rejected requests.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_distributor_received_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "rate",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"request_rate\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"request_rate\"})\n",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor requests received (accepted) rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Newest seen sample age\nThe age of the newest received sample seen in the distributors.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "time() - max(cortex_distributor_latest_seen_sample_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "age",
+ "legendLink": null
+ }
+ ],
+ "title": "Newest seen sample age",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor discarded requests rate\nThe rate of each request's discarding reason.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by (reason) (rate(cortex_discarded_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{ reason }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor discarded requests rate",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Distributor ingestion requests",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor samples incoming rate\nThe rate of samples that have come in to the distributor, including rejected or deduped exemplars.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_distributor_samples_in_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "rate",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor samples incoming rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor samples received (accepted) rate\nThe rate of received samples, excluding rejected and deduped samples.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "rate",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ingestion_rate\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ingestion_rate\"})\n",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor samples received (accepted) rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor deduplicated/non-HA\nThe rate of deduplicated samples and the rate of received samples for a user that has HA tracking turned on, but the sample didn't contain both HA labels.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_distributor_deduped_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "deduplicated",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_distributor_non_ha_samples_received_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "non-HA",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor deduplicated/non-HA",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor and ingester discarded samples rate\nThe rate of each sample's discarding reason.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by (reason) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{ reason }} (distributor)",
+ "legendLink": null
+ },
+ {
+ "expr": "sum by (reason) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{ reason }} (ingester)",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor and ingester discarded samples rate",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Samples ingestion funnel",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor exemplars incoming rate\nThe rate of exemplars that have come in to the distributor, including rejected or deduped exemplars.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_distributor_exemplars_in_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "rate",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor exemplars incoming rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor exemplars received (accepted) rate\nThe rate of received exemplars, excluding rejected and deduped exemplars.\nThis number can be sensibly lower than incoming rate because we dedupe the HA sent exemplars, and then reject based on time.\nSee discarded rate for reasons why exemplars are being discarded.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_distributor_received_exemplars_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "rate",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor exemplars received (accepted) rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor discarded exemplars rate\nThe rate of each exmplars' discarding reason.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by (reason) (rate(cortex_discarded_exemplars_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{ reason }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor discarded exemplars rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Ingester appended exemplars rate\nTotal number of exemplars appended in the ingesters.\nThis can be lower than ingested exemplars rate since TSDB does not append the same exemplar twice, and those can be frequent.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 20,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(\n rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval])\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
+ "format": "time_series",
+ "legendFormat": "rate",
+ "legendLink": null
+ }
+ ],
+ "title": "Ingester appended exemplars rate",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Exemplars ingestion funnel",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Symbol table size for loaded blocks\nSize of symbol table in memory for loaded blocks, averaged by ingester.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 21,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (job) (cortex_ingester_tsdb_symbol_table_size_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"})",
+ "format": "time_series",
+ "legendFormat": "{{ job }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Symbol table size for loaded blocks",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Space used by local blocks\nThe number of bytes that are currently used for local storage by all blocks.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 22,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (job) (cortex_ingester_tsdb_storage_blocks_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"})",
+ "format": "time_series",
+ "legendFormat": "{{ job }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Space used by local blocks",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingesters' storage",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Number of groups\nTotal number of rule groups for a tenant.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 23,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "count(sum by (rule_group) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}))",
+ "format": "time_series",
+ "legendFormat": "groups",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ruler_max_rule_groups_per_tenant\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ruler_max_rule_groups_per_tenant\"})\n",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "Number of groups",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Number of rules\nTotal number of rules for a tenant.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 24,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"})",
+ "format": "time_series",
+ "legendFormat": "rules",
+ "legendLink": null
+ }
+ ],
+ "title": "Number of rules",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 25,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "rate",
+ "legendLink": null
+ }
+ ],
+ "title": "Total evaluations rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 26,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by (rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) > 0",
+ "format": "time_series",
+ "legendFormat": "{{ rule_group }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Failed evaluations rate",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Rules",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 27,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "rules",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #A",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit, sum by (rule_group) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}))",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit biggest groups",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 28,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "seconds",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value #A",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit, sum by (rule_group) (cortex_prometheus_rule_group_last_duration_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}))",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit slowest groups (last evaluation)",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Top rules",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 29,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "rate",
+ "legendLink": null
+ }
+ ],
+ "title": "Sent notifications rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "rate"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 30,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "rate",
+ "legendLink": null
+ }
+ ],
+ "title": "Failed notifications rate",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Notifications",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 31,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (user) (cortex_alertmanager_alerts{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"})",
+ "format": "time_series",
+ "legendFormat": "alerts",
+ "legendLink": null
+ },
+ {
+ "expr": "sum by (user) (cortex_alertmanager_silences{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"})",
+ "format": "time_series",
+ "legendFormat": "silences",
+ "legendLink": null
+ }
+ ],
+ "title": "Alerts",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 32,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "(\nsum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))\n-\non() (sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) or on () vector(0))\n) > 0\n",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "NPS",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 33,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "(\nsum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration)\n-\n(sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration) or\n (sum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration) * 0)\n)) > 0\n",
+ "format": "time_series",
+ "legendFormat": "success - {{ integration }}",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration)",
+ "format": "time_series",
+ "legendFormat": "failed - {{ integration }}",
+ "legendLink": null
+ }
+ ],
+ "title": "NPS by integration",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Alertmanager",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 34,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Queries / Sec",
+ "legendLink": null
+ }
+ ],
+ "title": "Rate of Read Requests - query-frontend",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 35,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\", user=\"$user\"})",
+ "format": "time_series",
+ "legendFormat": "Queue Length",
+ "legendLink": null
+ }
+ ],
+ "title": "Number of Queries Queued - query-scheduler",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Read Path - Queries (User)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 36,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", user=\"$user\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Queries / Sec",
+ "legendLink": null
+ }
+ ],
+ "title": "Rate of Read Requests - ruler-query-frontend",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 37,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\", user=\"$user\"})",
+ "format": "time_series",
+ "legendFormat": "Queue Length",
+ "legendLink": null
+ }
+ ],
+ "title": "Number of Queries Queued - ruler-query-scheduler",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Read Path - Queries (Ruler)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Estimated Compaction Jobs\nEstimated number of compaction jobs for selected user, based on latest version of bucket index. When user sends data, ingesters upload new user blocks every 2 hours\n(shortly after 01:00 UTC, 03:00 UTC, 05:00 UTC, etc.), and compactors should process all of the blocks within 2h interval.\nIf this graph regularly goes to zero (or close to zero) in 2 hour intervals, then compaction for this user works correctly.\n\nDepending on the configuration, there are two types of jobs: `split` jobs and `merge` jobs. Split jobs will only show up when user is configured with positive number of `compactor_split_and_merge_shards`.\nValues for split and merge jobs are stacked.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 50,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 38,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (type) (cortex_bucket_index_estimated_compaction_jobs{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", user=\"$user\"})\nand ignoring(type)\n(sum(rate(cortex_bucket_index_estimated_compaction_jobs_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) == 0)\n",
+ "format": "time_series",
+ "legendFormat": "{{ job }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Estimated Compaction Jobs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Number of blocks\nNumber of blocks stored in long-term storage for this user.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 39,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "max by (user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", user=\"$user\"})\n",
+ "format": "time_series",
+ "legendFormat": "{{ job }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Blocks",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Compactions",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "user",
+ "multi": false,
+ "name": "user",
+ "options": [ ],
+ "query": "label_values(cortex_ingester_active_series{cluster=~\"$cluster\", namespace=~\"$namespace\"}, user)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": "10",
+ "value": "10"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "limit",
+ "options": [
+ {
+ "selected": true,
+ "text": "10",
+ "value": "10"
+ },
+ {
+ "selected": false,
+ "text": "50",
+ "value": "50"
+ },
+ {
+ "selected": false,
+ "text": "100",
+ "value": "100"
+ },
+ {
+ "selected": false,
+ "text": "500",
+ "value": "500"
+ },
+ {
+ "selected": false,
+ "text": "1000",
+ "value": "1000"
+ }
+ ],
+ "type": "custom"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Tenants",
+ "uid": "35fa247ce651ba189debf33d7ae41611",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-top-tenants.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-top-tenants.json
new file mode 100644
index 0000000..1d50197
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-top-tenants.json
@@ -0,0 +1,1639 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "25px",
+ "panels": [
+ {
+ "content": "\n This dashboard shows the top tenants based on multiple selection criterias.\n Rows are collapsed by default to avoid querying all of them.\n Use the templating variable \"limit\" above to select the amount of users to be shown.\n
\n",
+ "datasource": null,
+ "description": "",
+ "id": 1,
+ "mode": "markdown",
+ "span": 12,
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Top tenants dashboard description",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "series",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "user",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "user",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "string"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit,\n sum by (user) (\n cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n )\n)\n",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit users by active series",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By active series",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 3,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "series",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "user",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "user",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "string"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit, sum by (user) (\n (\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n -\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n )\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} )\n)\n)",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit users by in-memory series (series created - series removed)",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By in-memory series",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 12,
+ "targets": [
+ {
+ "expr": "sum by (user) (\n (\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n -\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n )\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} )\n)\n\nand\ntopk($limit, sum by (user) (\n (\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ end())\n -\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ end())\n )\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} @ end())\n)\n - sum by (user) (\n (\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ start())\n -\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ start())\n )\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} @ start())\n)\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{ user }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Top $limit users by in-memory series (series created - series removed) that grew the most between query range start and query range end",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By in-memory series growth",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 5,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "samples/s",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "user",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "user",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "string"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit, sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[5m])))",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit users by received samples rate in last 5m",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By samples rate",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 12,
+ "targets": [
+ {
+ "expr": "sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))\nand\ntopk($limit,\n sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ end()))\n -\n sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ start()))\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{ user }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Top $limit users by received samples rate that grew the most between query range start and query range end",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By samples rate growth",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 7,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "samples/s",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "user",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "user",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "string"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit, sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[5m])))",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit users by discarded samples rate in last 5m",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By discarded samples rate",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 12,
+ "targets": [
+ {
+ "expr": "sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))\nand\ntopk($limit,\n sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ end()))\n -\n sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ start()))\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{ user }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Top $limit users by discarded samples rate that grew the most between query range start and query range end",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By discarded samples rate growth",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 9,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "series",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "user",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "user",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "string"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit,\n sum by (user) (\n cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n )\n)\n",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit users by series with exemplars",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By series with exemplars",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 10,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "exemplars/s",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "user",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "user",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "string"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit, sum by (user) (rate(cortex_distributor_received_exemplars_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[5m])))",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit users by received exemplars rate in last 5m",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By exemplars rate",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 11,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 3,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "rules",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "user",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "user",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "string"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit, sum by (rule_group, user) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}))",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit biggest groups",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By rule group size",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 12,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 3,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "seconds",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "user",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "user",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "string"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit, sum by (rule_group, user) (cortex_prometheus_rule_group_last_duration_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}))",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit slowest groups (last evaluation)",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By rule group evaluation time",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "id": 13,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "Compaction Jobs",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 0,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "Value",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "user",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "link": false,
+ "linkTargetBlank": false,
+ "linkTooltip": "Drill down",
+ "linkUrl": "",
+ "pattern": "user",
+ "thresholds": [ ],
+ "type": "number",
+ "unit": "string"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [ ],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "topk($limit,\n sum by (user) (cortex_bucket_index_estimated_compaction_jobs{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"})\n and ignoring(user)\n (sum(rate(cortex_bucket_index_estimated_compaction_jobs_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) == 0)\n)\n",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top $limit users by estimated compaction jobs from bucket-index",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transform": "table",
+ "type": "table",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "By estimated compaction jobs from bucket-index",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": "10",
+ "value": "10"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "limit",
+ "options": [
+ {
+ "selected": true,
+ "text": "10",
+ "value": "10"
+ },
+ {
+ "selected": false,
+ "text": "50",
+ "value": "50"
+ },
+ {
+ "selected": false,
+ "text": "100",
+ "value": "100"
+ }
+ ],
+ "type": "custom"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Top tenants",
+ "uid": "bc6e12d4fe540e4a1785b9d3ca0ffdd9",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-writes-networking.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-writes-networking.json
new file mode 100644
index 0000000..5a757f8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-writes-networking.json
@@ -0,0 +1,820 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Summary",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Distributor",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Receive bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Transmit bandwidth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"})",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"})",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ }
+ ],
+ "title": "Inflight requests (per pod)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}))",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ },
+ {
+ "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}))",
+ "format": "time_series",
+ "legendFormat": "highest",
+ "legendLink": null
+ },
+ {
+ "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ }
+ ],
+ "title": "TCP connections (per pod)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingester",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Writes networking",
+ "uid": "978c1cb452585c96697a238eaac7fe2d",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-writes-resources.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-writes-resources.json
new file mode 100644
index 0000000..503a6b8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-writes-resources.json
@@ -0,0 +1,1186 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 1,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 2,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 3,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Summary",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 4,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 5,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 6,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Distributor",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (cortex_ingester_memory_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "In-memory series",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"cpu\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "CPU",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingester",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (RSS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "request"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#FFC000",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "limit"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E02F44",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ },
+ {
+ "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
+ "format": "time_series",
+ "legendFormat": "limit",
+ "legendLink": null
+ },
+ {
+ "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
+ "format": "time_series",
+ "legendFormat": "request",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (workingset)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
+ "format": "time_series",
+ "legendFormat": "{{pod}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Memory (go heap inuse)",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"ingester\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk writes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"ingester\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
+ "format": "time_series",
+ "legendFormat": "{{pod}} - {{device}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk reads",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "custom": {
+ "fillOpacity": 0
+ },
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(ingester).*\"\n }\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{persistentvolumeclaim}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Disk space utilization",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "prod",
+ "value": "prod"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": false,
+ "label": "namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Writes resources",
+ "uid": "bc9160e50b52e89e0e49c840fea3d379",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-writes.json b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-writes.json
new file mode 100644
index 0000000..c91b1b0
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/dashboards/mimir-writes.json
@@ -0,0 +1,3253 @@
+{
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "8.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 1,
+ "hideControls": false,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "mimir"
+ ],
+ "targetBlank": false,
+ "title": "Mimir dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "refresh": "10s",
+ "rows": [
+ {
+ "collapse": false,
+ "height": "125px",
+ "panels": [
+ {
+ "content": "\n This dashboard shows various health metrics for the write path.\n It is broken into sections for each service on the write path,\n and organized by the order in which the write request flows.\n
\n Incoming metrics data travels from the gateway → distributor → ingester.\n
\n For each service, there are 3 panels showing\n (1) requests per second to that service,\n (2) average, median, and p99 latency of requests to that service, and\n (3) p99 latency of requests to each instance of that service.\n
\n\n It also includes metrics for the key-value (KV) stores used to manage\n the high-availability tracker and the ingesters.\n
\n",
+ "datasource": null,
+ "description": "",
+ "id": 1,
+ "mode": "markdown",
+ "span": 12,
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Writes dashboard description",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "100px",
+ "panels": [
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "short",
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(cluster_namespace_job:cortex_distributor_received_samples:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Samples / sec",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### Exemplars / sec\nThe total number of received exemplars by the distributors, excluding rejected and deduped exemplars, but not necessarily ingested by the ingesters.\n\n",
+ "fill": 1,
+ "format": "short",
+ "id": 3,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(cluster_namespace_job:cortex_distributor_received_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Exemplars / sec",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### In-memory series\nThe number of series not yet flushed to object storage that are held in ingester memory.\n\n",
+ "fill": 1,
+ "format": "short",
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(cortex_ingester_memory_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n/ on(cluster, namespace) group_left\nmax by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}))\n",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "In-memory series",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "### Exemplars in ingesters\nNumber of TSDB exemplars currently in ingesters' storage.\n\n",
+ "fill": 1,
+ "format": "short",
+ "id": 5,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(cortex_ingester_tsdb_exemplar_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n/ on(cluster, namespace) group_left\nmax by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}))\n",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Exemplars in ingesters",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": { },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "format": "short",
+ "id": 6,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [ ],
+ "spaceLength": 10,
+ "span": 2,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "count(count by(user) (cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}))",
+ "format": "time_series",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "70,80",
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Tenants",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "singlestat",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [ ]
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Headlines",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Requests / sec\nThe rate of successful, failed and rejected requests to distributor.\nRejected requests are requests that distributor fails to handle because of distributor instance limits.\nWhen distributor is configured to use \"early\" request rejection, then rejected requests are NOT included in other metrics.\nWhen distributor is not configured to use \"early\" request rejection, then rejected requests are also counted as \"errors\".\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 7,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 8,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 9,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "displayMode": "hidden",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval])))",
+ "format": "time_series",
+ "legendFormat": "",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod p99 latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Distributor",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Requests / sec\nThe rate of successful, failed and rejected requests to ingester.\nRejected requests are requests that ingester fails to handle because of ingester instance limits (ingester-max-inflight-push-requests, ingester-max-inflight-push-requests-bytes, ingester-max-ingestion-rate).\nWhen ingester is configured to use \"early\" request rejection, then rejected requests are NOT included in other metrics.\nWhen ingester is not configured to use \"early\" request rejection, then rejected requests are also counted as \"errors\".\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 10,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 11,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 12,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "displayMode": "hidden",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "span": 4,
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"}[$__rate_interval])))",
+ "format": "time_series",
+ "legendFormat": "",
+ "legendLink": null
+ }
+ ],
+ "title": "Per pod p99 latency",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingester",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Replicas\nThe maximum and current number of distributor replicas.\nNote: The current number of replicas can still show 1 replica even when scaled to 0.\nBecause HPA never reports 0 replicas, the query will report 0 only if the HPA is not active.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/Max .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/Current .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/Min .+/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 13,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n)\n",
+ "format": "time_series",
+ "legendFormat": "Max {{ scaletargetref_name }}",
+ "legendLink": null
+ },
+ {
+ "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n # HPA doesn't go to 0 replicas, so we multiply by 0 if the HPA is not active\n * on (cluster, namespace, horizontalpodautoscaler)\n kube_horizontalpodautoscaler_status_condition{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\", condition=\"ScalingActive\", status=\"true\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n)\n",
+ "format": "time_series",
+ "legendFormat": "Current {{ scaletargetref_name }}",
+ "legendLink": null
+ },
+ {
+ "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n)\n",
+ "format": "time_series",
+ "legendFormat": "Min {{ scaletargetref_name }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Replicas",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Scaling metric (CPU): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 14,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{ scaler }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Scaling metric (CPU): Desired replicas",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Scaling metric (memory): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 15,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{ scaler }}",
+ "legendLink": null
+ }
+ ],
+ "title": "Scaling metric (memory): Desired replicas",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
+ "id": 16,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
+ "format": "time_series",
+ "legendFormat": "{{scaler}} failures",
+ "legendLink": null
+ }
+ ],
+ "title": "Autoscaler failures rate",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Distributor - autoscaling",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 17,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 18,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Distributor - key-value store for high-availability (HA) deduplication",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 19,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 20,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Distributor - key-value store for distributors ring",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 21,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 22,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingester - key-value store for the ingesters ring",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Uploaded blocks / sec\nThe rate of blocks being uploaded from the ingesters\nto object storage.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 23,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_ingester_shipper_uploads_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Uploaded blocks / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Upload latency\nThe average, median (50th percentile), and 99th percentile time\nthe ingesters take to upload blocks to object storage.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 24,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Upload latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingester - shipper",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Compactions per second\nIngesters maintain a local TSDB per-tenant on disk. Each TSDB maintains a head block for each\nactive time series; these blocks get periodically compacted (by default, every 2h).\nThis panel shows the rate of compaction operations across all TSDBs on all ingesters.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 25,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_ingester_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_ingester_tsdb_compactions_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Compactions / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Compaction latency\nThe average, median (50th percentile), and 99th percentile time ingesters take to compact TSDB head blocks\non the local filesystem.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
+ "id": 26,
+ "links": [ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "99th Percentile",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) by (le)) * 1e3",
+ "format": "time_series",
+ "legendFormat": "50th Percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "Average",
+ "refId": "C"
+ }
+ ],
+ "title": "Compactions latency",
+ "type": "timeseries",
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingester - TSDB head",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### WAL truncations per second\nThe WAL is truncated each time a new TSDB block is written. This panel measures the rate of\ntruncations.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 27,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "WAL truncations / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Checkpoints created per second\nCheckpoints are created as part of the WAL truncation process.\nThis metric measures the rate of checkpoint creation.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "successful"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 28,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "successful",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "failed",
+ "legendLink": null
+ }
+ ],
+ "title": "Checkpoints created / sec",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### WAL truncations latency (including checkpointing)\nAverage time taken to perform a full WAL truncation,\nincluding the time taken for the checkpointing to complete.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
+ "id": 29,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))\n/\nsum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) >= 0\n",
+ "format": "time_series",
+ "legendFormat": "avg",
+ "legendLink": null
+ }
+ ],
+ "title": "WAL truncations latency (includes checkpointing)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "WAL"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "mmap-ed chunks"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E28A42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "id": 30,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(rate(cortex_ingester_tsdb_wal_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "WAL",
+ "legendLink": null
+ },
+ {
+ "expr": "sum(rate(cortex_ingester_tsdb_mmap_chunk_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "mmap-ed chunks",
+ "legendLink": null
+ }
+ ],
+ "title": "Corruptions / sec",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ingester - TSDB write ahead log (WAL)",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor exemplars incoming rate\nThe rate of exemplars that have come in to the distributor, including rejected or deduped exemplars.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ex/s"
+ },
+ "overrides": [ ]
+ },
+ "id": 31,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(cluster_namespace_job:cortex_distributor_exemplars_in:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
+ "format": "time_series",
+ "legendFormat": "incoming exemplars",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor exemplars incoming rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Distributor exemplars received rate\nThe rate of received exemplars, excluding rejected and deduped exemplars.\nThis number can be sensibly lower than incoming rate because we dedupe the HA sent exemplars, and then reject based on time, see `cortex_discarded_exemplars_total` for specific reasons rates.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ex/s"
+ },
+ "overrides": [ ]
+ },
+ "id": 32,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(cluster_namespace_job:cortex_distributor_received_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
+ "format": "time_series",
+ "legendFormat": "received exemplars",
+ "legendLink": null
+ }
+ ],
+ "title": "Distributor exemplars received rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Ingester ingested exemplars rate\nThe rate of exemplars ingested in the ingesters.\nEvery exemplar is sent to the replication factor number of ingesters, so the sum of rates from all ingesters is divided by the replication factor.\nThis ingested exemplars rate should match the distributor's received exemplars rate.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ex/s"
+ },
+ "overrides": [ ]
+ },
+ "id": 33,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(\n cluster_namespace_job:cortex_ingester_ingested_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
+ "format": "time_series",
+ "legendFormat": "ingested exemplars",
+ "legendLink": null
+ }
+ ],
+ "title": "Ingester ingested exemplars rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "description": "### Ingester appended exemplars rate\nThe rate of exemplars appended in the ingesters.\nThis can be lower than ingested exemplars rate since TSDB does not append the same exemplar twice, and those can be frequent.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ex/s"
+ },
+ "overrides": [ ]
+ },
+ "id": 34,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 3,
+ "targets": [
+ {
+ "expr": "sum(\n cluster_namespace_job:cortex_ingester_tsdb_exemplar_exemplars_appended:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
+ "format": "time_series",
+ "legendFormat": "appended exemplars",
+ "legendLink": null
+ }
+ ],
+ "title": "Ingester appended exemplars rate",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Exemplars",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [ ]
+ },
+ "id": 35,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (reason) (rate(cortex_distributor_instance_rejected_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{reason}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Rejected distributor requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [ ]
+ },
+ "id": 36,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 6,
+ "targets": [
+ {
+ "expr": "sum by (reason) (rate(cortex_ingester_instance_rejected_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
+ "format": "time_series",
+ "legendFormat": "{{reason}}",
+ "legendLink": null
+ }
+ ],
+ "title": "Rejected ingester requests",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Instance Limits",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "mimir"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "label": "Data source",
+ "name": "datasource",
+ "options": [ ],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [ ],
+ "query": "label_values(cortex_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "$datasource",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [ ],
+ "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [ ],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Mimir / Writes",
+ "uid": "8280707b8f16e7b87b840fc1cc92d4c5",
+ "version": 0
+ }
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/mixins/rules.yaml b/observability/lgtm-distributed/charts/mimir-distributed/mixins/rules.yaml
new file mode 100644
index 0000000..bd28aa9
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/mixins/rules.yaml
@@ -0,0 +1,521 @@
+groups:
+- name: mimir_api_1
+ rules:
+ - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_request_duration_seconds:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_request_duration_seconds:50quantile
+ - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[1m]))
+ by (cluster, job)
+ record: cluster_job:cortex_request_duration_seconds:avg
+ - expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job)
+ record: cluster_job:cortex_request_duration_seconds_bucket:sum_rate
+ - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job)
+ record: cluster_job:cortex_request_duration_seconds_sum:sum_rate
+ - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)
+ record: cluster_job:cortex_request_duration_seconds_count:sum_rate
+ - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job)
+ record: cluster_job:cortex_request_duration_seconds:sum_rate
+- name: mimir_api_2
+ rules:
+ - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+ by (le, cluster, job, route))
+ record: cluster_job_route:cortex_request_duration_seconds:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+ by (le, cluster, job, route))
+ record: cluster_job_route:cortex_request_duration_seconds:50quantile
+ - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
+ / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
+ record: cluster_job_route:cortex_request_duration_seconds:avg
+ - expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job,
+ route)
+ record: cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate
+ - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
+ record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate
+ - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
+ record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate
+ - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job, route)
+ record: cluster_job_route:cortex_request_duration_seconds:sum_rate
+- name: mimir_api_3
+ rules:
+ - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+ by (le, cluster, namespace, job, route))
+ record: cluster_namespace_job_route:cortex_request_duration_seconds:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+ by (le, cluster, namespace, job, route))
+ record: cluster_namespace_job_route:cortex_request_duration_seconds:50quantile
+ - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
+ job, route) / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster,
+ namespace, job, route)
+ record: cluster_namespace_job_route:cortex_request_duration_seconds:avg
+ - expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
+ job, route)
+ record: cluster_namespace_job_route:cortex_request_duration_seconds_bucket:sum_rate
+ - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
+ job, route)
+ record: cluster_namespace_job_route:cortex_request_duration_seconds_sum:sum_rate
+ - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
+ job, route)
+ record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate
+ - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, namespace, job,
+ route)
+ record: cluster_namespace_job_route:cortex_request_duration_seconds:sum_rate
+- name: mimir_querier_api
+ rules:
+ - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_querier_request_duration_seconds:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_querier_request_duration_seconds:50quantile
+ - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+ job) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+ job)
+ record: cluster_job:cortex_querier_request_duration_seconds:avg
+ - expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
+ job)
+ record: cluster_job:cortex_querier_request_duration_seconds_bucket:sum_rate
+ - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+ job)
+ record: cluster_job:cortex_querier_request_duration_seconds_sum:sum_rate
+ - expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+ job)
+ record: cluster_job:cortex_querier_request_duration_seconds_count:sum_rate
+ - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+ by (le, cluster, job, route))
+ record: cluster_job_route:cortex_querier_request_duration_seconds:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+ by (le, cluster, job, route))
+ record: cluster_job_route:cortex_querier_request_duration_seconds:50quantile
+ - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+ job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by
+ (cluster, job, route)
+ record: cluster_job_route:cortex_querier_request_duration_seconds:avg
+ - expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
+ job, route)
+ record: cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate
+ - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+ job, route)
+ record: cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate
+ - expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+ job, route)
+ record: cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate
+ - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+ by (le, cluster, namespace, job, route))
+ record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
+ by (le, cluster, namespace, job, route))
+ record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:50quantile
+ - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+ namespace, job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m]))
+ by (cluster, namespace, job, route)
+ record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:avg
+ - expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
+ namespace, job, route)
+ record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate
+ - expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
+ namespace, job, route)
+ record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_sum:sum_rate
+ - expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
+ namespace, job, route)
+ record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_count:sum_rate
+- name: mimir_storage
+ rules:
+ - expr: histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_kv_request_duration_seconds:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_kv_request_duration_seconds:50quantile
+ - expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
+ / sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
+ record: cluster_job:cortex_kv_request_duration_seconds:avg
+ - expr: sum(rate(cortex_kv_request_duration_seconds_bucket[1m])) by (le, cluster,
+ job)
+ record: cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate
+ - expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
+ record: cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate
+ - expr: sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
+ record: cluster_job:cortex_kv_request_duration_seconds_count:sum_rate
+- name: mimir_queries
+ rules:
+ - expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_query_frontend_retries:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_query_frontend_retries:50quantile
+ - expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job) / sum(rate(cortex_query_frontend_retries_count[1m]))
+ by (cluster, job)
+ record: cluster_job:cortex_query_frontend_retries:avg
+ - expr: sum(rate(cortex_query_frontend_retries_bucket[1m])) by (le, cluster, job)
+ record: cluster_job:cortex_query_frontend_retries_bucket:sum_rate
+ - expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job)
+ record: cluster_job:cortex_query_frontend_retries_sum:sum_rate
+ - expr: sum(rate(cortex_query_frontend_retries_count[1m])) by (cluster, job)
+ record: cluster_job:cortex_query_frontend_retries_count:sum_rate
+ - expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_query_frontend_queue_duration_seconds:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_query_frontend_queue_duration_seconds:50quantile
+ - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
+ job) / sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by
+ (cluster, job)
+ record: cluster_job:cortex_query_frontend_queue_duration_seconds:avg
+ - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m])) by (le,
+ cluster, job)
+ record: cluster_job:cortex_query_frontend_queue_duration_seconds_bucket:sum_rate
+ - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
+ job)
+ record: cluster_job:cortex_query_frontend_queue_duration_seconds_sum:sum_rate
+ - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by (cluster,
+ job)
+ record: cluster_job:cortex_query_frontend_queue_duration_seconds_count:sum_rate
+- name: mimir_ingester_queries
+ rules:
+ - expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_ingester_queried_series:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_series_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_ingester_queried_series:50quantile
+ - expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_series_count[1m]))
+ by (cluster, job)
+ record: cluster_job:cortex_ingester_queried_series:avg
+ - expr: sum(rate(cortex_ingester_queried_series_bucket[1m])) by (le, cluster, job)
+ record: cluster_job:cortex_ingester_queried_series_bucket:sum_rate
+ - expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job)
+ record: cluster_job:cortex_ingester_queried_series_sum:sum_rate
+ - expr: sum(rate(cortex_ingester_queried_series_count[1m])) by (cluster, job)
+ record: cluster_job:cortex_ingester_queried_series_count:sum_rate
+ - expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_ingester_queried_samples:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_ingester_queried_samples:50quantile
+ - expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_samples_count[1m]))
+ by (cluster, job)
+ record: cluster_job:cortex_ingester_queried_samples:avg
+ - expr: sum(rate(cortex_ingester_queried_samples_bucket[1m])) by (le, cluster, job)
+ record: cluster_job:cortex_ingester_queried_samples_bucket:sum_rate
+ - expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job)
+ record: cluster_job:cortex_ingester_queried_samples_sum:sum_rate
+ - expr: sum(rate(cortex_ingester_queried_samples_count[1m])) by (cluster, job)
+ record: cluster_job:cortex_ingester_queried_samples_count:sum_rate
+ - expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_ingester_queried_exemplars:99quantile
+ - expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_exemplars_bucket[1m]))
+ by (le, cluster, job))
+ record: cluster_job:cortex_ingester_queried_exemplars:50quantile
+ - expr: sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job) /
+ sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)
+ record: cluster_job:cortex_ingester_queried_exemplars:avg
+ - expr: sum(rate(cortex_ingester_queried_exemplars_bucket[1m])) by (le, cluster,
+ job)
+ record: cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate
+ - expr: sum(rate(cortex_ingester_queried_exemplars_sum[1m])) by (cluster, job)
+ record: cluster_job:cortex_ingester_queried_exemplars_sum:sum_rate
+ - expr: sum(rate(cortex_ingester_queried_exemplars_count[1m])) by (cluster, job)
+ record: cluster_job:cortex_ingester_queried_exemplars_count:sum_rate
+- name: mimir_received_samples
+ rules:
+ - expr: |
+ sum by (cluster, namespace, job) (rate(cortex_distributor_received_samples_total[5m]))
+ record: cluster_namespace_job:cortex_distributor_received_samples:rate5m
+- name: mimir_exemplars_in
+ rules:
+ - expr: |
+ sum by (cluster, namespace, job) (rate(cortex_distributor_exemplars_in_total[5m]))
+ record: cluster_namespace_job:cortex_distributor_exemplars_in:rate5m
+- name: mimir_received_exemplars
+ rules:
+ - expr: |
+ sum by (cluster, namespace, job) (rate(cortex_distributor_received_exemplars_total[5m]))
+ record: cluster_namespace_job:cortex_distributor_received_exemplars:rate5m
+- name: mimir_exemplars_ingested
+ rules:
+ - expr: |
+ sum by (cluster, namespace, job) (rate(cortex_ingester_ingested_exemplars_total[5m]))
+ record: cluster_namespace_job:cortex_ingester_ingested_exemplars:rate5m
+- name: mimir_exemplars_appended
+ rules:
+ - expr: |
+ sum by (cluster, namespace, job) (rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total[5m]))
+ record: cluster_namespace_job:cortex_ingester_tsdb_exemplar_exemplars_appended:rate5m
+- name: mimir_scaling_rules
+ rules:
+ - expr: |
+ # Convenience rule to get the number of replicas for both a deployment and a statefulset.
+ # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+ sum by (cluster, namespace, deployment) (
+ label_replace(
+ kube_deployment_spec_replicas,
+ # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+ # always matches everything and the (optional) zone is not removed.
+ "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+ )
+ )
+ or
+ sum by (cluster, namespace, deployment) (
+ label_replace(kube_statefulset_replicas, "deployment", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?")
+ )
+ record: cluster_namespace_deployment:actual_replicas:count
+ - expr: |
+ ceil(
+ quantile_over_time(0.99,
+ sum by (cluster, namespace) (
+ cluster_namespace_job:cortex_distributor_received_samples:rate5m
+ )[24h:]
+ )
+ / 240000
+ )
+ labels:
+ deployment: distributor
+ reason: sample_rate
+ record: cluster_namespace_deployment_reason:required_replicas:count
+ - expr: |
+ ceil(
+ sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
+ * 0.59999999999999998 / 240000
+ )
+ labels:
+ deployment: distributor
+ reason: sample_rate_limits
+ record: cluster_namespace_deployment_reason:required_replicas:count
+ - expr: |
+ ceil(
+ quantile_over_time(0.99,
+ sum by (cluster, namespace) (
+ cluster_namespace_job:cortex_distributor_received_samples:rate5m
+ )[24h:]
+ )
+ * 3 / 80000
+ )
+ labels:
+ deployment: ingester
+ reason: sample_rate
+ record: cluster_namespace_deployment_reason:required_replicas:count
+ - expr: |
+ ceil(
+ quantile_over_time(0.99,
+ sum by(cluster, namespace) (
+ cortex_ingester_memory_series
+ )[24h:]
+ )
+ / 1500000
+ )
+ labels:
+ deployment: ingester
+ reason: active_series
+ record: cluster_namespace_deployment_reason:required_replicas:count
+ - expr: |
+ ceil(
+ sum by (cluster, namespace) (cortex_limits_overrides{limit_name="max_global_series_per_user"})
+ * 3 * 0.59999999999999998 / 1500000
+ )
+ labels:
+ deployment: ingester
+ reason: active_series_limits
+ record: cluster_namespace_deployment_reason:required_replicas:count
+ - expr: |
+ ceil(
+ sum by (cluster, namespace) (cortex_limits_overrides{limit_name="ingestion_rate"})
+ * 0.59999999999999998 / 80000
+ )
+ labels:
+ deployment: ingester
+ reason: sample_rate_limits
+ record: cluster_namespace_deployment_reason:required_replicas:count
+ - expr: |
+ ceil(
+ (sum by (cluster, namespace) (
+ cortex_ingester_tsdb_storage_blocks_bytes{job=~".+/ingester.*"}
+ ) / 4)
+ /
+ avg by (cluster, namespace) (
+ memcached_limit_bytes{job=~".+/memcached"}
+ )
+ )
+ labels:
+ deployment: memcached
+ reason: active_series
+ record: cluster_namespace_deployment_reason:required_replicas:count
+ - expr: |
+ sum by (cluster, namespace, deployment) (
+ label_replace(
+ label_replace(
+ sum by (cluster, namespace, pod)(rate(container_cpu_usage_seconds_total[1m])),
+ "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+ ),
+ # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+ # always matches everything and the (optional) zone is not removed.
+ "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+ )
+ )
+ record: cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate
+ - expr: |
+ # Convenience rule to get the CPU request for both a deployment and a statefulset.
+ # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+ # This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
+ # that remove resource metrics, ref:
+ # - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
+ # - https://github.com/kubernetes/kube-state-metrics/pull/1004
+ #
+ # This is the old expression, compatible with kube-state-metrics < v2.0.0,
+ # where kube_pod_container_resource_requests_cpu_cores was removed:
+ (
+ sum by (cluster, namespace, deployment) (
+ label_replace(
+ label_replace(
+ kube_pod_container_resource_requests_cpu_cores,
+ "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+ ),
+ # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+ # always matches everything and the (optional) zone is not removed.
+ "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+ )
+ )
+ )
+ or
+ # This expression is compatible with kube-state-metrics >= v1.4.0,
+ # where kube_pod_container_resource_requests was introduced.
+ (
+ sum by (cluster, namespace, deployment) (
+ label_replace(
+ label_replace(
+ kube_pod_container_resource_requests{resource="cpu"},
+ "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+ ),
+ # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+ # always matches everything and the (optional) zone is not removed.
+ "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+ )
+ )
+ )
+ record: cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
+ - expr: |
+ # Jobs should be sized to their CPU usage.
+ # We do this by comparing 99th percentile usage over the last 24hrs to
+ # their current provisioned #replicas and resource requests.
+ ceil(
+ cluster_namespace_deployment:actual_replicas:count
+ *
+ quantile_over_time(0.99, cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate[24h])
+ /
+ cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
+ )
+ labels:
+ reason: cpu_usage
+ record: cluster_namespace_deployment_reason:required_replicas:count
+ - expr: |
+ # Convenience rule to get the Memory utilization for both a deployment and a statefulset.
+ # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+ sum by (cluster, namespace, deployment) (
+ label_replace(
+ label_replace(
+ container_memory_usage_bytes{image!=""},
+ "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+ ),
+ # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+ # always matches everything and the (optional) zone is not removed.
+ "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+ )
+ )
+ record: cluster_namespace_deployment:container_memory_usage_bytes:sum
+ - expr: |
+ # Convenience rule to get the Memory request for both a deployment and a statefulset.
+ # Multi-zone deployments are grouped together removing the "zone-X" suffix.
+ # This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
+ # that remove resource metrics, ref:
+ # - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
+ # - https://github.com/kubernetes/kube-state-metrics/pull/1004
+ #
+ # This is the old expression, compatible with kube-state-metrics < v2.0.0,
+ # where kube_pod_container_resource_requests_memory_bytes was removed:
+ (
+ sum by (cluster, namespace, deployment) (
+ label_replace(
+ label_replace(
+ kube_pod_container_resource_requests_memory_bytes,
+ "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+ ),
+ # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+ # always matches everything and the (optional) zone is not removed.
+ "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+ )
+ )
+ )
+ or
+ # This expression is compatible with kube-state-metrics >= v1.4.0,
+ # where kube_pod_container_resource_requests was introduced.
+ (
+ sum by (cluster, namespace, deployment) (
+ label_replace(
+ label_replace(
+ kube_pod_container_resource_requests{resource="memory"},
+ "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
+ ),
+ # The question mark in "(.*?)" is used to make it non-greedy, otherwise it
+ # always matches everything and the (optional) zone is not removed.
+ "deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
+ )
+ )
+ )
+ record: cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
+ - expr: |
+ # Jobs should be sized to their Memory usage.
+ # We do this by comparing 99th percentile usage over the last 24hrs to
+ # their current provisioned #replicas and resource requests.
+ ceil(
+ cluster_namespace_deployment:actual_replicas:count
+ *
+ quantile_over_time(0.99, cluster_namespace_deployment:container_memory_usage_bytes:sum[24h])
+ /
+ cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
+ )
+ labels:
+ reason: memory_usage
+ record: cluster_namespace_deployment_reason:required_replicas:count
+- name: mimir_alertmanager_rules
+ rules:
+ - expr: |
+ sum by (cluster, job, pod) (cortex_alertmanager_alerts)
+ record: cluster_job_pod:cortex_alertmanager_alerts:sum
+ - expr: |
+ sum by (cluster, job, pod) (cortex_alertmanager_silences)
+ record: cluster_job_pod:cortex_alertmanager_silences:sum
+ - expr: |
+ sum by (cluster, job) (rate(cortex_alertmanager_alerts_received_total[5m]))
+ record: cluster_job:cortex_alertmanager_alerts_received_total:rate5m
+ - expr: |
+ sum by (cluster, job) (rate(cortex_alertmanager_alerts_invalid_total[5m]))
+ record: cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m
+ - expr: |
+ sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_total[5m]))
+ record: cluster_job_integration:cortex_alertmanager_notifications_total:rate5m
+ - expr: |
+ sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_failed_total[5m]))
+ record: cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m
+ - expr: |
+ sum by (cluster, job) (rate(cortex_alertmanager_state_replication_total[5m]))
+ record: cluster_job:cortex_alertmanager_state_replication_total:rate5m
+ - expr: |
+ sum by (cluster, job) (rate(cortex_alertmanager_state_replication_failed_total[5m]))
+ record: cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m
+ - expr: |
+ sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_total[5m]))
+ record: cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m
+ - expr: |
+ sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_failed_total[5m]))
+ record: cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m
+- name: mimir_ingester_rules
+ rules:
+ - expr: |
+ sum by(cluster, namespace, pod) (rate(cortex_ingester_ingested_samples_total[1m]))
+ record: cluster_namespace_pod:cortex_ingester_ingested_samples_total:rate1m
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/small.yaml b/observability/lgtm-distributed/charts/mimir-distributed/small.yaml
new file mode 100644
index 0000000..771b9db
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/small.yaml
@@ -0,0 +1,207 @@
+# These values configure the Grafana Mimir or Grafana Enterprise Metrics cluster
+# for a more production-ready setup. The setup targets 70% CPU and memory utilization
+# so that the cluster has room to grow. The resource requests reflect 70% utilization
+# and the limits reflect 100% utilization. The values do not set CPU limits,
+# because CPU limits have caused severe issues elsewhere, so we don't apply any in our helm chart:
+# see https://engineering.indeedblog.com/blog/2019/12/unthrottled-fixing-cpu-limits-in-the-cloud/
+# If you require CPU limits for billing purposes see capped-small.yaml
+#
+# These values are suitable for ingestion of ~1M series and scrape interval of 15s.
+# This implies ingestion rate of around 66000 samples per second.
+#
+# Query requirements can vary dramatically depending on query rate and query
+# ranges. The values here satisfy a "usual" query load of around 5 queries per second
+# as seen from our production clusters at this scale.
+#
+# The values in this file also add podAntiAffinity rules for ingesters and store-gateways.
+# The rules ensure that the replicas of the same component are not scheduled on the same
+# Kubernetes Node. Zone-aware replication is enabled by default on new installation.
+# Refer to [Migrate from single zone to zone-aware replication with Helm](https://grafana.com/docs/mimir/latest/migration-guide/migrating-from-single-zone-with-helm) and
+# [Zone-Aware Replication](https://grafana.com/docs/mimir/latest/configure/configure-zone-aware-replication/)
+# for more information.
+#
+# MinIO is no longer enabled, and you are encouraged to use your cloud providers
+# object storage service such as S3 or GCS.
+
+alertmanager:
+ persistentVolume:
+ enabled: true
+ replicas: 2
+ resources:
+ limits:
+ memory: 1.4Gi
+ requests:
+ cpu: 1
+ memory: 1Gi
+ statefulSet:
+ enabled: true
+
+compactor:
+ persistentVolume:
+ size: 20Gi
+ resources:
+ limits:
+ memory: 2.1Gi
+ requests:
+ cpu: 1
+ memory: 1.5Gi
+
+distributor:
+ replicas: 2
+ resources:
+ limits:
+ memory: 5.7Gi
+ requests:
+ cpu: 2
+ memory: 4Gi
+
+ingester:
+ persistentVolume:
+ size: 50Gi
+ replicas: 3
+ resources:
+ limits:
+ memory: 12Gi
+ requests:
+ cpu: 3.5
+ memory: 8Gi
+ topologySpreadConstraints: {}
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: target # support for enterprise.legacyLabels
+ operator: In
+ values:
+ - ingester
+ topologyKey: 'kubernetes.io/hostname'
+
+ - labelSelector:
+ matchExpressions:
+ - key: app.kubernetes.io/component
+ operator: In
+ values:
+ - ingester
+ topologyKey: 'kubernetes.io/hostname'
+
+ zoneAwareReplication:
+ topologyKey: 'kubernetes.io/hostname'
+
+admin-cache:
+ enabled: true
+ replicas: 2
+
+chunks-cache:
+ enabled: true
+ replicas: 2
+
+index-cache:
+ enabled: true
+ replicas: 3
+
+metadata-cache:
+ enabled: true
+
+results-cache:
+ enabled: true
+ replicas: 2
+
+minio:
+ enabled: false
+
+overrides_exporter:
+ replicas: 1
+ resources:
+ limits:
+ memory: 128Mi
+ requests:
+ cpu: 100m
+ memory: 128Mi
+
+querier:
+ replicas: 1
+ resources:
+ limits:
+ memory: 5.6Gi
+ requests:
+ cpu: 2
+ memory: 4Gi
+
+query_frontend:
+ replicas: 1
+ resources:
+ limits:
+ memory: 2.8Gi
+ requests:
+ cpu: 2
+ memory: 2Gi
+
+ruler:
+ replicas: 1
+ resources:
+ limits:
+ memory: 2.8Gi
+ requests:
+ cpu: 1
+ memory: 2Gi
+
+store_gateway:
+ persistentVolume:
+ size: 10Gi
+ replicas: 3
+ resources:
+ limits:
+ memory: 2.1Gi
+ requests:
+ cpu: 1
+ memory: 1.5Gi
+ topologySpreadConstraints: {}
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: target # support for enterprise.legacyLabels
+ operator: In
+ values:
+ - store-gateway
+ topologyKey: 'kubernetes.io/hostname'
+
+ - labelSelector:
+ matchExpressions:
+ - key: app.kubernetes.io/component
+ operator: In
+ values:
+ - store-gateway
+ topologyKey: 'kubernetes.io/hostname'
+ zoneAwareReplication:
+ topologyKey: 'kubernetes.io/hostname'
+
+nginx:
+ replicas: 1
+ resources:
+ limits:
+ memory: 731Mi
+ requests:
+ cpu: 1
+ memory: 512Mi
+
+# Grafana Enterprise Metrics feature related
+admin_api:
+ replicas: 1
+ resources:
+ limits:
+ memory: 128Mi
+ requests:
+ cpu: 100m
+ memory: 64Mi
+
+gateway:
+ replicas: 1
+ resources:
+ limits:
+ memory: 731Mi
+ requests:
+ cpu: 1
+ memory: 512Mi
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/NOTES.txt b/observability/lgtm-distributed/charts/mimir-distributed/templates/NOTES.txt
new file mode 100644
index 0000000..b5e585f
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/NOTES.txt
@@ -0,0 +1,36 @@
+{{ if .Values.enterprise.enabled }}
+Welcome to Grafana Enterprise Metrics!
+{{- else }}
+Welcome to Grafana Mimir!
+{{- end }}
+{{- $gateway := .Values.nginx }}
+{{- if eq (include "mimir.gateway.isEnabled" .) "true" }}
+{{- $gateway = .Values.gateway }}
+{{- end }}
+Remote write endpoints for Prometheus or Grafana Agent:
+{{ with $gateway.ingress -}}
+{{- if .enabled -}}
+From outside the cluster via ingress:
+{{ range .hosts }} http{{ if .tls }}s{{ end }}://{{ .host }}/api/v1/push
+{{ end }}
+{{- else -}}
+Ingress is not enabled, see the {{ if eq (include "mimir.gateway.isEnabled" $) "true" }}gateway{{ else }}nginx{{ end }}.ingress values.
+{{- end -}}
+{{- end }}
+From inside the cluster:
+ {{ include "mimir.remoteWriteUrl.inCluster" $ }}
+
+Read address, Grafana data source (Prometheus) URL:
+{{ with $gateway.ingress -}}
+{{- if .enabled -}}
+From outside the cluster via ingress:
+{{ range .hosts }} http{{ if .tls }}s{{ end }}://{{ .host }}{{ template "mimir.prometheusHttpPrefix" $ }}
+{{ end }}
+{{- else -}}
+Ingress is not enabled, see the {{ if eq (include "mimir.gateway.isEnabled" $) "true" }}gateway{{ else }}nginx{{ end }}.ingress values.
+{{- end -}}
+{{- end }}
+From inside the cluster:
+ {{ include "mimir.gatewayUrl" $ }}{{ template "mimir.prometheusHttpPrefix" $ }}
+
+**IMPORTANT**: Always consult CHANGELOG.md file at https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/CHANGELOG.md and the deprecation list there to learn about breaking changes that require action during upgrade.
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/_config-render.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/_config-render.tpl
new file mode 100644
index 0000000..447421b
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/_config-render.tpl
@@ -0,0 +1 @@
+{{- tpl .Values.mimir.config . -}}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/_helpers.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/_helpers.tpl
new file mode 100644
index 0000000..ba0000e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/_helpers.tpl
@@ -0,0 +1,699 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mimir.name" -}}
+{{- default ( include "mimir.infixName" . ) .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mimir.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default ( include "mimir.infixName" . ) .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Calculate the infix for naming
+*/}}
+{{- define "mimir.infixName" -}}
+{{- if and .Values.enterprise.enabled .Values.enterprise.legacyLabels -}}enterprise-metrics{{- else -}}mimir{{- end -}}
+{{- end -}}
+
+{{/*
+Calculate the gateway url
+*/}}
+{{- define "mimir.gatewayUrl" -}}
+{{- if eq (include "mimir.gateway.isEnabled" . ) "true" -}}
+http://{{ include "mimir.gateway.service.name" . }}.{{ .Release.Namespace }}.svc:{{ .Values.gateway.service.port | default (include "mimir.serverHttpListenPort" . ) }}
+{{- else -}}
+http://{{ template "mimir.fullname" . }}-nginx.{{ .Release.Namespace }}.svc:{{ .Values.nginx.service.port }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mimir.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Calculate image name based on whether enterprise features are requested
+*/}}
+{{- define "mimir.imageReference" -}}
+{{- if .Values.enterprise.enabled -}}{{ .Values.enterprise.image.repository }}:{{ .Values.enterprise.image.tag }}{{- else -}}{{ .Values.image.repository }}:{{ .Values.image.tag }}{{- end -}}
+{{- end -}}
+
+{{/*
+For compatibility and to support upgrade from enterprise-metrics chart calculate minio bucket name
+*/}}
+{{- define "mimir.minioBucketPrefix" -}}
+{{- if .Values.enterprise.legacyLabels -}}enterprise-metrics{{- else -}}mimir{{- end -}}
+{{- end -}}
+
+{{/*
+Create the name of the general service account
+*/}}
+{{- define "mimir.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+ {{ default (include "mimir.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+ {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create the name of the ruler service account
+*/}}
+{{- define "mimir.ruler.serviceAccountName" -}}
+{{- if and .Values.ruler.serviceAccount.create (eq .Values.ruler.serviceAccount.name "") -}}
+{{- $sa := default (include "mimir.fullname" .) .Values.serviceAccount.name }}
+{{- printf "%s-%s" $sa "ruler" }}
+{{- else if and .Values.ruler.serviceAccount.create (not (eq .Values.ruler.serviceAccount.name "")) -}}
+{{- .Values.ruler.serviceAccount.name -}}
+{{- else -}}
+{{- include "mimir.serviceAccountName" . -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create the app name for clients. Defaults to the same logic as "mimir.fullname", and default client expects "prometheus".
+*/}}
+{{- define "client.name" -}}
+{{- if .Values.client.name -}}
+{{- .Values.client.name -}}
+{{- else if .Values.client.fullnameOverride -}}
+{{- .Values.client.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default "prometheus" .Values.client.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Calculate the config from structured and unstructured text input
+*/}}
+{{- define "mimir.calculatedConfig" -}}
+{{ tpl (mergeOverwrite (include "mimir.unstructuredConfig" . | fromYaml) .Values.mimir.structuredConfig | toYaml) . }}
+{{- end -}}
+
+{{/*
+Calculate the config from the unstructured text input
+*/}}
+{{- define "mimir.unstructuredConfig" -}}
+{{ include (print $.Template.BasePath "/_config-render.tpl") . }}
+{{- end -}}
+
+{{/*
+The volume to mount for mimir configuration
+*/}}
+{{- define "mimir.configVolume" -}}
+{{- if eq .Values.configStorageType "Secret" -}}
+secret:
+ secretName: {{ tpl .Values.externalConfigSecretName . }}
+{{- else if eq .Values.configStorageType "ConfigMap" -}}
+configMap:
+ name: {{ tpl .Values.externalConfigSecretName . }}
+ items:
+ - key: "mimir.yaml"
+ path: "mimir.yaml"
+{{- end -}}
+{{- end -}}
+
+{{/*
+Internal servers http listen port - derived from Mimir default
+*/}}
+{{- define "mimir.serverHttpListenPort" -}}
+{{ (((.Values.mimir).structuredConfig).server).http_listen_port | default "8080" }}
+{{- end -}}
+
+{{/*
+Internal servers grpc listen port - derived from Mimir default
+*/}}
+{{- define "mimir.serverGrpcListenPort" -}}
+{{ (((.Values.mimir).structuredConfig).server).grpc_listen_port | default "9095" }}
+{{- end -}}
+
+{{/*
+Alertmanager cluster bind address
+*/}}
+{{- define "mimir.alertmanagerClusterBindAddress" -}}
+{{- if (include "mimir.calculatedConfig" . | fromYaml).alertmanager -}}
+{{ (include "mimir.calculatedConfig" . | fromYaml).alertmanager.cluster_bind_address | default "" }}
+{{- end -}}
+{{- end -}}
+
+{{- define "mimir.chunksCacheAddress" -}}
+dns+{{ template "mimir.fullname" . }}-chunks-cache.{{ .Release.Namespace }}.svc:{{ (index .Values "chunks-cache").port }}
+{{- end -}}
+
+{{- define "mimir.indexCacheAddress" -}}
+dns+{{ template "mimir.fullname" . }}-index-cache.{{ .Release.Namespace }}.svc:{{ (index .Values "index-cache").port }}
+{{- end -}}
+
+{{- define "mimir.metadataCacheAddress" -}}
+dns+{{ template "mimir.fullname" . }}-metadata-cache.{{ .Release.Namespace }}.svc:{{ (index .Values "metadata-cache").port }}
+{{- end -}}
+
+{{- define "mimir.resultsCacheAddress" -}}
+dns+{{ template "mimir.fullname" . }}-results-cache.{{ .Release.Namespace }}.svc:{{ (index .Values "results-cache").port }}
+{{- end -}}
+
+{{- define "mimir.adminCacheAddress" -}}
+dns+{{ template "mimir.fullname" . }}-admin-cache.{{ .Release.Namespace }}.svc:{{ (index .Values "admin-cache").port }}
+{{- end -}}
+
+{{/*
+Memberlist bind port
+*/}}
+{{- define "mimir.memberlistBindPort" -}}
+{{ (((.Values.mimir).structuredConfig).memberlist).bind_port | default "7946" }}
+{{- end -}}
+
+{{/*
+Resource name template
+Params:
+ ctx = . context
+ component = component name (optional)
+ rolloutZoneName = rollout zone name (optional)
+*/}}
+{{- define "mimir.resourceName" -}}
+{{- $resourceName := include "mimir.fullname" .ctx -}}
+{{- if .component -}}{{- $resourceName = printf "%s-%s" $resourceName .component -}}{{- end -}}
+{{- if and (not .component) .rolloutZoneName -}}{{- printf "Component name cannot be empty if rolloutZoneName (%s) is set" .rolloutZoneName | fail -}}{{- end -}}
+{{- if .rolloutZoneName -}}{{- $resourceName = printf "%s-%s" $resourceName .rolloutZoneName -}}{{- end -}}
+{{- if gt (len $resourceName) 253 -}}{{- printf "Resource name (%s) exceeds kubernetes limit of 253 character. To fix: shorten release name if this will be a fresh install or shorten zone names (e.g. \"a\" instead of \"zone-a\") if using zone-awareness." $resourceName | fail -}}{{- end -}}
+{{- $resourceName -}}
+{{- end -}}
+
+{{/*
+Resource labels
+Params:
+ ctx = . context
+ component = component name (optional)
+ rolloutZoneName = rollout zone name (optional)
+*/}}
+{{- define "mimir.labels" -}}
+{{- if .ctx.Values.enterprise.legacyLabels }}
+{{- if .component -}}
+app: {{ include "mimir.name" .ctx }}-{{ .component }}
+{{- else -}}
+app: {{ include "mimir.name" .ctx }}
+{{- end }}
+chart: {{ template "mimir.chart" .ctx }}
+heritage: {{ .ctx.Release.Service }}
+release: {{ .ctx.Release.Name }}
+
+{{- else -}}
+
+helm.sh/chart: {{ include "mimir.chart" .ctx }}
+app.kubernetes.io/name: {{ include "mimir.name" .ctx }}
+app.kubernetes.io/instance: {{ .ctx.Release.Name }}
+{{- if .component }}
+app.kubernetes.io/component: {{ .component }}
+{{- end }}
+{{- if .memberlist }}
+app.kubernetes.io/part-of: memberlist
+{{- end }}
+{{- if .ctx.Chart.AppVersion }}
+app.kubernetes.io/version: {{ .ctx.Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .ctx.Release.Service }}
+{{- end }}
+{{- if .rolloutZoneName }}
+{{- if not .component }}
+{{- printf "Component name cannot be empty if rolloutZoneName (%s) is set" .rolloutZoneName | fail }}
+{{- end }}
+name: "{{ .component }}-{{ .rolloutZoneName }}" {{- /* Currently required for rollout-operator. https://github.com/grafana/rollout-operator/issues/15 */}}
+rollout-group: {{ .component }}
+zone: {{ .rolloutZoneName }}
+{{- end }}
+{{- end -}}
+
+{{/*
+POD labels
+Params:
+ ctx = . context
+ component = name of the component
+ memberlist = true if part of memberlist gossip ring
+ rolloutZoneName = rollout zone name (optional)
+*/}}
+{{- define "mimir.podLabels" -}}
+{{ with .ctx.Values.global.podLabels -}}
+{{ toYaml . }}
+{{ end }}
+{{- if .ctx.Values.enterprise.legacyLabels }}
+{{- if .component -}}
+app: {{ include "mimir.name" .ctx }}-{{ .component }}
+{{- if not .rolloutZoneName }}
+name: {{ .component }}
+{{- end }}
+{{- end }}
+{{- if .memberlist }}
+gossip_ring_member: "true"
+{{- end -}}
+{{- if .component }}
+target: {{ .component }}
+release: {{ .ctx.Release.Name }}
+{{- end }}
+{{- else -}}
+helm.sh/chart: {{ include "mimir.chart" .ctx }}
+app.kubernetes.io/name: {{ include "mimir.name" .ctx }}
+app.kubernetes.io/instance: {{ .ctx.Release.Name }}
+app.kubernetes.io/version: {{ .ctx.Chart.AppVersion | quote }}
+app.kubernetes.io/managed-by: {{ .ctx.Release.Service }}
+{{- if .component }}
+app.kubernetes.io/component: {{ .component }}
+{{- end }}
+{{- if .memberlist }}
+app.kubernetes.io/part-of: memberlist
+{{- end }}
+{{- end }}
+{{- $componentSection := include "mimir.componentSectionFromName" . | fromYaml }}
+{{- with ($componentSection).podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- if .rolloutZoneName }}
+{{- if not .component }}
+{{- printf "Component name cannot be empty if rolloutZoneName (%s) is set" .rolloutZoneName | fail }}
+{{- end }}
+name: "{{ .component }}-{{ .rolloutZoneName }}" {{- /* Currently required for rollout-operator. https://github.com/grafana/rollout-operator/issues/15 */}}
+rollout-group: {{ .component }}
+zone: {{ .rolloutZoneName }}
+{{- end }}
+{{- end -}}
+
+{{/*
+POD annotations
+Params:
+ ctx = . context
+ component = name of the component
+*/}}
+{{- define "mimir.podAnnotations" -}}
+{{- if .ctx.Values.useExternalConfig }}
+checksum/config: {{ .ctx.Values.externalConfigVersion | quote }}
+{{- else -}}
+checksum/config: {{ include (print .ctx.Template.BasePath "/mimir-config.yaml") .ctx | sha256sum }}
+{{- end }}
+{{- with .ctx.Values.global.podAnnotations }}
+{{ toYaml . }}
+{{- end }}
+{{- if .component }}
+{{- if .ctx.Values.vaultAgent.enabled }}
+{{- include "mimir.vaultAgent.annotations" (dict "ctx" .ctx "component" .component) }}
+{{- end }}
+{{- $componentSection := include "mimir.componentSectionFromName" . | fromYaml }}
+{{- with ($componentSection).podAnnotations }}
+{{ toYaml . }}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Service selector labels
+Params:
+ ctx = . context
+ component = name of the component
+ rolloutZoneName = rollout zone name (optional)
+*/}}
+{{- define "mimir.selectorLabels" -}}
+{{- if .ctx.Values.enterprise.legacyLabels }}
+{{- if .component -}}
+app: {{ include "mimir.name" .ctx }}-{{ .component }}
+{{- end }}
+release: {{ .ctx.Release.Name }}
+{{- else -}}
+app.kubernetes.io/name: {{ include "mimir.name" .ctx }}
+app.kubernetes.io/instance: {{ .ctx.Release.Name }}
+{{- if .component }}
+app.kubernetes.io/component: {{ .component }}
+{{- end }}
+{{- end -}}
+{{- if .rolloutZoneName }}
+{{- if not .component }}
+{{- printf "Component name cannot be empty if rolloutZoneName (%s) is set" .rolloutZoneName | fail }}
+{{- end }}
+rollout-group: {{ .component }}
+zone: {{ .rolloutZoneName }}
+{{- end }}
+{{- end -}}
+
+
+{{/*
+Alertmanager http prefix
+*/}}
+{{- define "mimir.alertmanagerHttpPrefix" -}}
+{{- if (include "mimir.calculatedConfig" . | fromYaml).api }}
+{{- (include "mimir.calculatedConfig" . | fromYaml).api.alertmanager_http_prefix | default "/alertmanager" -}}
+{{- else -}}
+{{- print "/alertmanager" -}}
+{{- end -}}
+{{- end -}}
+
+
+{{/*
+Prometheus http prefix
+*/}}
+{{- define "mimir.prometheusHttpPrefix" -}}
+{{- if (include "mimir.calculatedConfig" . | fromYaml).api }}
+{{- (include "mimir.calculatedConfig" . | fromYaml).api.prometheus_http_prefix | default "/prometheus" -}}
+{{- else -}}
+{{- print "/prometheus" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+KEDA Autoscaling Prometheus address
+*/}}
+{{- define "mimir.kedaPrometheusAddress" -}}
+{{- if not .ctx.Values.kedaAutoscaling.prometheusAddress -}}
+{{ include "mimir.metaMonitoring.metrics.remoteReadUrl" . }}
+{{- else -}}
+{{ .ctx.Values.kedaAutoscaling.prometheusAddress }}
+{{- end -}}
+{{- end -}}
+
+
+{{/*
+Cluster name that shows up in dashboard metrics
+*/}}
+{{- define "mimir.clusterName" -}}
+{{ (include "mimir.calculatedConfig" . | fromYaml).cluster_name | default .Release.Name }}
+{{- end -}}
+
+{{/* Allow KubeVersion to be overridden. */}}
+{{- define "mimir.kubeVersion" -}}
+ {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}}
+{{- end -}}
+
+{{/* Get API Versions */}}
+{{- define "mimir.podDisruptionBudget.apiVersion" -}}
+ {{- if semverCompare ">= 1.21-0" (include "mimir.kubeVersion" .) -}}
+ {{- print "policy/v1" -}}
+ {{- else -}}
+ {{- print "policy/v1beta1" -}}
+ {{- end -}}
+{{- end -}}
+
+{{/*
+mimir.componentSectionFromName returns the sections from the user .Values in YAML
+that corresponds to the requested component. mimir.componentSectionFromName takes two arguments
+ .ctx = the root context of the chart
+ .component = the name of the component. mimir.componentSectionFromName uses an internal mapping to know
+ which component lives where in the values.yaml
+Examples:
+ $componentSection := include "mimir.componentSectionFromName" (dict "ctx" . "component" "store-gateway") | fromYaml
+ $componentSection.podLabels ...
+*/}}
+{{- define "mimir.componentSectionFromName" -}}
+{{- $componentsMap := dict
+ "admin-api" "admin_api"
+ "admin-cache" "admin-cache"
+ "alertmanager" "alertmanager"
+ "chunks-cache" "chunks-cache"
+ "compactor" "compactor"
+ "continuous-test" "continuous_test"
+ "distributor" "distributor"
+ "gateway" "gateway"
+ "gr-aggr-cache" "gr-aggr-cache"
+ "gr-metricname-cache" "gr-metricname-cache"
+ "graphite-querier" "graphite.querier"
+ "graphite-write-proxy" "graphite.write_proxy"
+ "index-cache" "index-cache"
+ "ingester" "ingester"
+ "memcached" "memcached"
+ "metadata-cache" "metadata-cache"
+ "nginx" "nginx"
+ "overrides-exporter" "overrides_exporter"
+ "querier" "querier"
+ "query-frontend" "query_frontend"
+ "query-scheduler" "query_scheduler"
+ "results-cache" "results-cache"
+ "ruler" "ruler"
+ "smoke-test" "smoke_test"
+ "store-gateway" "store_gateway"
+ "tokengen" "tokengenJob"
+-}}
+{{- $componentSection := index $componentsMap .component -}}
+{{- if not $componentSection -}}{{- printf "No component section mapping for %s not found in values; submit a bug report if you are a user, edit mimir.componentSectionFromName if you are a contributor" .component | fail -}}{{- end -}}
+{{- $section := .ctx.Values -}}
+{{- range regexSplit "\\." $componentSection -1 -}}
+ {{- $section = index $section . -}}
+ {{- if not $section -}}{{- printf "Component section %s not found in values; values: %s" . ($.ctx.Values | toJson | abbrev 100) | fail -}}{{- end -}}
+{{- end -}}
+{{- $section | toYaml -}}
+{{- end -}}
+
+{{/*
+Return the Vault Agent pod annotations if enabled and required by the component
+mimir.vaultAgent.annotations takes 2 arguments
+ .ctx = the root context of the chart
+ .component = the name of the component
+*/}}
+{{- define "mimir.vaultAgent.annotations" -}}
+{{- $vaultEnabledComponents := dict
+ "admin-api" true
+ "alertmanager" true
+ "compactor" true
+ "distributor" true
+ "gateway" true
+ "ingester" true
+ "overrides-exporter" true
+ "querier" true
+ "query-frontend" true
+ "query-scheduler" true
+ "ruler" true
+ "store-gateway" true
+-}}
+{{- if hasKey $vaultEnabledComponents .component }}
+vault.hashicorp.com/agent-inject: 'true'
+vault.hashicorp.com/role: '{{ .ctx.Values.vaultAgent.roleName }}'
+vault.hashicorp.com/agent-inject-secret-client.crt: '{{ .ctx.Values.vaultAgent.clientCertPath }}'
+vault.hashicorp.com/agent-inject-secret-client.key: '{{ .ctx.Values.vaultAgent.clientKeyPath }}'
+vault.hashicorp.com/agent-inject-secret-server.crt: '{{ .ctx.Values.vaultAgent.serverCertPath }}'
+vault.hashicorp.com/agent-inject-secret-server.key: '{{ .ctx.Values.vaultAgent.serverKeyPath }}'
+vault.hashicorp.com/agent-inject-secret-root.crt: '{{ .ctx.Values.vaultAgent.caCertPath }}'
+{{- end}}
+{{- end -}}
+
+{{/*
+Get the no_auth_tenant from the configuration
+*/}}
+{{- define "mimir.noAuthTenant" -}}
+{{- (include "mimir.calculatedConfig" . | fromYaml).no_auth_tenant | default "anonymous" -}}
+{{- end -}}
+
+{{/*
+Return if we should create a PodSecurityPolicy. Takes into account user values and supported kubernetes versions.
+*/}}
+{{- define "mimir.rbac.usePodSecurityPolicy" -}}
+{{- and
+ (
+ or (semverCompare "< 1.24-0" (include "mimir.kubeVersion" .))
+ (and (semverCompare "< 1.25-0" (include "mimir.kubeVersion" .)) .Values.rbac.forcePSPOnKubernetes124)
+ )
+ (and .Values.rbac.create (eq .Values.rbac.type "psp"))
+-}}
+{{- end -}}
+
+{{/*
+Return if we should create a SecurityContextConstraints. Takes into account user values and supported openshift versions.
+*/}}
+{{- define "mimir.rbac.useSecurityContextConstraints" -}}
+{{- and .Values.rbac.create (eq .Values.rbac.type "scc") -}}
+{{- end -}}
+
+{{- define "mimir.remoteWriteUrl.inCluster" -}}
+{{ include "mimir.gatewayUrl" . }}/api/v1/push
+{{- end -}}
+
+{{- define "mimir.remoteReadUrl.inCluster" -}}
+{{ include "mimir.gatewayUrl" . }}{{ include "mimir.prometheusHttpPrefix" . }}
+{{- end -}}
+
+{{/*
+Creates dict for zone-aware replication configuration
+Params:
+ ctx = . context
+ component = component name
+Return value:
+ {
+ zoneName: {
+ affinity: ,
+ nodeSelector: ,
+ replicas: ,
+ storageClass:
+ },
+ ...
+ }
+During migration there is a special case where an extra "zone" is generated with zonaName == "" empty string.
+The empty string evaluates to false in boolean expressions so it is treated as the default (non zone-aware) zone,
+which allows us to keep generating everything for the default zone.
+*/}}
+{{- define "mimir.zoneAwareReplicationMap" -}}
+{{- $zonesMap := (dict) -}}
+{{- $componentSection := include "mimir.componentSectionFromName" . | fromYaml -}}
+{{- $defaultZone := (dict "affinity" $componentSection.affinity "nodeSelector" $componentSection.nodeSelector "replicas" $componentSection.replicas "storageClass" $componentSection.storageClass) -}}
+
+{{- if $componentSection.zoneAwareReplication.enabled -}}
+{{- $numberOfZones := len $componentSection.zoneAwareReplication.zones -}}
+{{- if lt $numberOfZones 3 -}}
+{{- fail "When zone-awareness is enabled, you must have at least 3 zones defined." -}}
+{{- end -}}
+
+{{- $requestedReplicas := $componentSection.replicas -}}
+{{- if and (has .component (list "ingester" "alertmanager")) $componentSection.zoneAwareReplication.migration.enabled (not $componentSection.zoneAwareReplication.migration.writePath) -}}
+{{- $requestedReplicas = $componentSection.zoneAwareReplication.migration.replicas }}
+{{- end -}}
+{{- $replicaPerZone := div (add $requestedReplicas $numberOfZones -1) $numberOfZones -}}
+
+{{- range $idx, $rolloutZone := $componentSection.zoneAwareReplication.zones -}}
+{{- $_ := set $zonesMap $rolloutZone.name (dict
+ "affinity" (($rolloutZone.extraAffinity | default (dict)) | mergeOverwrite (include "mimir.zoneAntiAffinity" (dict "component" $.component "rolloutZoneName" $rolloutZone.name "topologyKey" $componentSection.zoneAwareReplication.topologyKey ) | fromYaml ) )
+ "nodeSelector" ($rolloutZone.nodeSelector | default (dict) )
+ "replicas" $replicaPerZone
+ "storageClass" $rolloutZone.storageClass
+ ) -}}
+{{- end -}}
+{{- if $componentSection.zoneAwareReplication.migration.enabled -}}
+{{- if $componentSection.zoneAwareReplication.migration.scaleDownDefaultZone -}}
+{{- $_ := set $defaultZone "replicas" 0 -}}
+{{- end -}}
+{{- $_ := set $zonesMap "" $defaultZone -}}
+{{- end -}}
+
+{{- else -}}
+{{- $_ := set $zonesMap "" $defaultZone -}}
+{{- end -}}
+{{- $zonesMap | toYaml }}
+
+{{- end -}}
+
+{{/*
+Calculate anti-affinity for a zone
+Params:
+ component = component name
+ rolloutZoneName = name of the rollout zone
+ topologyKey = topology key
+*/}}
+{{- define "mimir.zoneAntiAffinity" -}}
+{{- if .topologyKey -}}
+podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: rollout-group
+ operator: In
+ values:
+ - {{ .component }}
+ - key: zone
+ operator: NotIn
+ values:
+ - {{ .rolloutZoneName }}
+ topologyKey: {{ .topologyKey | quote }}
+{{- else -}}
+{}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Calculate annotations with zone-awareness
+Params:
+ ctx = . context
+ component = component name
+ rolloutZoneName = rollout zone name (optional)
+*/}}
+{{- define "mimir.componentAnnotations" -}}
+{{- $componentSection := include "mimir.componentSectionFromName" . | fromYaml -}}
+{{- if and (or $componentSection.zoneAwareReplication.enabled $componentSection.zoneAwareReplication.migration.enabled) .rolloutZoneName }}
+{{- $map := dict "rollout-max-unavailable" ($componentSection.zoneAwareReplication.maxUnavailable | toString) -}}
+{{- toYaml (deepCopy $map | mergeOverwrite $componentSection.annotations) }}
+{{- else -}}
+{{ toYaml $componentSection.annotations }}
+{{- end -}}
+{{- end -}}
+
+
+{{- define "mimir.var_dump" -}}
+{{- . | mustToPrettyJson | printf "\nThe JSON output of the dumped var is: \n%s" | fail }}
+{{- end -}}
+
+
+{{/*
+siToBytes is used to convert Kubernetes byte units to bytes.
+Works for a sub set of SI suffixes: m, k, M, G, T, and their power-of-two equivalents: Ki, Mi, Gi, Ti.
+
+mimir.siToBytes takes 1 argument
+ .value = the input value with SI unit
+*/}}
+{{- define "mimir.siToBytes" -}}
+ {{- if (hasSuffix "Ki" .value) -}}
+ {{- trimSuffix "Ki" .value | float64 | mul 1024 | ceil | int64 -}}
+ {{- else if (hasSuffix "Mi" .value) -}}
+ {{- trimSuffix "Mi" .value | float64 | mul 1048576 | ceil | int64 -}}
+ {{- else if (hasSuffix "Gi" .value) -}}
+ {{- trimSuffix "Gi" .value | float64 | mul 1073741824 | ceil | int64 -}}
+ {{- else if (hasSuffix "Ti" .value) -}}
+ {{- trimSuffix "Ti" .value | float64 | mul 1099511627776 | ceil | int64 -}}
+ {{- else if (hasSuffix "k" .value) -}}
+ {{- trimSuffix "k" .value | float64 | mul 1000 | ceil | int64 -}}
+ {{- else if (hasSuffix "M" .value) -}}
+ {{- trimSuffix "M" .value | float64 | mul 1000000 | ceil | int64 -}}
+ {{- else if (hasSuffix "G" .value) -}}
+ {{- trimSuffix "G" .value | float64 | mul 1000000000 | ceil | int64 -}}
+ {{- else if (hasSuffix "T" .value) -}}
+ {{- trimSuffix "T" .value | float64 | mul 1000000000000 | ceil | int64 -}}
+ {{- else if (hasSuffix "m" .value) -}}
+ {{- trimSuffix "m" .value | float64 | mulf 0.001 | ceil | int64 -}}
+ {{- else -}}
+ {{- .value }}
+ {{- end -}}
+{{- end -}}
+
+{{/*
+parseCPU is used to convert Kubernetes CPU units to the corresponding float value of CPU cores.
+The returned value is a string representation. If you need to do any math on it, please parse the string first.
+
+mimir.parseCPU takes 1 argument
+ .value = the Kubernetes CPU request value
+*/}}
+{{- define "mimir.parseCPU" -}}
+ {{- $value_string := .value | toString -}}
+ {{- if (hasSuffix "m" $value_string) -}}
+ {{ trimSuffix "m" $value_string | float64 | mulf 0.001 -}}
+ {{- else -}}
+ {{- $value_string }}
+ {{- end -}}
+{{- end -}}
+
+{{/*
+cpuToMilliCPU is used to convert Kubernetes CPU units to MilliCPU.
+The returned value is a string representation. If you need to do any math on it, please parse the string first.
+
+mimir.cpuToMilliCPU takes 1 argument
+ .value = the Kubernetes CPU request value
+*/}}
+{{- define "mimir.cpuToMilliCPU" -}}
+ {{- $value_string := .value | toString -}}
+ {{- if (hasSuffix "m" $value_string) -}}
+ {{ trimSuffix "m" $value_string -}}
+ {{- else -}}
+ {{- $value_string | float64 | mulf 1000 | toString }}
+ {{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-dep.yaml
new file mode 100644
index 0000000..3923ca1
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-dep.yaml
@@ -0,0 +1,138 @@
+{{- if .Values.enterprise.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ annotations:
+ {{- toYaml .Values.admin_api.annotations | nindent 4 }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 4 }}
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "admin-api") }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ replicas: {{ .Values.admin_api.replicas }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.admin_api.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "admin-api") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.admin_api.priorityClassName }}
+ priorityClassName: {{ .Values.admin_api.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "admin-api") | nindent 8 }}
+ {{- with .Values.admin_api.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ containers:
+ - name: admin-api
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=admin-api"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- range $key, $value := .Values.admin_api.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.admin_api.extraVolumeMounts }}
+ {{ toYaml .Values.admin_api.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: license
+ mountPath: /license
+ - name: storage
+ mountPath: "/data"
+ subPath: {{ .Values.admin_api.persistence.subPath }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ - name: memberlist
+ containerPort: {{ include "mimir.memberlistBindPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.admin_api.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.admin_api.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.admin_api.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.admin_api.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.admin_api.env }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.admin_api }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.admin_api.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.admin_api.extraContainers }}
+ {{ toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.admin_api.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.admin_api.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "admin-api") | nindent 6 }}
+ {{- with .Values.admin_api.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.admin_api.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+ {{- if .Values.admin_api.extraVolumes }}
+ {{ toYaml .Values.admin_api.extraVolumes | nindent 8}}
+ {{- end }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-pdb.yaml
new file mode 100644
index 0000000..2b88fff
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.enterprise.enabled -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "admin-api" "memberlist" true) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-servmon.yaml
new file mode 100644
index 0000000..3b068fe
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.enterprise.enabled -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" . "component" "admin-api" "memberlist" true) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-svc.yaml
new file mode 100644
index 0000000..64dfb60
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-api/admin-api-svc.yaml
@@ -0,0 +1,27 @@
+{{- if .Values.enterprise.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "admin-api") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 4 }}
+ {{- with .Values.admin_api.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.admin_api.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 4 }}
+{{- end -}}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-pdb.yaml
new file mode 100644
index 0000000..079ff0e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled (index .Values "admin-cache" "enabled") -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "admin-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-servmon.yaml
new file mode 100644
index 0000000..94a19bd
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled (index .Values "admin-cache" "enabled") -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "admin-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-statefulset.yaml
new file mode 100644
index 0000000..619aeef
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-statefulset.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled (index .Values "admin-cache" "enabled") -}}
+{{- include "mimir.memcached.statefulSet" (dict "ctx" $ "component" "admin-cache" ) }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-svc-headless.yaml
new file mode 100644
index 0000000..a4224d3
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/admin-cache/admin-cache-svc-headless.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled (index .Values "admin-cache" "enabled") -}}
+{{- include "mimir.memcached.service" (dict "ctx" $ "component" "admin-cache" ) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-config.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-config.yaml
new file mode 100644
index 0000000..bae1b7c
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-config.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.alertmanager.enabled -}}
+{{- if .Values.alertmanager.fallbackConfig -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }}-fallback-config
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.alertmanager.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+data:
+ alertmanager_fallback_config.yaml: |
+ {{- .Values.alertmanager.fallbackConfig | nindent 4 }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-dep.yaml
new file mode 100644
index 0000000..15f90a1
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-dep.yaml
@@ -0,0 +1,155 @@
+{{- if .Values.alertmanager.enabled -}}
+{{- if not .Values.alertmanager.statefulSet.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.alertmanager.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ replicas: {{ .Values.alertmanager.replicas }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.alertmanager.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "alertmanager") | nindent 8 }}
+ {{- if .Values.alertmanager.fallbackConfig }}
+ checksum/alertmanager-fallback-config: {{ include (print .Template.BasePath "/alertmanager/alertmanager-config.yaml") . | sha256sum }}
+ {{- end }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.alertmanager.priorityClassName }}
+ priorityClassName: {{ .Values.query_frontend.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "alertmanager") | nindent 8 }}
+ {{- with .Values.alertmanager.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end}}
+ {{- end }}
+ containers:
+ - name: alertmanager
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=alertmanager"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- range $key, $value := .Values.alertmanager.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.alertmanager.extraVolumeMounts }}
+ {{ toYaml .Values.alertmanager.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: storage
+ mountPath: "/data"
+ subPath: {{ .Values.alertmanager.persistence.subPath }}
+ {{- if .Values.alertmanager.fallbackConfig }}
+ - name: alertmanager-fallback-config
+ mountPath: /configs/
+ {{- end }}
+ - name: tmp
+ mountPath: /tmp
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ - name: memberlist
+ containerPort: {{ include "mimir.memberlistBindPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.alertmanager.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.alertmanager.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.alertmanager.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.alertmanager.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.alertmanager.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.alertmanager.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+{{- if .Values.alertmanager.extraContainers }}
+{{ toYaml .Values.alertmanager.extraContainers | indent 8}}
+{{- end }}
+ {{- with .Values.alertmanager.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.alertmanager.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "alertmanager") | nindent 6 }}
+ {{- with .Values.alertmanager.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.alertmanager.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+{{- if .Values.alertmanager.extraVolumes }}
+{{ toYaml .Values.alertmanager.extraVolumes | indent 8}}
+{{- end }}
+ - name: storage
+ emptyDir: {}
+ - name: tmp
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+ {{- if .Values.alertmanager.fallbackConfig }}
+ - name: alertmanager-fallback-config
+ configMap:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }}-fallback-config
+ {{- end }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-pdb.yaml
new file mode 100644
index 0000000..c0acf97
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.alertmanager.enabled -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "alertmanager" "memberlist" true) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-servmon.yaml
new file mode 100644
index 0000000..8e45219
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.alertmanager.enabled -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "alertmanager" "memberlist" true) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml
new file mode 100644
index 0000000..2daeab2
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml
@@ -0,0 +1,217 @@
+{{- if .Values.alertmanager.enabled -}}
+{{- if .Values.alertmanager.statefulSet.enabled -}}
+{{- $args := dict "ctx" . "component" "alertmanager" "memberlist" true -}}
+{{- $zonesMap := include "mimir.zoneAwareReplicationMap" $args | fromYaml -}}
+{{- range $zoneName, $rolloutZone := $zonesMap }}
+{{- with $ -}}
+{{- $_ := set $args "rolloutZoneName" $zoneName -}}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: {{ include "mimir.resourceName" $args }}
+ labels:
+ {{- include "mimir.labels" $args | nindent 4 }}
+ annotations:
+ {{- include "mimir.componentAnnotations" $args | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ replicas: {{ $rolloutZone.replicas }}
+ {{- if and (semverCompare ">= 1.23-0" (include "mimir.kubeVersion" .)) (.Values.alertmanager.persistentVolume.enableRetentionPolicy) }}
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: {{ .Values.alertmanager.persistentVolume.whenDeleted }}
+ whenScaled: {{ .Values.alertmanager.persistentVolume.whenScaled }}
+ {{- end }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" $args | nindent 6 }}
+ updateStrategy:
+ {{- if $zoneName }}
+ type: OnDelete
+ {{- else }}
+ {{- toYaml .Values.alertmanager.statefulStrategy | nindent 4 }}
+ {{- end }}
+ serviceName: {{ template "mimir.fullname" . }}-alertmanager
+ {{- if .Values.alertmanager.persistentVolume.enabled }}
+ volumeClaimTemplates:
+ - metadata:
+ name: storage
+ {{- if .Values.alertmanager.persistentVolume.annotations }}
+ annotations:
+ {{ toYaml .Values.alertmanager.persistentVolume.annotations | nindent 10 }}
+ {{- end }}
+ spec:
+ {{- $storageClass := default .Values.alertmanager.persistentVolume.storageClass $rolloutZone.storageClass }}
+ {{- if $storageClass }}
+ {{- if (eq "-" $storageClass) }}
+ storageClassName: ""
+ {{- else }}
+ storageClassName: {{ $storageClass }}
+ {{- end }}
+ {{- end }}
+ accessModes:
+ {{- toYaml .Values.alertmanager.persistentVolume.accessModes | nindent 10 }}
+ resources:
+ requests:
+ storage: "{{ .Values.alertmanager.persistentVolume.size }}"
+ {{- end }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" $args | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" $args | nindent 8 }}
+ {{- if .Values.alertmanager.fallbackConfig }}
+ checksum/alertmanager-fallback-config: {{ include (print .Template.BasePath "/alertmanager/alertmanager-config.yaml") . | sha256sum }}
+ {{- end }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ {{- with .Values.alertmanager.schedulerName }}
+ schedulerName: {{ . | quote }}
+ {{- end }}
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.alertmanager.priorityClassName }}
+ priorityClassName: {{ .Values.alertmanager.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "alertmanager") | nindent 8 }}
+ {{- with .Values.alertmanager.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ {{- with $rolloutZone.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with $rolloutZone.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "alertmanager") | nindent 6 }}
+ {{- with .Values.alertmanager.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.alertmanager.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+ {{- if not .Values.alertmanager.persistentVolume.enabled }}
+ - name: storage
+ emptyDir: {}
+ {{- end }}
+ - name: tmp
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+ {{- if .Values.alertmanager.fallbackConfig }}
+ - name: alertmanager-fallback-config
+ configMap:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }}-fallback-config
+ {{- end }}
+ {{- if .Values.alertmanager.extraVolumes }}
+ {{ toYaml .Values.alertmanager.extraVolumes | nindent 8 }}
+ {{- end }}
+ containers:
+ {{- if .Values.alertmanager.extraContainers }}
+ {{ toYaml .Values.alertmanager.extraContainers | nindent 8 }}
+ {{- end }}
+ - name: alertmanager
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=alertmanager"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- if $zoneName }}
+ - "-alertmanager.sharding-ring.instance-availability-zone={{ $zoneName }}"
+ {{- else }}
+ {{- if .Values.alertmanager.zoneAwareReplication.enabled }}
+ - "-alertmanager.sharding-ring.instance-availability-zone=zone-default"
+ {{- end }}
+ {{- end }}
+ {{- range $key, $value := .Values.alertmanager.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.alertmanager.extraVolumeMounts }}
+ {{ toYaml .Values.alertmanager.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: storage
+ mountPath: "/data"
+ {{- if .Values.alertmanager.persistentVolume.subPath }}
+ subPath: {{ .Values.alertmanager.persistentVolume.subPath }}
+ {{- else }}
+ {{- end }}
+ {{- if .Values.alertmanager.fallbackConfig }}
+ - name: alertmanager-fallback-config
+ mountPath: /configs/
+ {{- end }}
+ - name: tmp
+ mountPath: /tmp
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ - name: memberlist
+ containerPort: {{ include "mimir.memberlistBindPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.alertmanager.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.alertmanager.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.alertmanager.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.alertmanager.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.alertmanager.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.alertmanager }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.alertmanager.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+
+---
+{{ end }}
+{{ end }}
+{{ end }}
+{{ end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml
new file mode 100644
index 0000000..f3ced60
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml
@@ -0,0 +1,34 @@
+{{- if .Values.alertmanager.enabled -}}
+{{- $clusterPort := regexReplaceAll ".+[:]" (default "0.0.0.0:9094" (include "mimir.alertmanagerClusterBindAddress" .) ) "" -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }}-headless
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }}
+ prometheus.io/service-monitor: "false"
+ {{- with .Values.alertmanager.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.alertmanager.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ publishNotReadyAddresses: true
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ - port: {{ $clusterPort }}
+ protocol: TCP
+ name: cluster
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml
new file mode 100644
index 0000000..da6a718
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml
@@ -0,0 +1,40 @@
+{{- if .Values.alertmanager.enabled -}}
+{{- $args := dict "ctx" . "component" "alertmanager" "memberlist" true -}}
+{{- $zonesMap := include "mimir.zoneAwareReplicationMap" $args | fromYaml -}}
+{{- range $zoneName, $rolloutZone := $zonesMap }}
+{{- with $ -}}
+{{- $_ := set $args "rolloutZoneName" $zoneName -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" $args }}
+ labels:
+ {{- include "mimir.labels" $args | nindent 4 }}
+ {{- if and $zoneName .Values.alertmanager.zoneAwareReplication.migration.enabled }}
+ # Prevent scraping PODs via this service during migration as the original non zone-aware service already scrapes all PODs and you get duplicate metrics.
+ prometheus.io/service-monitor: "false"
+ {{- end }}
+ {{- with .Values.alertmanager.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.alertmanager.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" $args | nindent 4 }}
+
+---
+{{ end }}
+{{ end }}
+{{ end }}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-pdb.yaml
new file mode 100644
index 0000000..ede7b03
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if index .Values "chunks-cache" "enabled" }}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "chunks-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-servmon.yaml
new file mode 100644
index 0000000..1e28405
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if index .Values "chunks-cache" "enabled" }}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "chunks-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml
new file mode 100644
index 0000000..3a836f7
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml
@@ -0,0 +1 @@
+{{- include "mimir.memcached.statefulSet" (dict "ctx" $ "component" "chunks-cache" ) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-svc-headless.yaml
new file mode 100644
index 0000000..d2ece24
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/chunks-cache/chunks-cache-svc-headless.yaml
@@ -0,0 +1 @@
+{{- include "mimir.memcached.service" (dict "ctx" $ "component" "chunks-cache" ) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-pdb.yaml
new file mode 100644
index 0000000..095703b
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-pdb.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "compactor" "memberlist" true) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-servmon.yaml
new file mode 100644
index 0000000..e60bf6b
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-servmon.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "compactor" "memberlist" true) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-statefulset.yaml
new file mode 100644
index 0000000..d906867
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-statefulset.yaml
@@ -0,0 +1,176 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "compactor") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.compactor.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ podManagementPolicy: {{ .Values.compactor.podManagementPolicy }}
+ {{- if and (semverCompare ">= 1.23-0" (include "mimir.kubeVersion" .)) (.Values.compactor.persistentVolume.enableRetentionPolicy) }}
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: {{ .Values.compactor.persistentVolume.whenDeleted }}
+ whenScaled: {{ .Values.compactor.persistentVolume.whenScaled }}
+ {{- end }}
+ replicas: {{ .Values.compactor.replicas }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 6 }}
+ updateStrategy:
+ {{- toYaml .Values.compactor.strategy | nindent 4 }}
+ serviceName: {{ template "mimir.fullname" . }}-compactor
+ {{- if .Values.compactor.persistentVolume.enabled }}
+ volumeClaimTemplates:
+ - metadata:
+ name: storage
+ {{- if .Values.compactor.persistentVolume.annotations }}
+ annotations:
+ {{ toYaml .Values.compactor.persistentVolume.annotations | nindent 10 }}
+ {{- end }}
+ spec:
+ {{- if .Values.compactor.persistentVolume.storageClass }}
+ {{- if (eq "-" .Values.compactor.persistentVolume.storageClass) }}
+ storageClassName: ""
+ {{- else }}
+ storageClassName: "{{ .Values.compactor.persistentVolume.storageClass }}"
+ {{- end }}
+ {{- end }}
+ accessModes:
+ {{- toYaml .Values.compactor.persistentVolume.accessModes | nindent 10 }}
+ resources:
+ requests:
+ storage: "{{ .Values.compactor.persistentVolume.size }}"
+ {{- end }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "compactor") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ {{- with .Values.compactor.schedulerName }}
+ schedulerName: {{ . | quote }}
+ {{- end }}
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.compactor.priorityClassName }}
+ priorityClassName: {{ .Values.compactor.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "compactor") | nindent 8 }}
+ {{- with .Values.compactor.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ {{- with .Values.compactor.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.compactor.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "compactor") | nindent 6 }}
+ {{- with .Values.compactor.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.compactor.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+ {{- if not .Values.compactor.persistentVolume.enabled }}
+ - name: storage
+ emptyDir: {}
+ {{- end }}
+ {{- if .Values.compactor.extraVolumes }}
+ {{ toYaml .Values.compactor.extraVolumes | nindent 8 }}
+ {{- end }}
+ - name: active-queries
+ emptyDir: {}
+ containers:
+ {{- if .Values.compactor.extraContainers }}
+ {{ toYaml .Values.compactor.extraContainers | nindent 8 }}
+ {{- end }}
+ - name: compactor
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=compactor"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- range $key, $value := .Values.compactor.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.compactor.extraVolumeMounts }}
+ {{ toYaml .Values.compactor.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: storage
+ mountPath: "/data"
+ {{- if .Values.compactor.persistentVolume.subPath }}
+ subPath: {{ .Values.compactor.persistentVolume.subPath }}
+ {{- end }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ - name: memberlist
+ containerPort: {{ include "mimir.memberlistBindPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.compactor.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.compactor.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.compactor.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.compactor.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.compactor.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.compactor }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.compactor.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-svc.yaml
new file mode 100644
index 0000000..a761a54
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/compactor/compactor-svc.yaml
@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "compactor") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 4 }}
+ {{- with .Values.compactor.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.compactor.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 4 }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/continuous_test/continuous-test-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/continuous_test/continuous-test-dep.yaml
new file mode 100644
index 0000000..31c9050
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/continuous_test/continuous-test-dep.yaml
@@ -0,0 +1,118 @@
+{{- if .Values.continuous_test.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ annotations:
+ {{- toYaml .Values.continuous_test.annotations | nindent 4 }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "continuous-test") | nindent 4 }}
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "continuous-test") }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ replicas: {{ .Values.continuous_test.replicas }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "continuous-test") | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.continuous_test.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "continuous-test") | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "continuous-test") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.continuous_test.priorityClassName }}
+ priorityClassName: {{ .Values.continuous_test.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "continuous-test") | nindent 8 }}
+ {{- with .Values.continuous_test.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ containers:
+ - name: continuous-test
+ image: {{ .Values.continuous_test.image.repository }}:{{ .Values.continuous_test.image.tag }}
+ imagePullPolicy: {{ .Values.continuous_test.image.pullPolicy }}
+ args:
+ - "-server.metrics-port={{ include "mimir.serverHttpListenPort" . }}"
+ - "-tests.write-read-series-test.num-series={{ .Values.continuous_test.numSeries }}"
+ - "-tests.write-read-series-test.max-query-age={{ .Values.continuous_test.maxQueryAge }}"
+ - "-tests.write-endpoint={{ template "mimir.gatewayUrl" . }}"
+ - "-tests.read-endpoint={{ template "mimir.gatewayUrl" . }}/prometheus"
+ - "-tests.run-interval={{ .Values.continuous_test.runInterval }}"
+ {{- if eq .Values.continuous_test.auth.type "tenantId" }}
+ - "-tests.tenant-id={{ .Values.continuous_test.auth.tenant }}"
+ {{- end }}
+ {{- if eq .Values.continuous_test.auth.type "basicAuth" }}
+ - "-tests.basic-auth-password={{ .Values.continuous_test.auth.password }}"
+ - "-tests.basic-auth-user={{ .Values.continuous_test.auth.tenant }}"
+ {{- end }}
+ {{- if eq .Values.continuous_test.auth.type "bearerToken" }}
+ - "-tests.bearer-token={{ .Values.continuous_test.auth.bearerToken }}"
+ {{- end }}
+ {{- range $key, $value := .Values.continuous_test.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.continuous_test.extraVolumeMounts }}
+ {{ toYaml .Values.continuous_test.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ resources:
+ {{- toYaml .Values.continuous_test.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.continuous_test.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.continuous_test.env }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.continuous_test }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.continuous_test.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.continuous_test.extraContainers }}
+ {{ toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.continuous_test.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.continuous_test.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "continuous-test") | nindent 6 }}
+ {{- with .Values.continuous_test.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.continuous_test.terminationGracePeriodSeconds }}
+ volumes:
+ {{- if .Values.continuous_test.extraVolumes }}
+ {{ toYaml .Values.continuous_test.extraVolumes | nindent 8}}
+ {{- end }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/continuous_test/continuous-test-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/continuous_test/continuous-test-servmon.yaml
new file mode 100644
index 0000000..a025708
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/continuous_test/continuous-test-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.continuous_test.enabled -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "continuous-test") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/continuous_test/continuous-test-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/continuous_test/continuous-test-svc-headless.yaml
new file mode 100644
index 0000000..59aceb6
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/continuous_test/continuous-test-svc-headless.yaml
@@ -0,0 +1,25 @@
+{{- if .Values.continuous_test.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "continuous-test") }}-headless
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "continuous-test") | nindent 4 }}
+ {{- with .Values.continuous_test.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.continuous_test.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ publishNotReadyAddresses: true
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "continuous-test") | nindent 4 }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-dep.yaml
new file mode 100644
index 0000000..3fccf57
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-dep.yaml
@@ -0,0 +1,167 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "distributor") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.distributor.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ {{- if or (not .Values.distributor.kedaAutoscaling.enabled) (.Values.distributor.kedaAutoscaling.preserveReplicas) }}
+ # If replicas is not number (when using values file it's float64, when using --set arg it's int64) and is false (i.e. null) don't set it
+ {{- if or (or (kindIs "int64" .Values.distributor.replicas) (kindIs "float64" .Values.distributor.replicas)) (.Values.distributor.replicas) }}
+ replicas: {{ .Values.distributor.replicas }}
+ {{- end }}
+ {{- end }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.distributor.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "distributor") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.distributor.priorityClassName }}
+ priorityClassName: {{ .Values.distributor.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "distributor") | nindent 8 }}
+ {{- with .Values.distributor.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end}}
+ {{- end }}
+ containers:
+ - name: distributor
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=distributor"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ # When write requests go through distributors via gRPC, we want gRPC clients to re-resolve the distributors DNS
+ # endpoint before the distributor process is terminated, in order to avoid any failures during graceful shutdown.
+ # To achieve it, we set a shutdown delay greater than the gRPC max connection age.
+ - "-server.grpc.keepalive.max-connection-age=60s"
+ - "-server.grpc.keepalive.max-connection-age-grace=5m"
+ - "-server.grpc.keepalive.max-connection-idle=1m"
+ - "-shutdown-delay=90s"
+ {{- if .Values.ingester.zoneAwareReplication.migration.enabled }}
+ {{- if not .Values.ingester.zoneAwareReplication.migration.writePath }}
+ - "-ingester.ring.zone-awareness-enabled=false"
+ {{- end }}
+ {{- if .Values.ingester.zoneAwareReplication.migration.excludeDefaultZone }}
+ - "-ingester.ring.excluded-zones=zone-default"
+ {{- end }}
+ {{- end }}
+ {{- range $key, $value := .Values.distributor.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.distributor.extraVolumeMounts }}
+ {{ toYaml .Values.distributor.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: storage
+ mountPath: "/data"
+ subPath: {{ .Values.distributor.persistence.subPath }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ - name: memberlist
+ containerPort: {{ include "mimir.memberlistBindPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.distributor.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.distributor.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.distributor.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.distributor.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.distributor.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $cpu_request := dig "requests" "cpu" nil .Values.distributor.resources }}
+ {{- if $cpu_request }}
+ {{- $cpu_request_doubled := include "mimir.parseCPU" (dict "value" $cpu_request) | float64 | mulf 2 | ceil }}
+ - name: "GOMAXPROCS"
+ value: {{ max $cpu_request_doubled 8 | toString | toYaml }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.distributor }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.distributor.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+{{- if .Values.distributor.extraContainers }}
+{{ toYaml .Values.distributor.extraContainers | indent 8}}
+{{- end }}
+ {{- with .Values.distributor.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.distributor.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "distributor") | nindent 6 }}
+ {{- with .Values.distributor.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.distributor.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+{{- if .Values.distributor.extraVolumes }}
+{{ toYaml .Values.distributor.extraVolumes | indent 8}}
+{{- end }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-pdb.yaml
new file mode 100644
index 0000000..f99c602
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-pdb.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "distributor" "memberlist" true) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-servmon.yaml
new file mode 100644
index 0000000..d113fd9
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-servmon.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "distributor" "memberlist" true) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-so.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-so.yaml
new file mode 100644
index 0000000..a0680f8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-so.yaml
@@ -0,0 +1,44 @@
+{{- if .Values.distributor.kedaAutoscaling.enabled }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "distributor") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "distributor") | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.distributor.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ advanced:
+ horizontalPodAutoscalerConfig:
+ {{- with .Values.distributor.kedaAutoscaling.behavior }}
+ behavior:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ maxReplicaCount: {{ .Values.distributor.kedaAutoscaling.maxReplicaCount }}
+ minReplicaCount: {{ .Values.distributor.kedaAutoscaling.minReplicaCount }}
+ pollingInterval: {{ .Values.kedaAutoscaling.pollingInterval }}
+ scaleTargetRef:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "distributor") }}
+ apiVersion: apps/v1
+ kind: Deployment
+ triggers:
+ - metadata:
+ query: max_over_time(sum(sum by (pod) (rate(container_cpu_usage_seconds_total{container="distributor",namespace="{{ .Release.Namespace }}"}[5m])) and max by (pod) (up{container="distributor",namespace="{{ .Release.Namespace }}"}) > 0)[15m:]) * 1000
+ serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
+ {{- $cpu_request := dig "requests" "cpu" nil .Values.distributor.resources }}
+ threshold: {{ mulf (include "mimir.cpuToMilliCPU" (dict "value" $cpu_request)) (divf .Values.distributor.kedaAutoscaling.targetCPUUtilizationPercentage 100) | floor | int64 | quote }}
+ {{- if .Values.kedaAutoscaling.customHeaders }}
+ customHeaders: {{ (include "mimir.lib.mapToCSVString" (dict "map" .Values.kedaAutoscaling.customHeaders)) | quote }}
+ {{- end }}
+ type: prometheus
+ - metadata:
+ query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="distributor",namespace="{{ .Release.Namespace }}"}) and max by (pod) (up{container="distributor",namespace="{{ .Release.Namespace }}"}) > 0) or vector(0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="distributor",namespace="{{ .Release.Namespace }}", resource="memory"}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="distributor",namespace="{{ .Release.Namespace }}"}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor",namespace="{{ .Release.Namespace }}", reason="OOMKilled"}) or vector(0))
+ serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
+ {{- $mem_request := dig "requests" "memory" nil .Values.distributor.resources }}
+ threshold: {{ mulf (include "mimir.siToBytes" (dict "value" $mem_request)) (divf .Values.distributor.kedaAutoscaling.targetMemoryUtilizationPercentage 100) | floor | int64 | quote }}
+ {{- if .Values.kedaAutoscaling.customHeaders }}
+ customHeaders: {{ (include "mimir.lib.mapToCSVString" (dict "map" .Values.kedaAutoscaling.customHeaders)) | quote }}
+ {{- end }}
+ type: prometheus
+{{- end }}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-svc-headless.yaml
new file mode 100644
index 0000000..d59b78c
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-svc-headless.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "distributor") }}-headless
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }}
+ prometheus.io/service-monitor: "false"
+ {{- with .Values.distributor.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.distributor.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-svc.yaml
new file mode 100644
index 0000000..868949d
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/distributor/distributor-svc.yaml
@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "distributor") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }}
+ {{- with .Values.distributor.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.distributor.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/extra-manifests.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/extra-manifests.yaml
new file mode 100644
index 0000000..a9bb3b6
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/extra-manifests.yaml
@@ -0,0 +1,4 @@
+{{ range .Values.extraObjects }}
+---
+{{ tpl (toYaml .) $ }}
+{{ end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/_helpers.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/_helpers.tpl
new file mode 100644
index 0000000..7c25d79
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/_helpers.tpl
@@ -0,0 +1,21 @@
+{{/*
+nginx auth Secret name
+*/}}
+{{- define "mimir.gateway.nginx.authSecret" -}}
+{{ .Values.gateway.nginx.basicAuth.existingSecret | default (include "mimir.resourceName" (dict "ctx" . "component" "gateway-nginx") ) }}
+{{- end }}
+
+{{/*
+Name of the gateway Service resource
+*/}}
+{{- define "mimir.gateway.service.name" -}}
+{{ .Values.gateway.service.nameOverride | default (include "mimir.resourceName" (dict "ctx" . "component" "gateway") ) }}
+{{- end }}
+
+
+{{/*
+Returns "true" or "false" strings if the gateway component (nginx or GEM gateway) should be deployed
+*/}}
+{{- define "mimir.gateway.isEnabled" -}}
+{{- or .Values.gateway.enabledNonEnterprise .Values.enterprise.enabled -}}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-dep.yaml
new file mode 100644
index 0000000..0bcbf63
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-dep.yaml
@@ -0,0 +1,183 @@
+{{- if (eq (include "mimir.gateway.isEnabled" .) "true") }}
+{{- with .Values.gateway }}
+{{- $isGEMGateway := $.Values.enterprise.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ annotations:
+ {{- toYaml .annotations | nindent 4 }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "gateway") | nindent 4 }}
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "gateway") }}
+ namespace: {{ $.Release.Namespace | quote }}
+spec:
+{{- if not .autoscaling.enabled }}
+ replicas: {{ .replicas }}
+{{- end }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "gateway") | nindent 6 }}
+{{- with .strategy }}
+ strategy:
+{{ toYaml . | trim | indent 4 }}
+{{- end }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" $ "component" "gateway") | nindent 8 }}
+ annotations:
+ {{- $annotations := include "mimir.podAnnotations" (dict "ctx" $ "component" "gateway") | fromYaml -}}
+ {{- if not $isGEMGateway -}}
+ {{- $annotations = (dict "checksum/config" (include (print $.Template.BasePath "/gateway/nginx-configmap.yaml") $ | sha256sum )) | mergeOverwrite $annotations -}}
+ {{- end -}}
+ {{- $annotations | toYaml | nindent 8 }}
+ namespace: {{ $.Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ include "mimir.serviceAccountName" $ }}
+ {{- if .priorityClassName }}
+ priorityClassName: {{ .priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" $ "component" "gateway") | nindent 8 }}
+ {{- with .initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if $.Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range $.Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ containers:
+ {{- if $isGEMGateway }}
+ - name: gateway
+ image: {{ include "mimir.imageReference" $ | quote }}
+ imagePullPolicy: {{ $.Values.image.pullPolicy }}
+ args:
+ - "-target=gateway"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- range $key, $value := .extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- with .extraVolumeMounts }}
+ {{ toYaml . | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" $ }}
+ - name: license
+ mountPath: /license
+ - name: tmp
+ mountPath: /data
+ - name: active-queries
+ mountPath: /active-query-tracker
+ {{- else }}
+ - name: nginx
+ image: {{ .nginx.image.registry }}/{{ .nginx.image.repository }}:{{ .nginx.image.tag }}
+ imagePullPolicy: {{ $.Values.image.pullPolicy }}
+ args:
+ {{- with .extraArgs }}
+ {{- range $key, $value := . }}
+ - "-{{ $key }} {{ $value }}"
+ {{- end }}
+ {{- end }}
+ volumeMounts:
+ {{- if .extraVolumeMounts }}
+ {{ toYaml .extraVolumeMounts | nindent 12 }}
+ {{- end }}
+ - name: nginx-config
+ mountPath: /etc/nginx/nginx.conf
+ subPath: nginx.conf
+ {{- if .nginx.basicAuth.enabled }}
+ - name: auth
+ mountPath: /etc/nginx/secrets
+ {{- end }}
+ - name: tmp
+ mountPath: /tmp
+ - name: docker-entrypoint-d-override
+ mountPath: /docker-entrypoint.d
+ {{- end }}
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" $ }}
+ protocol: TCP
+ readinessProbe:
+ {{- toYaml .readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .containerSecurityContext | nindent 12 }}
+ env:
+ {{- with $.Values.global.extraEnv }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .env }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil . }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with $.Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .extraContainers }}
+ {{ toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" $ "component" "gateway") | nindent 6 }}
+ {{- with .tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" $ | nindent 10 }}
+ - name: runtime-config
+ configMap:
+ name: {{ include "mimir.fullname" $ }}-runtime
+ {{- if not $isGEMGateway }}
+ - name: nginx-config
+ configMap:
+ name: {{ include "mimir.fullname" $ }}-gateway-nginx
+ - name: docker-entrypoint-d-override
+ emptyDir: {}
+ {{- end }}
+ {{- if .nginx.basicAuth.enabled }}
+ - name: auth
+ secret:
+ secretName: {{ include "mimir.gateway.nginx.authSecret" $ }}
+ {{- end }}
+ - name: tmp
+ emptyDir: {}
+ {{- if $isGEMGateway }}
+ - name: license
+ secret:
+ secretName: {{ tpl $.Values.license.secretName $ }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+ {{- end -}}
+ {{- if .extraVolumes }}
+ {{- toYaml .extraVolumes | nindent 8 }}
+ {{- end }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-ingress.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-ingress.yaml
new file mode 100644
index 0000000..b6880de
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-ingress.yaml
@@ -0,0 +1,44 @@
+{{- if and (eq (include "mimir.gateway.isEnabled" .) "true") .Values.gateway.ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: {{ .Values.gateway.ingress.nameOverride | default (include "mimir.resourceName" (dict "ctx" . "component" "gateway") ) }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "gateway") | nindent 4 }}
+ {{- with .Values.gateway.ingress.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ {{- with .Values.gateway.ingress.ingressClassName }}
+ ingressClassName: {{ . }}
+ {{- end -}}
+ {{- with .Values.gateway.ingress.tls }}
+ tls:
+ {{- range . }}
+ - hosts:
+ {{- range .hosts }}
+ - {{ tpl . $ | quote }}
+ {{- end }}
+ {{- with .secretName }}
+ secretName: {{ . }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ rules:
+ {{- range .Values.gateway.ingress.hosts }}
+ - host: {{ tpl .host $ | quote }}
+ http:
+ paths:
+ {{- range .paths }}
+ - path: {{ .path }}
+ pathType: {{ .pathType }}
+ backend:
+ service:
+ name: {{ include "mimir.gateway.service.name" $ }}
+ port:
+ number: {{ $.Values.gateway.service.port }}
+ {{- end }}
+ {{- end }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-pdb.yaml
new file mode 100644
index 0000000..98ed873
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if eq (include "mimir.gateway.isEnabled" .) "true" -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "gateway") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-route.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-route.yaml
new file mode 100644
index 0000000..a8a0887
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-route.yaml
@@ -0,0 +1,25 @@
+{{- if .Values.gateway.route.enabled }}
+kind: Route
+apiVersion: route.openshift.io/v1
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "gateway") | nindent 4 }}
+ {{- with .Values.gateway.route.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ host: {{ tpl .Values.gateway.route.host $ | quote }}
+ to:
+ kind: Service
+ name: {{ include "mimir.gateway.service.name" . }}
+ weight: 100
+ port:
+ targetPort: http-metrics
+{{- with .Values.gateway.route.tls }}
+ tls:
+ {{- toYaml . | nindent 4 }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-servmon.yaml
new file mode 100644
index 0000000..1902f73
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if and (eq (include "mimir.gateway.isEnabled" .) "true") (not .Values.gateway.enabledNonEnterprise) -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "gateway") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-svc.yaml
new file mode 100644
index 0000000..67853fb
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-svc.yaml
@@ -0,0 +1,37 @@
+{{- if eq (include "mimir.gateway.isEnabled" .) "true" -}}
+{{- with .Values.gateway -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.gateway.service.name" $ }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "gateway") | nindent 4 }}
+ {{- with .service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .service.annotations | nindent 4 }}
+ namespace: {{ $.Release.Namespace | quote }}
+spec:
+ type: {{ .service.type }}
+ {{- with .service.clusterIP }}
+ clusterIP: {{ . }}
+ {{- end }}
+ {{- if and (eq "LoadBalancer" .service.type) .service.loadBalancerIP }}
+ loadBalancerIP: {{ .service.loadBalancerIP }}
+ {{- end }}
+ ports:
+ - port: {{ .service.port }}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ {{- with .service.legacyPort }}
+ - port: {{ . }}
+ protocol: TCP
+ name: legacy-http-metrics
+ targetPort: http-metrics
+ {{- end }}
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "gateway") | nindent 4 }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-v2-hpa.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-v2-hpa.yaml
new file mode 100644
index 0000000..b6ae526
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-v2-hpa.yaml
@@ -0,0 +1,35 @@
+{{- if .Values.gateway.autoscaling.enabled }}
+{{- if eq (include "mimir.hpa.version" .) "autoscaling/v2" }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "gateway") | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway") }}
+ minReplicas: {{ .Values.gateway.autoscaling.minReplicas }}
+ maxReplicas: {{ .Values.gateway.autoscaling.maxReplicas }}
+ metrics:
+ {{- with .Values.gateway.autoscaling.targetMemoryUtilizationPercentage }}
+ - type: Resource
+ resource:
+ name: memory
+ target:
+ type: Utilization
+ averageUtilization: {{ . }}
+ {{- end }}
+ {{- with .Values.gateway.autoscaling.targetCPUUtilizationPercentage }}
+ - type: Resource
+ resource:
+ name: cpu
+ target:
+ type: Utilization
+ averageUtilization: {{ . }}
+ {{- end }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-v2beta1-hpa.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-v2beta1-hpa.yaml
new file mode 100644
index 0000000..90e08cc
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/gateway-v2beta1-hpa.yaml
@@ -0,0 +1,31 @@
+{{- if .Values.gateway.autoscaling.enabled }}
+{{- if eq (include "mimir.hpa.version" .) "autoscaling/v2beta1" }}
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "gateway") | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway") }}
+ minReplicas: {{ .Values.gateway.autoscaling.minReplicas }}
+ maxReplicas: {{ .Values.gateway.autoscaling.maxReplicas }}
+ metrics:
+ {{- with .Values.gateway.autoscaling.targetMemoryUtilizationPercentage }}
+ - type: Resource
+ resource:
+ name: memory
+ targetAverageUtilization: {{ . }}
+ {{- end }}
+ {{- with .Values.gateway.autoscaling.targetCPUUtilizationPercentage }}
+ - type: Resource
+ resource:
+ name: cpu
+ targetAverageUtilization: {{ . }}
+ {{- end }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/nginx-configmap.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/nginx-configmap.yaml
new file mode 100644
index 0000000..8a24a4e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/nginx-configmap.yaml
@@ -0,0 +1,12 @@
+{{- if and (not .Values.enterprise.enabled) .Values.gateway.enabledNonEnterprise -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway-nginx") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "gateway-nginx") | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+data:
+ nginx.conf: |
+ {{- tpl .Values.gateway.nginx.config.file . | nindent 4 }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/nginx-secret.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/nginx-secret.yaml
new file mode 100644
index 0000000..1540273
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gateway/nginx-secret.yaml
@@ -0,0 +1,14 @@
+{{- if not .Values.enterprise.enabled -}}
+{{- if and .Values.gateway.enabledNonEnterprise .Values.gateway.nginx.basicAuth.enabled (not .Values.gateway.nginx.basicAuth.existingSecret) }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway-nginx") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "gateway-nginx") | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+stringData:
+ .htpasswd: |
+ {{- tpl .Values.gateway.nginx.basicAuth.htpasswd $ | nindent 4 }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gossip-ring/_helpers.gossip-ring.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/gossip-ring/_helpers.gossip-ring.tpl
new file mode 100644
index 0000000..b4a932e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gossip-ring/_helpers.gossip-ring.tpl
@@ -0,0 +1,11 @@
+{{/*
+gossip-ring selector labels
+*/}}
+{{- define "mimir.gossipRingSelectorLabels" -}}
+{{ include "mimir.selectorLabels" . }}
+{{- if .ctx.Values.enterprise.legacyLabels }}
+gossip_ring_member: "true"
+{{- else }}
+app.kubernetes.io/part-of: memberlist
+{{- end }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml
new file mode 100644
index 0000000..32da7fa
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gossip-ring") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "gossip-ring") | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ ports:
+ - name: gossip-ring
+ port: {{ include "mimir.memberlistBindPort" . }}
+ appProtocol: tcp
+ protocol: TCP
+ targetPort: {{ include "mimir.memberlistBindPort" . }}
+ publishNotReadyAddresses: true
+ selector:
+ {{- include "mimir.gossipRingSelectorLabels" (dict "ctx" .) | nindent 4 }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-pdb.yaml
new file mode 100644
index 0000000..b545f57
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "gr-aggr-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-servmon.yaml
new file mode 100644
index 0000000..37077a8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "gr-aggr-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-statefulset.yaml
new file mode 100644
index 0000000..1cba14a
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-statefulset.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.memcached.statefulSet" (dict "ctx" $ "component" "gr-aggr-cache" ) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-svc-headless.yaml
new file mode 100644
index 0000000..a9adf93
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-svc-headless.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.memcached.service" (dict "ctx" $ "component" "gr-aggr-cache" ) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-pdb.yaml
new file mode 100644
index 0000000..bd778c8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "gr-metricname-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-servmon.yaml
new file mode 100644
index 0000000..b4249b9
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "gr-metricname-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-statefulset.yaml
new file mode 100644
index 0000000..c9aecb2
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-statefulset.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.memcached.statefulSet" (dict "ctx" $ "component" "gr-metricname-cache" ) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-svc-headless.yaml
new file mode 100644
index 0000000..0844f2a
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-svc-headless.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.memcached.service" (dict "ctx" $ "component" "gr-metricname-cache" ) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-dep.yaml
new file mode 100644
index 0000000..c4b6ca1
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-dep.yaml
@@ -0,0 +1,138 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "graphite-querier") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "graphite-querier") | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.graphite.querier.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ replicas: {{ .Values.graphite.querier.replicas }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "graphite-querier") | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.graphite.querier.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "graphite-querier") | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "graphite-querier") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.graphite.querier.priorityClassName }}
+ priorityClassName: {{ .Values.graphite.querier.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "graphite-querier") | nindent 8 }}
+ {{- with .Values.graphite.querier.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ containers:
+ {{- if .Values.graphite.querier.extraContainers }}
+ {{ toYaml .Values.graphite.querier.extraContainers | nindent 8 }}
+ {{- end }}
+ - name: graphite-querier
+ image: {{ include "mimir.imageReference" . | quote }}
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - -target=graphite-querier
+ - -config.file=/etc/mimir/mimir.yaml
+ {{- range $key, $value := .Values.graphite.querier.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.graphite.querier.extraVolumeMounts }}
+ {{ toYaml .Values.graphite.querier.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: graphite-schemas
+ mountPath: /etc/graphite-proxy
+ - name: license
+ mountPath: /license
+ - name: storage
+ mountPath: "/data"
+ subPath: {{ .Values.graphite.querier.persistence.subPath }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.graphite.querier.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.graphite.querier.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.graphite.querier.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.graphite.querier.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.graphite.querier.env }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.graphite.querier }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.admin_api.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.graphite.querier.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.graphite.querier.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.graphite.querier.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.graphite.querier.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+ - name: graphite-schemas
+ configMap:
+ name: {{ template "mimir.fullname" . }}-graphite-schemas
+ {{- if .Values.graphite.querier.extraVolumes }}
+ {{ toYaml .Values.graphite.querier.extraVolumes | indent 8}}
+ {{- end }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-pdb.yaml
new file mode 100644
index 0000000..4a717a4
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "graphite-querier") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-schemas-configmap.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-schemas-configmap.yaml
new file mode 100644
index 0000000..1445a58
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-schemas-configmap.yaml
@@ -0,0 +1,14 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "graphite-schemas") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "graphite-schemas") | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+data:
+ storage-schemas.conf: |
+{{ .Values.graphite.querier.schemasConfiguration.storageSchemas | indent 4 }}
+ storage-aggregations.conf: |
+{{ .Values.graphite.querier.schemasConfiguration.storageAggregations | indent 4 }}
+{{- end -}}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-servmon.yaml
new file mode 100644
index 0000000..6092015
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "graphite-querier") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-svc.yaml
new file mode 100644
index 0000000..b954ebe
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-querier/graphite-querier-svc.yaml
@@ -0,0 +1,27 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "graphite-querier") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "graphite-querier") | nindent 4 }}
+ {{- with .Values.graphite.querier.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.graphite.querier.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "graphite-querier") | nindent 4 }}
+{{- end -}}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-dep.yaml
new file mode 100644
index 0000000..af33079
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-dep.yaml
@@ -0,0 +1,136 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "graphite-write-proxy") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "graphite-write-proxy") | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.graphite.write_proxy.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ replicas: {{ .Values.graphite.write_proxy.replicas }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "graphite-write-proxy") | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.graphite.write_proxy.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "graphite-write-proxy") | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "graphite-write-proxy") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.graphite.write_proxy.priorityClassName }}
+ priorityClassName: {{ .Values.graphite.write_proxy.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "graphite-write-proxy") | nindent 8 }}
+ {{- with .Values.graphite.write_proxy.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end}}
+ {{- end }}
+ containers:
+ {{- if .Values.graphite.write_proxy.extraContainers }}
+ {{ toYaml .Values.graphite.write_proxy.extraContainers | nindent 8 }}
+ {{- end }}
+ - name: graphite-write-proxy
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - -target=graphite-write-proxy
+ - -config.file=/etc/mimir/mimir.yaml
+ {{- range $key, $value := .Values.graphite.write_proxy.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.graphite.write_proxy.extraVolumeMounts }}
+ {{ toYaml .Values.graphite.write_proxy.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: license
+ mountPath: /license
+ - name: storage
+ mountPath: "/data"
+ subPath: {{ .Values.graphite.write_proxy.persistence.subPath }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.graphite.write_proxy.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.graphite.write_proxy.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.graphite.write_proxy.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.graphite.write_proxy.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.graphite.write_proxy.env }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.graphite.write_proxy }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.graphite.write_proxy.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.graphite.write_proxy.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.graphite.write_proxy.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.graphite.write_proxy.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.graphite.write_proxy.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+ - name: graphite-schemas
+ configMap:
+ name: {{ template "mimir.fullname" . }}-graphite-schemas
+ {{- if .Values.graphite.write_proxy.extraVolumes }}
+ {{ toYaml .Values.graphite.write_proxy.extraVolumes | indent 8}}
+ {{- end }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-pdb.yaml
new file mode 100644
index 0000000..163431e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "graphite-write-proxy") }}
+{{- end -}}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-servmon.yaml
new file mode 100644
index 0000000..8f7dbdb
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "graphite-write-proxy") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-svc.yaml
new file mode 100644
index 0000000..07c02f1
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/graphite-proxy/graphite-write-proxy/graphite-write-proxy-svc.yaml
@@ -0,0 +1,27 @@
+{{- if and .Values.enterprise.enabled .Values.graphite.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "graphite-write-proxy") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "graphite-write-proxy") | nindent 4 }}
+ {{- with .Values.graphite.write_proxy.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.graphite.write_proxy.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "graphite-write-proxy") | nindent 4 }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-pdb.yaml
new file mode 100644
index 0000000..0d0232d
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if index .Values "index-cache" "enabled" }}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "index-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-servmon.yaml
new file mode 100644
index 0000000..3fc39b0
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if index .Values "index-cache" "enabled" }}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "index-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml
new file mode 100644
index 0000000..98362ee
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml
@@ -0,0 +1 @@
+{{- include "mimir.memcached.statefulSet" (dict "ctx" $ "component" "index-cache" ) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-svc-headless.yaml
new file mode 100644
index 0000000..b86b205
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/index-cache/index-cache-svc-headless.yaml
@@ -0,0 +1 @@
+{{- include "mimir.memcached.service" (dict "ctx" $ "component" "index-cache" ) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-pdb.yaml
new file mode 100644
index 0000000..80b49b2
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-pdb.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "ingester" "memberlist" true) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-servmon.yaml
new file mode 100644
index 0000000..cb49e62
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-servmon.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "ingester" "memberlist" true) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-statefulset.yaml
new file mode 100644
index 0000000..2a8c4af
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -0,0 +1,200 @@
+{{- $args := dict "ctx" . "component" "ingester" "memberlist" true -}}
+{{- $zonesMap := include "mimir.zoneAwareReplicationMap" $args | fromYaml -}}
+{{- range $zoneName, $rolloutZone := $zonesMap }}
+{{- with $ -}}
+{{- $_ := set $args "rolloutZoneName" $zoneName -}}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: {{ include "mimir.resourceName" $args }}
+ labels:
+ {{- include "mimir.labels" $args | nindent 4 }}
+ annotations:
+ {{- include "mimir.componentAnnotations" $args | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ podManagementPolicy: {{ .Values.ingester.podManagementPolicy }}
+ replicas: {{ $rolloutZone.replicas }}
+ {{- if and (semverCompare ">= 1.23-0" (include "mimir.kubeVersion" .)) (.Values.ingester.persistentVolume.enableRetentionPolicy) }}
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: {{ .Values.ingester.persistentVolume.whenDeleted }}
+ whenScaled: {{ .Values.ingester.persistentVolume.whenScaled }}
+ {{- end }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" $args | nindent 6 }}
+ updateStrategy:
+ {{- if $zoneName }}
+ type: OnDelete
+ {{- else }}
+ {{- toYaml .Values.ingester.statefulStrategy | nindent 4 }}
+ {{- end }}
+ serviceName: {{ template "mimir.fullname" . }}-ingester{{- if not .Values.enterprise.legacyLabels -}}-headless{{- end -}}
+ {{- if .Values.ingester.persistentVolume.enabled }}
+ volumeClaimTemplates:
+ - metadata:
+ name: storage
+ {{- if .Values.ingester.persistentVolume.annotations }}
+ annotations:
+ {{- toYaml .Values.ingester.persistentVolume.annotations | nindent 10 }}
+ {{- end }}
+ spec:
+ {{- $storageClass := default .Values.ingester.persistentVolume.storageClass $rolloutZone.storageClass }}
+ {{- if $storageClass }}
+ {{- if (eq "-" $storageClass) }}
+ storageClassName: ""
+ {{- else }}
+ storageClassName: {{ $storageClass }}
+ {{- end }}
+ {{- end }}
+ accessModes:
+ {{- toYaml .Values.ingester.persistentVolume.accessModes | nindent 10 }}
+ resources:
+ requests:
+ storage: "{{ .Values.ingester.persistentVolume.size }}"
+ {{- end }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" $args | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" $args | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ {{- with .Values.ingester.schedulerName }}
+ schedulerName: {{ . | quote }}
+ {{- end }}
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.ingester.priorityClassName }}
+ priorityClassName: {{ .Values.ingester.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" $args | nindent 8 }}
+ {{- with .Values.ingester.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ {{- with $rolloutZone.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with $rolloutZone.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "ingester") | nindent 6 }}
+ {{- with .Values.ingester.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.ingester.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+ {{- if not .Values.ingester.persistentVolume.enabled }}
+ - name: storage
+ emptyDir: {}
+ {{- end }}
+ {{- if .Values.ingester.extraVolumes }}
+ {{ toYaml .Values.ingester.extraVolumes | nindent 8 }}
+ {{- end }}
+ - name: active-queries
+ emptyDir: {}
+ containers:
+ {{- if .Values.ingester.extraContainers }}
+ {{ toYaml .Values.ingester.extraContainers | nindent 8 }}
+ {{- end }}
+ - name: ingester
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=ingester"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- if $zoneName }}
+ - "-ingester.ring.instance-availability-zone={{ $zoneName }}"
+ {{- else }}
+ - "-ingester.ring.instance-availability-zone=zone-default"
+ {{- if .Values.ingester.zoneAwareReplication.migration.enabled }}
+ - "-blocks-storage.tsdb.flush-blocks-on-shutdown=true"
+ - "-ingester.ring.unregister-on-shutdown=true"
+ {{- end }}
+ - "-server.grpc-max-concurrent-streams=500"
+ {{- end }}
+ {{- range $key, $value := .Values.ingester.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.ingester.extraVolumeMounts }}
+ {{ toYaml .Values.ingester.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: storage
+ mountPath: "/data"
+ {{- if .Values.ingester.persistentVolume.subPath }}
+ subPath: {{ .Values.ingester.persistentVolume.subPath }}
+ {{- else }}
+ {{- end }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ - name: memberlist
+ containerPort: {{ include "mimir.memberlistBindPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.ingester.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.ingester.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.ingester.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.ingester.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.ingester.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.ingester }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.ingester.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+---
+{{ end }}
+{{ end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-svc-headless.yaml
new file mode 100644
index 0000000..0c5c430
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-svc-headless.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ingester") }}-headless
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 4 }}
+ prometheus.io/service-monitor: "false"
+ {{- with .Values.ingester.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.ingester.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 4 }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-svc.yaml
new file mode 100644
index 0000000..21afa4d
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingester/ingester-svc.yaml
@@ -0,0 +1,38 @@
+{{- $args := dict "ctx" . "component" "ingester" "memberlist" true -}}
+{{- $zonesMap := include "mimir.zoneAwareReplicationMap" $args | fromYaml -}}
+{{- range $zoneName, $rolloutZone := $zonesMap }}
+{{- with $ -}}
+{{- $_ := set $args "rolloutZoneName" $zoneName -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" $args }}
+ labels:
+ {{- include "mimir.labels" $args | nindent 4 }}
+ {{- if and $zoneName .Values.ingester.zoneAwareReplication.migration.enabled }}
+ # Prevent scraping PODs via this service during migration as the original non zone-aware service already scrapes all PODs and you get duplicate metrics.
+ prometheus.io/service-monitor: "false"
+ {{- end }}
+ {{- with .Values.ingester.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.ingester.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" $args | nindent 4 }}
+
+---
+{{ end }}
+{{ end }}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ingress.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingress.yaml
new file mode 100644
index 0000000..97eb406
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ingress.yaml
@@ -0,0 +1,46 @@
+{{- if .Values.ingress.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: {{ include "mimir.fullname" . }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ {{- with .Values.ingress.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ {{- with .Values.ingress.ingressClassName }}
+ ingressClassName: {{ . }}
+ {{- end -}}
+ {{- with .Values.ingress.tls }}
+ tls:
+ {{- range . }}
+ - hosts:
+ {{- range .hosts }}
+ - {{ tpl . $ | quote }}
+ {{- end }}
+ {{- with .secretName }}
+ secretName: {{ . }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ rules:
+ {{- range $.Values.ingress.hosts }}
+ - host: {{ tpl . $ | quote }}
+ http:
+ paths:
+ {{- range $svcName, $paths := $.Values.ingress.paths }}
+ {{- range $paths }}
+ - path: {{ .path }}
+ pathType: {{ .pathType | default "Prefix" }}
+ backend:
+ service:
+ name: {{ include "mimir.fullname" $ }}-{{ $svcName }}
+ port:
+ number: {{ include "mimir.serverHttpListenPort" $ }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/map-to-csv.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/map-to-csv.tpl
new file mode 100644
index 0000000..e31974d
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/map-to-csv.tpl
@@ -0,0 +1,17 @@
+{{/*
+Convert labels to string like: key1=value1, key2=value2, ...
+Example:
+ customHeaders:
+ X-Scope-OrgID: tenant-1
+becomes:
+ customHeaders: "X-Scope-OrgID=tenant-1"
+Params:
+ map = map to convert to csv string
+*/}}
+{{- define "mimir.lib.mapToCSVString" -}}
+{{- $list := list -}}
+{{- range $k, $v := $.map -}}
+{{- $list = append $list (printf "%s=%s" $k $v) -}}
+{{- end -}}
+{{ join "," $list }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/pdb.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/pdb.tpl
new file mode 100644
index 0000000..81239bb
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/pdb.tpl
@@ -0,0 +1,23 @@
+{{/*
+Mimir common PodDisruptionBudget definition
+Params:
+ ctx = . context
+ component = name of the component
+*/}}
+{{- define "mimir.lib.podDisruptionBudget" -}}
+{{- $componentSection := include "mimir.componentSectionFromName" . | fromYaml }}
+{{ with ($componentSection).podDisruptionBudget }}
+apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" $.ctx }}
+kind: PodDisruptionBudget
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $.ctx "component" $.component) }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $.ctx "component" $.component) | nindent 4 }}
+ namespace: {{ $.ctx.Release.Namespace | quote }}
+spec:
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" $.ctx "component" $.component) | nindent 6 }}
+{{ toYaml . | indent 2 }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/podSecurityContext.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/podSecurityContext.tpl
new file mode 100644
index 0000000..334c9db
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/podSecurityContext.tpl
@@ -0,0 +1,13 @@
+{{/*
+Mimir common podSecurityContext
+Params:
+ ctx = . context
+ component = name of the component
+*/}}
+{{- define "mimir.lib.podSecurityContext" -}}
+{{- $componentSection := include "mimir.componentSectionFromName" . | fromYaml }}
+{{- /* Some components have security context in the values.yalm as podSecurityContext and others as securityContext */}}
+{{- $podSecurityContextSection := $componentSection.securityContext | default $componentSection.podSecurityContext -}}
+{{- $defaultContext := deepCopy .ctx.Values.rbac.podSecurityContext }}
+{{- $podSecurityContextSection | mustMergeOverwrite $defaultContext | toYaml -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/service-monitor.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/service-monitor.tpl
new file mode 100644
index 0000000..ab0dbb2
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/service-monitor.tpl
@@ -0,0 +1,73 @@
+{{/*
+Mimir common ServiceMonitor definition
+Params:
+ ctx = . context
+ component = name of the component
+ memberlist = true/false, whether component is part of memberlist
+*/}}
+{{- define "mimir.lib.serviceMonitor" -}}
+{{- with .ctx.Values.metaMonitoring.serviceMonitor }}
+{{- if .enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ name: {{ include "mimir.resourceName" $ }}
+ namespace: {{ .namespace | default $.ctx.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" $ | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ namespaceSelector:
+ {{- if .namespaceSelector }}
+ {{- toYaml .namespaceSelector | nindent 4 }}
+ {{- else }}
+ matchNames:
+ - {{ $.ctx.Release.Namespace }}
+ {{- end }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" $ | nindent 6 }}
+ matchExpressions:
+ - key: prometheus.io/service-monitor
+ operator: NotIn
+ values:
+ - "false"
+ endpoints:
+ - port: http-metrics
+ {{- with .interval }}
+ interval: {{ . }}
+ {{- end }}
+ {{- with .scrapeTimeout }}
+ scrapeTimeout: {{ . }}
+ {{- end }}
+ relabelings:
+ - sourceLabels: [job]
+ replacement: "{{ $.ctx.Release.Namespace }}/{{ $.component }}"
+ targetLabel: job
+ {{- if kindIs "string" .clusterLabel }}
+ - replacement: "{{ .clusterLabel | default (include "mimir.clusterName" $.ctx) }}"
+ targetLabel: cluster
+ {{- end }}
+ {{- with .relabelings }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .metricRelabelings }}
+ metricRelabelings:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .scheme }}
+ scheme: {{ . }}
+ {{- end }}
+ {{- with .tlsConfig }}
+ tlsConfig:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/topologySpreadConstraints.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/topologySpreadConstraints.tpl
new file mode 100644
index 0000000..06cea50
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/lib/topologySpreadConstraints.tpl
@@ -0,0 +1,21 @@
+{{- define "mimir.lib.topologySpreadConstraints" -}}
+{{- $componentSection := include "mimir.componentSectionFromName" . | fromYaml -}}
+{{- $topologySpreadConstraintsSection := $componentSection.topologySpreadConstraints -}}
+{{- $selectorLabels := include "mimir.selectorLabels" . -}}
+{{- if $topologySpreadConstraintsSection -}}
+{{- $constraints := kindIs "slice" $topologySpreadConstraintsSection | ternary $topologySpreadConstraintsSection (list $topologySpreadConstraintsSection) -}}
+topologySpreadConstraints:
+{{- range $constraint := $constraints }}
+- maxSkew: {{ $constraint.maxSkew }}
+ topologyKey: {{ $constraint.topologyKey }}
+ whenUnsatisfiable: {{ $constraint.whenUnsatisfiable }}
+ labelSelector:
+ {{- if $constraint.labelSelector -}}
+ {{- $constraint.labelSelector | toYaml | nindent 4 -}}
+ {{- else }}
+ matchLabels:
+ {{- $selectorLabels | nindent 6 }}
+ {{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/license-secret.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/license-secret.yaml
new file mode 100644
index 0000000..c992e88
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/license-secret.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.enterprise.enabled -}}
+{{- if not .Values.license.external }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ tpl .Values.license.secretName . }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+data:
+ license.jwt: {{ .Values.license.contents | b64enc }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/memcached/_memcached-statefulset.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/memcached/_memcached-statefulset.tpl
new file mode 100644
index 0000000..0c4bd00
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/memcached/_memcached-statefulset.tpl
@@ -0,0 +1,149 @@
+{{/*
+memcached StatefulSet
+*/}}
+{{- define "mimir.memcached.statefulSet" -}}
+{{ with (index $.ctx.Values $.component) }}
+{{- if .enabled -}}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $.ctx "component" $.component) }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $.ctx "component" "memcached") | nindent 4 }}
+ annotations:
+ {{- toYaml .annotations | nindent 4 }}
+ namespace: {{ $.ctx.Release.Namespace | quote }}
+spec:
+ podManagementPolicy: {{ .podManagementPolicy }}
+ replicas: {{ .replicas }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" $.ctx "component" $.component) | nindent 6 }}
+ updateStrategy:
+ {{- toYaml .statefulStrategy | nindent 4 }}
+ serviceName: {{ template "mimir.fullname" $.ctx }}-{{ $.component }}
+
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" $ | nindent 8 }}
+ annotations:
+ {{- with $.ctx.Values.global.podAnnotations }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .podAnnotations }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" $.ctx }}
+ {{- if .priorityClassName }}
+ priorityClassName: {{ .priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" $.ctx "component" "memcached") | nindent 8 }}
+ {{- with .initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" $ | nindent 6 }}
+ {{- with .tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .terminationGracePeriodSeconds }}
+ {{- if $.ctx.Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range $.ctx.Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ {{- if .extraVolumes }}
+ volumes:
+ {{- toYaml .extraVolumes | nindent 8 }}
+ {{- end }}
+ containers:
+ {{- if .extraContainers }}
+ {{ toYaml .extraContainers | nindent 8 }}
+ {{- end }}
+ - name: memcached
+ {{- with $.ctx.Values.memcached.image }}
+ image: {{ .repository }}:{{ .tag }}
+ imagePullPolicy: {{ .pullPolicy }}
+ {{- end }}
+ resources:
+ {{- if .resources }}
+ {{- toYaml .resources | nindent 12 }}
+ {{- else }}
+ {{- /* Calculate requested memory as round(allocatedMemory * 1.2). But with integer built-in operators. */}}
+ {{- $requestMemory := div (add (mul .allocatedMemory 12) 5) 10 }}
+ limits:
+ memory: {{ $requestMemory }}Mi
+ requests:
+ cpu: 500m
+ memory: {{ $requestMemory }}Mi
+ {{- end }}
+ ports:
+ - containerPort: {{ .port }}
+ name: client
+ args:
+ - -m {{ .allocatedMemory }}
+ - --extended=modern,track_sizes{{ with .extraExtendedOptions }},{{ . }}{{ end }}
+ - -I {{ .maxItemMemory }}m
+ - -c {{ .connectionLimit }}
+ - -v
+ - -u {{ .port }}
+ {{- range $key, $value := .extraArgs }}
+ - "-{{ $key }}{{ if $value }} {{ $value }}{{ end }}"
+ {{- end }}
+ env:
+ {{- with $.ctx.Values.global.extraEnv }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ envFrom:
+ {{- with $.ctx.Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ securityContext:
+ {{- toYaml $.ctx.Values.memcached.containerSecurityContext | nindent 12 }}
+ {{- if .extraVolumeMounts }}
+ volumeMounts:
+ {{- toYaml .extraVolumeMounts | nindent 12 }}
+ {{- end }}
+
+ {{- if $.ctx.Values.memcachedExporter.enabled }}
+ - name: exporter
+ {{- with $.ctx.Values.memcachedExporter.image }}
+ image: {{ .repository}}:{{ .tag }}
+ imagePullPolicy: {{ .pullPolicy }}
+ {{- end }}
+ ports:
+ - containerPort: 9150
+ name: http-metrics
+ args:
+ - "--memcached.address=localhost:{{ .port }}"
+ - "--web.listen-address=0.0.0.0:9150"
+ {{- range $key, $value := $.ctx.Values.memcachedExporter.extraArgs }}
+ - "--{{ $key }}{{ if $value }}={{ $value }}{{ end }}"
+ {{- end }}
+ resources:
+ {{- toYaml $.ctx.Values.memcachedExporter.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml $.ctx.Values.memcachedExporter.containerSecurityContext | nindent 12 }}
+ {{- if .extraVolumeMounts }}
+ volumeMounts:
+ {{- toYaml .extraVolumeMounts | nindent 12 }}
+ {{- end }}
+ {{- end }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/memcached/_memcached-svc.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/memcached/_memcached-svc.tpl
new file mode 100644
index 0000000..b70862a
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/memcached/_memcached-svc.tpl
@@ -0,0 +1,35 @@
+{{/*
+memcached Service
+*/}}
+{{- define "mimir.memcached.service" -}}
+{{ with (index $.ctx.Values $.component) }}
+{{- if .enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $.ctx "component" $.component) }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $.ctx "component" $.component) | nindent 4 }}
+ {{- with .service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .service.annotations | nindent 4 }}
+ namespace: {{ $.ctx.Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ ports:
+ - name: memcached-client
+ port: {{ .port }}
+ targetPort: {{ .port }}
+ {{ if $.ctx.Values.memcachedExporter.enabled }}
+ - name: http-metrics
+ port: 9150
+ targetPort: 9150
+ {{ end }}
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" $.ctx "component" $.component) | nindent 4 }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-pdb.yaml
new file mode 100644
index 0000000..dc03f8e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if index .Values "metadata-cache" "enabled" }}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "metadata-cache" ) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-servmon.yaml
new file mode 100644
index 0000000..c6d9139
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if index .Values "metadata-cache" "enabled" }}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "metadata-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml
new file mode 100644
index 0000000..5a73fe4
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml
@@ -0,0 +1 @@
+{{- include "mimir.memcached.statefulSet" (dict "ctx" $ "component" "metadata-cache" ) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-svc-headless.yaml
new file mode 100644
index 0000000..5cc5ad6
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metadata-cache/metadata-cache-svc-headless.yaml
@@ -0,0 +1 @@
+{{- include "mimir.memcached.service" (dict "ctx" $ "component" "metadata-cache" ) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/_helpers.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/_helpers.tpl
new file mode 100644
index 0000000..88008fe
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/_helpers.tpl
@@ -0,0 +1,70 @@
+{{- define "mimir.metaMonitoring.metrics.remoteWrite" -}}
+{{- $writeBackToMimir := not .url -}}
+{{- $url := .url -}}
+{{- if $writeBackToMimir -}}
+{{- $url = include "mimir.remoteWriteUrl.inCluster" .ctx }}
+{{- end -}}
+- url: {{ $url }}
+ {{- if .auth }}
+ basicAuth:
+ {{- if .auth.username }}
+ username:
+ name: {{ include "mimir.resourceName" (dict "ctx" $.ctx "component" "metrics-instance-usernames") }}
+ key: {{ .usernameKey | quote }}
+ {{- end }}
+ {{- with .auth }}
+ {{- if and .passwordSecretKey .passwordSecretName }}
+ password:
+ name: {{ .passwordSecretName | quote }}
+ key: {{ .passwordSecretKey | quote }}
+ {{- else if or .passwordSecretKey .passwordSecretName }}{{ required "Set either both passwordSecretKey and passwordSecretName or neither" nil }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ {{- $headers := .headers -}}
+ {{- if $writeBackToMimir -}}{{- $_ := set $headers "X-Scope-OrgID" "metamonitoring" -}}{{- end -}}
+ {{- with $headers }}
+ headers:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+{{- end -}}
+
+{{- define "mimir.metaMonitoring.logs.client" -}}
+{{- if .url }}
+- url: {{ .url }}
+ {{- if .auth }}
+ {{- if .auth.tenantId }}
+ tenantId: {{ .auth.tenantId | quote }}
+ {{- end }}
+ basicAuth:
+ {{- if .auth.username }}
+ username:
+ name: {{ include "mimir.resourceName" (dict "ctx" $.ctx "component" "logs-instance-usernames") }}
+ key: {{ .usernameKey | quote }}
+ {{- end }}
+ {{- with .auth }}
+ {{- if and .passwordSecretKey .passwordSecretName }}
+ password:
+ name: {{ .passwordSecretName | quote }}
+ key: {{ .passwordSecretKey | quote }}
+ {{- else if or .passwordSecretKey .passwordSecretName }}
+ {{ required "Set either both passwordSecretKey and passwordSecretName or neither" nil }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ externalLabels:
+ cluster: {{ include "mimir.clusterName" $.ctx | quote}}
+{{- end -}}
+{{- end -}}
+
+{{- define "mimir.metaMonitoring.metrics.remoteReadUrl" -}}
+{{- with $.ctx.Values.metaMonitoring.grafanaAgent.metrics }}
+{{- $writeBackToMimir := not (.remote).url -}}
+{{- if $writeBackToMimir -}}
+{{- include "mimir.remoteReadUrl.inCluster" $.ctx }}
+{{- else -}}
+{{- $parsed := urlParse (.remote).url -}}
+{{ $parsed.scheme }}://{{ $parsed.host }}/prometheus
+{{- end }}
+{{- end -}}
+{{- end -}}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-agent-cluster-role-binding.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-agent-cluster-role-binding.yaml
new file mode 100644
index 0000000..47d8cf5
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-agent-cluster-role-binding.yaml
@@ -0,0 +1,19 @@
+{{- with (.Values.metaMonitoring).grafanaAgent }}
+{{- if .enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "grafana-agent") }}
+ namespace: {{ .namespace | default $.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "meta-monitoring" ) | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "grafana-agent") }}
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "mimir.serviceAccountName" $ }}
+ namespace: {{ .namespace | default $.Release.Namespace }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-agent-cluster-role.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-agent-cluster-role.yaml
new file mode 100644
index 0000000..678d916
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-agent-cluster-role.yaml
@@ -0,0 +1,39 @@
+{{- with (.Values.metaMonitoring).grafanaAgent }}
+{{- if .enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "grafana-agent") }}
+ namespace: {{ .namespace | default $.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "meta-monitoring" ) | nindent 4 }}
+rules:
+ - apiGroups:
+ - ""
+ resources:
+ - nodes
+ - nodes/proxy
+ - nodes/metrics
+ - services
+ - endpoints
+ - pods
+ - events
+ verbs:
+ - get
+ - list
+ - watch
+ - apiGroups:
+ - networking.k8s.io
+ resources:
+ - ingresses
+ verbs:
+ - get
+ - list
+ - watch
+ - nonResourceURLs:
+ - /metrics
+ - /metrics/cadvisor
+ verbs:
+ - get
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-agent.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-agent.yaml
new file mode 100644
index 0000000..9d0afaa
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-agent.yaml
@@ -0,0 +1,59 @@
+{{- with (.Values.metaMonitoring).grafanaAgent }}
+{{- if .enabled }}
+apiVersion: monitoring.grafana.com/v1alpha1
+kind: GrafanaAgent
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "meta-monitoring") }}
+ namespace: {{ .namespace | default $.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "meta-monitoring" ) | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ serviceAccountName: {{ include "mimir.serviceAccountName" $ }}
+ {{- with .podSecurityContext }}
+ securityContext:
+ {{- . | toYaml | nindent 4 }}
+ {{- end }}
+ containers:
+ # The container specs here are merged with the ones in the operator using a strategic merge patch.
+ - name: config-reloader
+ {{- if .imageRepo }}{{ if .imageRepo.configReloader }}
+ {{- with .imageRepo.configReloader }}
+ image: {{ .repo }}/{{ .image }}:{{ .tag }}
+ {{- end }}
+ {{- end }}{{ end }}
+ {{- with .containerSecurityContext }}
+ securityContext:
+ {{- . | toYaml | nindent 8 }}
+ {{- end }}
+ - name: grafana-agent
+ {{- if .imageRepo }}{{ if .imageRepo.grafanaAgent }}
+ {{- with .imageRepo.grafanaAgent }}
+ image: {{ .repo }}/{{ .image }}:{{ .tag }}
+ {{- end }}
+ {{- end }}{{ end }}
+ {{- with .containerSecurityContext }}
+ securityContext:
+ {{- . | toYaml | nindent 8 }}
+ {{- end }}
+
+ logs:
+ instanceSelector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "meta-monitoring") | nindent 8 }}
+ # cluster label for logs is added in the LogsInstance
+ metrics:
+ scrapeInterval: {{ .metrics.scrapeInterval }}
+ instanceSelector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "meta-monitoring") | nindent 8 }}
+ externalLabels:
+ cluster: {{ include "mimir.clusterName" $ }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml
new file mode 100644
index 0000000..1b05dbc
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml
@@ -0,0 +1,27 @@
+{{- $currentScope := .Values.metaMonitoring.dashboards }}
+{{- if $currentScope.enabled }}
+{{ range $path, $_ := $.Files.Glob "mixins/dashboards/*.json" }}
+{{ $file := $path | base }}
+{{ $name := print ($file | trimSuffix ".json") "-dashboard" }}
+{{- with $currentScope}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ $name }}
+ namespace: {{ $.Release.Namespace }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $) | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+data:
+ {{ $file }}: |-
+{{ $.Files.Get $path | indent 4 }}
+---
+{{- end }}
+{{ end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/kube-state-metrics-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/kube-state-metrics-servmon.yaml
new file mode 100644
index 0000000..d8b82aa
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/kube-state-metrics-servmon.yaml
@@ -0,0 +1,49 @@
+{{- if and .Values.metaMonitoring.grafanaAgent.enabled .Values.metaMonitoring.grafanaAgent.metrics.enabled .Values.metaMonitoring.grafanaAgent.metrics.scrapeK8s.enabled }}
+{{- with .Values.metaMonitoring.serviceMonitor }}
+{{- if .enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "k8s-ksm") }}
+ namespace: {{ .namespace | default $.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "meta-monitoring") | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ endpoints:
+ - port: {{ $.Values.metaMonitoring.grafanaAgent.metrics.scrapeK8s.kubeStateMetrics.service.port }}
+ metricRelabelings:
+ - action: keep
+ regex: "(^|.*;){{ include "mimir.resourceName" (dict "ctx" $) }}.*"
+ sourceLabels:
+ - deployment
+ - statefulset
+ - pod
+ separator: ';'
+ {{- if kindIs "string" .clusterLabel }}
+ relabelings:
+ - targetLabel: cluster
+ replacement: "{{ .clusterLabel | default (include "mimir.clusterName" $) }}"
+ {{- end }}
+ path: /metrics
+ honorLabels: true # retain namespace label from kube-state-metrics
+ {{- with .scrapeTimeout }}
+ scrapeTimeout: {{ . }}
+ {{- end }}
+ {{- with ((((($.Values).metaMonitoring).grafanaAgent).metrics).scrapeK8s).kubeStateMetrics }}
+ namespaceSelector:
+ matchNames:
+ - {{ .namespace }}
+ selector:
+ matchLabels:
+ {{- toYaml .labelSelectors | nindent 6 }}
+ {{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/kubelet-cadvisor-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/kubelet-cadvisor-servmon.yaml
new file mode 100644
index 0000000..0446ce0
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/kubelet-cadvisor-servmon.yaml
@@ -0,0 +1,91 @@
+{{- if and .Values.metaMonitoring.grafanaAgent.enabled .Values.metaMonitoring.grafanaAgent.metrics.enabled .Values.metaMonitoring.grafanaAgent.metrics.scrapeK8s.enabled }}
+{{- with .Values.metaMonitoring.serviceMonitor }}
+{{- if .enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "k8s-kubelet-cadvisor") }}
+ namespace: {{ .namespace | default $.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "meta-monitoring") | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ endpoints:
+ - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+ {{- with .interval }}
+ interval: {{ . }}
+ {{- end }}
+ {{- with .scrapeTimeout }}
+ scrapeTimeout: {{ . }}
+ {{- end }}
+ port: https-metrics
+ honorLabels: true # retain namespace label from kubelet
+ relabelings:
+ - replacement: kubelet # add so that e.g. up{} metric doesn't get clashes with the other endpoint
+ targetLabel: source
+ {{- if kindIs "string" .clusterLabel }}
+ - targetLabel: cluster
+ replacement: "{{ .clusterLabel | default (include "mimir.clusterName" $) }}"
+ {{- end }}
+ {{- with .relabelings }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ metricRelabelings:
+ - action: keep
+ regex: storage-{{ include "mimir.resourceName" (dict "ctx" $) }}.*
+ sourceLabels:
+ - persistentvolumeclaim # present on kubelet_volume_stats* metrics
+ - targetLabel: instance # replace so that the metrics work with the default metrics mixin
+ sourceLabels:
+ - node
+ scheme: https
+ tlsConfig:
+ caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+ - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+ {{- with .interval }}
+ interval: {{ . }}
+ {{- end }}
+ {{- with .scrapeTimeout }}
+ scrapeTimeout: {{ . }}
+ {{- end }}
+ path: /metrics/cadvisor
+ port: https-metrics
+ honorLabels: true # retain namespace label from cadvisor
+ relabelings:
+ - replacement: cadvisor # add so that e.g. up{} metric doesn't get clashes with the other endpoint
+ targetLabel: source
+ - targetLabel: instance # replace so that the metrics work with the default metrics mixin
+ sourceLabels:
+ - node
+ {{- if kindIs "string" .clusterLabel }}
+ - targetLabel: cluster
+ replacement: "{{ .clusterLabel | default (include "mimir.clusterName" $) }}"
+ {{- end }}
+ {{- with .relabelings }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ metricRelabelings:
+ - action: keep
+ regex: {{ include "mimir.resourceName" (dict "ctx" $) }}.*
+ sourceLabels:
+ - pod
+ scheme: https
+ tlsConfig:
+ caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+ namespaceSelector:
+ matchNames:
+ # "default" is the default namespace in which the operator creates the kubelet service.
+ - default
+ selector:
+ matchLabels:
+ # This is a service added by the agent operator, so this labels is hardcoded to what the operator creates.
+ app.kubernetes.io/name: kubelet
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/logs-instance-usernames-secret.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/logs-instance-usernames-secret.yaml
new file mode 100644
index 0000000..22e558b
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/logs-instance-usernames-secret.yaml
@@ -0,0 +1,17 @@
+{{- if and .Values.metaMonitoring.grafanaAgent.enabled .Values.metaMonitoring.grafanaAgent.logs.enabled }}
+{{- with ((.Values.metaMonitoring).grafanaAgent).logs }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "logs-instance-usernames") }}
+ namespace: {{ (($.Values.metaMonitoring).grafanaAgent).namespace | default $.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "meta-monitoring") | nindent 4 }}
+data:
+ {{- range $i, $cfg := prepend (.additionalClientConfigs | default list) .remote }}
+ {{- if (($cfg).auth).username }}
+ username-{{ $i }}: {{ (($cfg).auth).username | b64enc | quote }}
+ {{- end }}
+ {{- end }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/logs-instance.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/logs-instance.yaml
new file mode 100644
index 0000000..11c47e5
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/logs-instance.yaml
@@ -0,0 +1,36 @@
+{{- with (.Values.metaMonitoring).grafanaAgent }}
+{{- if and .enabled .logs.enabled }}
+apiVersion: monitoring.grafana.com/v1alpha1
+kind: LogsInstance
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "meta-monitoring") }}
+ namespace: {{ (($.Values.metaMonitoring).grafanaAgent).namespace | default $.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "meta-monitoring" ) | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ clients:
+ {{- if or (.logs).additionalClientConfigs (.logs).remote }}
+ {{- range $i, $cfg := prepend ((.logs).additionalClientConfigs | default list) (.logs).remote }}
+ {{- with $cfg }}
+ {{- include "mimir.metaMonitoring.logs.client" (dict "ctx" $ "url" .url "auth" .auth "usernameKey" (printf "username-%d" $i)) | nindent 6 -}}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+
+ # Supply an empty namespace selector to look in all namespaces. Remove
+ # this to only look in the same namespace as the LogsInstance CR
+ podLogsNamespaceSelector: {}
+
+ podLogsSelector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" $) | nindent 6 }}
+
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/metrics-instance-usernames-secret.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/metrics-instance-usernames-secret.yaml
new file mode 100644
index 0000000..bd540fb
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/metrics-instance-usernames-secret.yaml
@@ -0,0 +1,19 @@
+{{- with .Values.metaMonitoring.grafanaAgent }}
+{{- if and .enabled .metrics.enabled }}
+{{- with .metrics }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "metrics-instance-usernames") }}
+ namespace: {{ $.Values.metaMonitoring.grafanaAgent.namespace | default $.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "meta-monitoring") | nindent 4 }}
+data:
+ {{- range $i, $cfg := prepend (.additionalRemoteWriteConfigs | default list) .remote }}
+ {{- if (($cfg).auth).username }}
+ username-{{ $i }}: {{ $cfg.auth.username | b64enc | quote }}
+ {{- end }}
+ {{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/metrics-instance.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/metrics-instance.yaml
new file mode 100644
index 0000000..4549ad1
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/metrics-instance.yaml
@@ -0,0 +1,36 @@
+{{- with .Values.metaMonitoring.grafanaAgent }}
+{{- if and .enabled .metrics.enabled }}
+apiVersion: monitoring.grafana.com/v1alpha1
+kind: MetricsInstance
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "meta-monitoring") }}
+ namespace: {{ $.Values.metaMonitoring.grafanaAgent.namespace | default $.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "meta-monitoring" ) | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ remoteWrite:
+ {{- if or (.metrics).additionalRemoteWriteConfigs (.metrics).remote }}
+ {{- range $i, $cfg := prepend ((.metrics).additionalRemoteWriteConfigs | default list) (.metrics).remote }}
+ {{- with $cfg }}
+ {{- include "mimir.metaMonitoring.metrics.remoteWrite" (dict "ctx" $ "url" .url "auth" .auth "usernameKey" (printf "username-%d" $i) "headers" .headers ) | nindent 6 -}}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+
+ # Supply an empty namespace selector to look in all namespaces. Remove
+ # this to only look in the same namespace as the MetricsInstance CR
+ serviceMonitorNamespaceSelector: {}
+
+ serviceMonitorSelector:
+ # Scrape ServiceMonitors from all components
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" $) | nindent 6 }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml
new file mode 100644
index 0000000..29023f3
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml
@@ -0,0 +1,20 @@
+{{- with .Values.metaMonitoring.prometheusRule }}
+{{- if .mimirAlerts }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "alerts") }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $) | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ namespace: {{ .namespace | default $.Release.Namespace | quote }}
+spec:
+ {{- $.Files.Get "mixins/alerts.yaml" | nindent 2 }}
+{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/pod-logs.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/pod-logs.yaml
new file mode 100644
index 0000000..78b799d
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/pod-logs.yaml
@@ -0,0 +1,45 @@
+{{- with (.Values.metaMonitoring).grafanaAgent }}
+{{- if and .enabled .logs.enabled }}
+apiVersion: monitoring.grafana.com/v1alpha1
+kind: PodLogs
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "meta-monitoring") }}
+ namespace: {{ .namespace | default $.Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $ "component" "meta-monitoring") | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ pipelineStages:
+ - cri: { }
+ relabelings:
+ - action: replace # For consistency with metrics
+ replacement: $1
+ separator: /
+ sourceLabels:
+ - __meta_kubernetes_namespace
+ - __meta_kubernetes_pod_container_name
+ targetLabel: job
+ - action: replace # Necessary for slow queries dashboard
+ sourceLabels:
+ - __meta_kubernetes_pod_container_name
+ targetLabel: name
+ - targetLabel: cluster
+ replacement: {{ include "mimir.clusterName" $ }}
+
+ namespaceSelector:
+ matchNames:
+ - {{ $.Release.Namespace | quote }}
+
+ selector:
+ matchLabels:
+ # Scrape logs from all components
+ {{- include "mimir.selectorLabels" (dict "ctx" $) | nindent 6 }}
+
+{{- end -}}
+{{- end -}}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/prometheusrule.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/prometheusrule.yaml
new file mode 100644
index 0000000..9af08e8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/prometheusrule.yaml
@@ -0,0 +1,21 @@
+{{- with .Values.metaMonitoring.prometheusRule }}
+{{- if .enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "prometheus-rule") }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $) | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ namespace: {{ .namespace | default $.Release.Namespace | quote }}
+spec:
+ groups:
+ {{- toYaml .groups | nindent 4 }}
+{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/recording-rules.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/recording-rules.yaml
new file mode 100644
index 0000000..682236f
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/metamonitoring/recording-rules.yaml
@@ -0,0 +1,20 @@
+{{- with .Values.metaMonitoring.prometheusRule }}
+{{- if .mimirRules }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "rules") }}
+ {{- with .annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" $) | nindent 4 }}
+ {{- with .labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ namespace: {{ .namespace | default $.Release.Namespace | quote }}
+spec:
+ {{- $.Files.Get "mixins/rules.yaml" | nindent 2 }}
+{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/mimir-config.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/mimir-config.yaml
new file mode 100644
index 0000000..af72cc8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/mimir-config.yaml
@@ -0,0 +1,21 @@
+{{- if not .Values.useExternalConfig }}
+apiVersion: v1
+{{- if eq .Values.configStorageType "Secret" }}
+kind: Secret
+{{- else }}
+kind: ConfigMap
+{{- end }}
+metadata:
+ name: {{ tpl .Values.externalConfigSecretName . }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+{{- if eq .Values.configStorageType "Secret" }}
+data:
+ mimir.yaml: {{ include "mimir.calculatedConfig" . | b64enc }}
+{{- else }}
+data:
+ mimir.yaml: |
+ {{ include "mimir.calculatedConfig" . | nindent 4 }}
+{{- end -}}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/minio-secrets.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/minio-secrets.yaml
new file mode 100644
index 0000000..e646f65
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/minio-secrets.yaml
@@ -0,0 +1,13 @@
+{{- if ((.Values.testing).minio).use_secret }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: mimir-minio-secret
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+type: Opaque
+data:
+ MINIO_ROOT_USER: {{ .Values.minio.rootUser | b64enc }}
+ MINIO_ROOT_PASSWORD: {{ .Values.minio.rootPassword | b64enc }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/minio/create-bucket-job.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/minio/create-bucket-job.yaml
new file mode 100644
index 0000000..7750dba
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/minio/create-bucket-job.yaml
@@ -0,0 +1,92 @@
+# Minio provides post-install hook to create bucket
+# however the hook won't be executed if helm install is run
+# with --wait flag. Hence this job is a workaround for that.
+# See https://github.com/grafana/mimir/issues/2464
+{{- if .Values.minio.enabled }}
+{{- if .Values.minio.buckets }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+ {{- $minioSub := index .Subcharts "minio" }}
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "make-minio-buckets") }}-{{ $minioSub.Chart.Version }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ app: {{ template "minio.name" . }}-make-bucket-job
+ chart: {{ template "minio.chart" . }}
+ release: {{ .Release.Name }}
+ heritage: {{ .Release.Service }}
+{{- with .Values.minio.makeBucketJob.annotations }}
+ annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+spec:
+ template:
+ metadata:
+ labels:
+ app: {{ template "minio.name" . }}-job
+ release: {{ .Release.Name }}
+{{- if .Values.minio.podLabels }}
+{{ toYaml .Values.minio.podLabels | indent 8 }}
+{{- end }}
+{{- if .Values.minio.makeBucketJob.podAnnotations }}
+ annotations:
+{{ toYaml .Values.minio.makeBucketJob.podAnnotations | indent 8 }}
+{{- end }}
+ spec:
+ restartPolicy: OnFailure
+{{- include "minio.imagePullSecrets" . | indent 6 }}
+{{- if .Values.minio.nodeSelector }}
+ nodeSelector:
+{{ toYaml .Values.minio.makeBucketJob.nodeSelector | indent 8 }}
+{{- end }}
+{{- with .Values.minio.makeBucketJob.affinity }}
+ affinity:
+{{ toYaml . | indent 8 }}
+{{- end }}
+{{- with .Values.minio.makeBucketJob.tolerations }}
+ tolerations:
+{{ toYaml . | indent 8 }}
+{{- end }}
+{{- if .Values.minio.makeBucketJob.securityContext.enabled }}
+ securityContext:
+ runAsUser: {{ .Values.minio.makeBucketJob.securityContext.runAsUser }}
+ runAsGroup: {{ .Values.minio.makeBucketJob.securityContext.runAsGroup }}
+ fsGroup: {{ .Values.minio.makeBucketJob.securityContext.fsGroup }}
+{{- end }}
+ volumes:
+ - name: minio-configuration
+ projected:
+ sources:
+ - configMap:
+ name: {{ .Release.Name }}-minio
+ - secret:
+ name: {{ .Release.Name }}-minio
+ {{- if .Values.minio.tls.enabled }}
+ - name: cert-secret-volume-mc
+ secret:
+ secretName: {{ .Values.minio.tls.certSecret }}
+ items:
+ - key: {{ .Values.minio.tls.publicCrt }}
+ path: CAs/public.crt
+ {{ end }}
+ containers:
+ - name: minio-mc
+ image: "{{ .Values.minio.mcImage.repository }}:{{ .Values.minio.mcImage.tag }}"
+ imagePullPolicy: {{ .Values.minio.mcImage.pullPolicy }}
+ command: ["/bin/sh", "/config/initialize"]
+ env:
+ - name: MINIO_ENDPOINT
+ value: {{ .Release.Name }}-minio
+ - name: MINIO_PORT
+ value: {{ .Values.minio.service.port | quote }}
+ volumeMounts:
+ - name: minio-configuration
+ mountPath: /config
+ {{- if .Values.minio.tls.enabled }}
+ - name: cert-secret-volume-mc
+ mountPath: {{ .Values.minio.configPathmc }}certs
+ {{ end }}
+ resources:
+{{ toYaml .Values.minio.makeBucketJob.resources | indent 10 }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/_helpers.tpl b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/_helpers.tpl
new file mode 100644
index 0000000..be55df3
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/_helpers.tpl
@@ -0,0 +1,17 @@
+{{/*
+nginx auth secret name
+*/}}
+{{- define "mimir.nginxAuthSecret" -}}
+{{ .Values.nginx.basicAuth.existingSecret | default (include "mimir.resourceName" (dict "ctx" . "component" "nginx") ) }}
+{{- end }}
+
+{{/*
+Returns the HorizontalPodAutoscaler API version for this version of kubernetes.
+*/}}
+{{- define "mimir.hpa.version" -}}
+{{- if semverCompare ">= 1.23-0" (include "mimir.kubeVersion" .) -}}
+autoscaling/v2
+{{- else -}}
+autoscaling/v2beta1
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/ingress.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/ingress.yaml
new file mode 100644
index 0000000..1ca7da7
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/ingress.yaml
@@ -0,0 +1,48 @@
+{{- if not .Values.enterprise.enabled -}}
+{{- if .Values.nginx.enabled -}}
+{{- if .Values.nginx.ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }}
+ {{- with .Values.nginx.ingress.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ {{- if .Values.nginx.ingress.ingressClassName }}
+ ingressClassName: {{ .Values.nginx.ingress.ingressClassName }}
+ {{- end -}}
+ {{- if .Values.nginx.ingress.tls }}
+ tls:
+ {{- range .Values.nginx.ingress.tls }}
+ - hosts:
+ {{- range .hosts }}
+ - {{ . | quote }}
+ {{- end }}
+ {{- with .secretName }}
+ secretName: {{ . }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ rules:
+ {{- range .Values.nginx.ingress.hosts }}
+ - host: {{ .host | quote }}
+ http:
+ paths:
+ {{- range .paths }}
+ - path: {{ .path }}
+ pathType: {{ .pathType }}
+ backend:
+ service:
+ name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "nginx") }}
+ port:
+ number: {{ $.Values.nginx.service.port }}
+ {{- end }}
+ {{- end }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-configmap.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-configmap.yaml
new file mode 100644
index 0000000..2f4ebef
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-configmap.yaml
@@ -0,0 +1,14 @@
+{{- if not .Values.enterprise.enabled -}}
+{{- if .Values.nginx.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+data:
+ nginx.conf: |
+ {{- tpl .Values.nginx.nginxConfig.file . | nindent 4 }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-dep.yaml
new file mode 100644
index 0000000..cc62d8c
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-dep.yaml
@@ -0,0 +1,132 @@
+{{- if not .Values.enterprise.enabled -}}
+{{- if .Values.nginx.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.nginx.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+{{- if not .Values.nginx.autoscaling.enabled }}
+ replicas: {{ .Values.nginx.replicas }}
+{{- end }}
+{{- with .Values.nginx.deploymentStrategy }}
+ strategy:
+{{ toYaml . | trim | indent 4 }}
+{{- end }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "nginx") | nindent 6 }}
+ template:
+ metadata:
+ annotations:
+ checksum/config: {{ include (print .Template.BasePath "/nginx/nginx-configmap.yaml") . | sha256sum }}
+ {{- with .Values.global.podAnnotations }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.nginx.podAnnotations }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "nginx") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ include "mimir.serviceAccountName" . }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ {{- if .Values.nginx.priorityClassName }}
+ priorityClassName: {{ .Values.nginx.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "nginx") | nindent 8 }}
+ terminationGracePeriodSeconds: {{ .Values.nginx.terminationGracePeriodSeconds }}
+ containers:
+ - name: nginx
+ image: {{ .Values.nginx.image.registry }}/{{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag }}
+ imagePullPolicy: {{ .Values.nginx.image.pullPolicy }}
+ {{- with .Values.nginx.extraArgs }}
+ args:
+ {{- range $key, $value := . }}
+ - "-{{ $key }} {{ $value }}"
+ {{- end }}
+ {{- end }}
+ ports:
+ - name: http-metric
+ containerPort: 8080
+ protocol: TCP
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.nginx.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.nginx.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ readinessProbe:
+ {{- toYaml .Values.nginx.readinessProbe | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.nginx.containerSecurityContext | nindent 12 }}
+ volumeMounts:
+ - name: config
+ mountPath: /etc/nginx/nginx.conf
+ subPath: nginx.conf
+ {{- if .Values.nginx.basicAuth.enabled }}
+ - name: auth
+ mountPath: /etc/nginx/secrets
+ {{- end }}
+ - name: tmp
+ mountPath: /tmp
+ - name: docker-entrypoint-d-override
+ mountPath: /docker-entrypoint.d
+ {{- if .Values.nginx.extraVolumeMounts }}
+ {{- toYaml .Values.nginx.extraVolumeMounts | nindent 12 }}
+ {{- end }}
+ resources:
+ {{- toYaml .Values.nginx.resources | nindent 12 }}
+ {{- if .Values.nginx.extraContainers }}
+ {{- toYaml .Values.nginx.extraContainers | nindent 8}}
+ {{- end }}
+ {{- with .Values.nginx.affinity }}
+ affinity:
+ {{- tpl . $ | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "nginx") | nindent 6 }}
+ {{- with .Values.nginx.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.nginx.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ volumes:
+ - name: config
+ configMap:
+ name: {{ include "mimir.fullname" . }}-nginx
+ {{- if .Values.nginx.basicAuth.enabled }}
+ - name: auth
+ secret:
+ secretName: {{ include "mimir.nginxAuthSecret" . }}
+ {{- end }}
+ - name: tmp
+ emptyDir: {}
+ - name: docker-entrypoint-d-override
+ emptyDir: {}
+ {{- if .Values.nginx.extraVolumes }}
+ {{- toYaml .Values.nginx.extraVolumes | nindent 8 }}
+ {{- end }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-pdb.yaml
new file mode 100644
index 0000000..712a665
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if and .Values.nginx.enabled (not .Values.enterprise.enabled) -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "nginx") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-secret.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-secret.yaml
new file mode 100644
index 0000000..3bb1d01
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-secret.yaml
@@ -0,0 +1,14 @@
+{{- if not .Values.enterprise.enabled -}}
+{{- if and .Values.nginx.enabled .Values.nginx.basicAuth.enabled (not .Values.nginx.basicAuth.existingSecret) }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+stringData:
+ .htpasswd: |
+ {{- tpl .Values.nginx.basicAuth.htpasswd $ | nindent 4 }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-svc.yaml
new file mode 100644
index 0000000..1cad15b
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-svc.yaml
@@ -0,0 +1,34 @@
+{{- if not .Values.enterprise.enabled -}}
+{{- if .Values.nginx.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }}
+ {{- with .Values.nginx.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.nginx.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: {{ .Values.nginx.service.type }}
+ {{- with .Values.nginx.service.clusterIP }}
+ clusterIP: {{ . }}
+ {{- end }}
+ {{- if and (eq "LoadBalancer" .Values.nginx.service.type) .Values.nginx.service.loadBalancerIP }}
+ loadBalancerIP: {{ .Values.nginx.service.loadBalancerIP }}
+ {{- end }}
+ ports:
+ - name: http-metric
+ port: {{ .Values.nginx.service.port }}
+ targetPort: http-metric
+ {{- if and (eq "NodePort" .Values.nginx.service.type) .Values.nginx.service.nodePort }}
+ nodePort: {{ .Values.nginx.service.nodePort }}
+ {{- end }}
+ protocol: TCP
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "nginx") | nindent 4 }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-v2-hpa.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-v2-hpa.yaml
new file mode 100644
index 0000000..3b7e022
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-v2-hpa.yaml
@@ -0,0 +1,37 @@
+{{- if not .Values.enterprise.enabled -}}
+{{- if .Values.nginx.autoscaling.enabled }}
+{{- if eq (include "mimir.hpa.version" .) "autoscaling/v2" }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ minReplicas: {{ .Values.nginx.autoscaling.minReplicas }}
+ maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }}
+ metrics:
+ {{- with .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }}
+ - type: Resource
+ resource:
+ name: memory
+ target:
+ type: Utilization
+ averageUtilization: {{ . }}
+ {{- end }}
+ {{- with .Values.nginx.autoscaling.targetCPUUtilizationPercentage }}
+ - type: Resource
+ resource:
+ name: cpu
+ target:
+ type: Utilization
+ averageUtilization: {{ . }}
+ {{- end }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-v2beta1-hpa.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-v2beta1-hpa.yaml
new file mode 100644
index 0000000..9ce76db
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/nginx-v2beta1-hpa.yaml
@@ -0,0 +1,33 @@
+{{- if not .Values.enterprise.enabled -}}
+{{- if .Values.nginx.autoscaling.enabled }}
+{{- if eq (include "mimir.hpa.version" .) "autoscaling/v2beta1" }}
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ minReplicas: {{ .Values.nginx.autoscaling.minReplicas }}
+ maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }}
+ metrics:
+ {{- with .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }}
+ - type: Resource
+ resource:
+ name: memory
+ targetAverageUtilization: {{ . }}
+ {{- end }}
+ {{- with .Values.nginx.autoscaling.targetCPUUtilizationPercentage }}
+ - type: Resource
+ resource:
+ name: cpu
+ targetAverageUtilization: {{ . }}
+ {{- end }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/route.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/route.yaml
new file mode 100644
index 0000000..4eb3645
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/nginx/route.yaml
@@ -0,0 +1,25 @@
+{{- if .Values.nginx.route.enabled }}
+kind: Route
+apiVersion: route.openshift.io/v1
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }}
+ {{- with .Values.nginx.route.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ host: {{ .Values.nginx.route.host }}
+ to:
+ kind: Service
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }}
+ weight: 100
+ port:
+ targetPort: http-metric
+{{- if .Values.nginx.route.tls }}
+ tls:
+ {{- toYaml .Values.nginx.route.tls | nindent 4 }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml
new file mode 100644
index 0000000..19b4142
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml
@@ -0,0 +1,139 @@
+{{- if .Values.overrides_exporter.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ annotations:
+ {{- toYaml .Values.overrides_exporter.annotations | nindent 4 }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "overrides-exporter") | nindent 4 }}
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "overrides-exporter") }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ replicas: {{ .Values.overrides_exporter.replicas }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "overrides-exporter") | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.overrides_exporter.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "overrides-exporter") | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "overrides-exporter") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.overrides_exporter.priorityClassName }}
+ priorityClassName: {{ .Values.overrides_exporter.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "overrides-exporter") | nindent 8 }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ {{- with .Values.overrides_exporter.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ containers:
+ - name: overrides-exporter
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=overrides-exporter"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- range $key, $value := .Values.overrides_exporter.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.overrides_exporter.extraVolumeMounts }}
+ {{ toYaml .Values.overrides_exporter.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: storage
+ mountPath: "/data"
+ subPath: {{ .Values.overrides_exporter.persistence.subPath }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.overrides_exporter.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.overrides_exporter.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.overrides_exporter.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.overrides_exporter.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.overrides_exporter.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.overrides_exporter }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.overrides_exporter.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.overrides_exporter.extraContainers }}
+ {{ toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.overrides_exporter.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.overrides_exporter.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "overrides-exporter") | nindent 6 }}
+ {{- with .Values.overrides_exporter.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.overrides_exporter.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+ {{- if .Values.overrides_exporter.extraVolumes }}
+ {{ toYaml .Values.overrides_exporter.extraVolumes | nindent 8}}
+ {{- end }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-pdb.yaml
new file mode 100644
index 0000000..ab30728
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.overrides_exporter.enabled -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "overrides-exporter") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-servmon.yaml
new file mode 100644
index 0000000..809d0e7
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.overrides_exporter.enabled -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "overrides-exporter") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml
new file mode 100644
index 0000000..5e42a7f
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml
@@ -0,0 +1,27 @@
+{{- if .Values.overrides_exporter.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "overrides-exporter") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "overrides-exporter") | nindent 4 }}
+ {{- with .Values.overrides_exporter.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.overrides_exporter.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "overrides-exporter") | nindent 4 }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/podsecuritypolicy.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/podsecuritypolicy.yaml
new file mode 100644
index 0000000..f9113d3
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/podsecuritypolicy.yaml
@@ -0,0 +1,45 @@
+{{- if eq (include "mimir.rbac.usePodSecurityPolicy" .) "true" }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" .) }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ annotations:
+ "seccomp.security.alpha.kubernetes.io/allowedProfileNames": {{ .Values.rbac.podSecurityPolicy.seccompProfile }}
+spec:
+ privileged: {{ .Values.rbac.podSecurityPolicy.privileged }}
+ allowPrivilegeEscalation: {{ .Values.rbac.podSecurityPolicy.allowPrivilegeEscalation }}
+ volumes:
+ - 'configMap'
+ - 'emptyDir'
+ - 'persistentVolumeClaim'
+ - 'secret'
+ {{- range $volumes := .Values.rbac.podSecurityPolicy.additionalVolumes }}
+ - '{{ $volumes }}'
+ {{- end }}
+ hostNetwork: {{ .Values.rbac.podSecurityPolicy.hostNetwork }}
+ hostIPC: {{ .Values.rbac.podSecurityPolicy.hostIPC }}
+ hostPID: {{ .Values.rbac.podSecurityPolicy.hostPID }}
+ runAsUser:
+ rule: {{ .Values.rbac.podSecurityPolicy.runAsUser.rule }}
+ seLinux:
+ rule: {{ .Values.rbac.podSecurityPolicy.seLinux.rule }}
+ supplementalGroups:
+ rule: {{ .Values.rbac.podSecurityPolicy.supplementalGroups.rule }}
+ ranges:
+ {{- range $range := .Values.rbac.podSecurityPolicy.supplementalGroups.ranges }}
+ - min: {{ $range.min }}
+ max: {{ $range.max }}
+ {{- end }}
+ fsGroup:
+ rule: {{ .Values.rbac.podSecurityPolicy.fsGroup.rule }}
+ ranges:
+ {{- range $range := .Values.rbac.podSecurityPolicy.fsGroup.ranges }}
+ - min: {{ $range.min }}
+ max: {{ $range.max }}
+ {{- end }}
+ readOnlyRootFilesystem: {{ .Values.rbac.podSecurityPolicy.readOnlyRootFilesystem }}
+ requiredDropCapabilities:
+ - ALL
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-dep.yaml
new file mode 100644
index 0000000..b9c6dbd
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-dep.yaml
@@ -0,0 +1,162 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "querier") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.querier.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ {{- if or (not .Values.querier.kedaAutoscaling.enabled) (.Values.querier.kedaAutoscaling.preserveReplicas) }}
+ # If replicas is not number (when using values file it's float64, when using --set arg it's int64) and is false (i.e. null) don't set it
+ {{- if or (or (kindIs "int64" .Values.querier.replicas) (kindIs "float64" .Values.querier.replicas)) (.Values.querier.replicas) }}
+ replicas: {{ .Values.querier.replicas }}
+ {{- end }}
+ {{- end }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.querier.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "querier") | nindent 8 }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.querier.priorityClassName }}
+ priorityClassName: {{ .Values.querier.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "querier") | nindent 8 }}
+ {{- with .Values.querier.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end}}
+ {{- end }}
+ containers:
+ - name: querier
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=querier"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- if .Values.ingester.zoneAwareReplication.migration.enabled }}
+ {{- if not .Values.ingester.zoneAwareReplication.migration.readPath }}
+ - "-ingester.ring.zone-awareness-enabled=false"
+ {{- end }}
+ {{- end }}
+ {{- if .Values.store_gateway.zoneAwareReplication.migration.enabled }}
+ {{- if not .Values.store_gateway.zoneAwareReplication.migration.readPath }}
+ - "-store-gateway.sharding-ring.prefix=collectors/"
+ - "-store-gateway.sharding-ring.zone-awareness-enabled=false"
+ {{- end }}
+ {{- end }}
+ {{- range $key, $value := .Values.querier.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.querier.extraVolumeMounts }}
+ {{ toYaml .Values.querier.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: storage
+ mountPath: "/data"
+ subPath: {{ .Values.querier.persistence.subPath }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ - name: memberlist
+ containerPort: {{ include "mimir.memberlistBindPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.querier.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.querier.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.querier.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.querier.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.querier.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $cpu_request := dig "requests" "cpu" nil .Values.querier.resources }}
+ {{- if $cpu_request }}
+ {{- $cpu_request_doubled := include "mimir.parseCPU" (dict "value" $cpu_request) | float64 | mulf 2 | ceil }}
+ {{- $cpu_request_plus_four := include "mimir.parseCPU" (dict "value" $cpu_request) | float64 | addf 4 | ceil }}
+ - name: "GOMAXPROCS"
+ value: {{ max $cpu_request_doubled $cpu_request_plus_four | toString | toYaml }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.querier }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.querier.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+{{- if .Values.querier.extraContainers }}
+{{ toYaml .Values.querier.extraContainers | indent 8}}
+{{- end }}
+ {{- with .Values.querier.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.querier.affinity}}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "querier") | nindent 6 }}
+ {{- with .Values.querier.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.querier.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+{{- if .Values.querier.extraVolumes }}
+{{ toYaml .Values.querier.extraVolumes | indent 8}}
+{{- end }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-pdb.yaml
new file mode 100644
index 0000000..07ed28d
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-pdb.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "querier" "memberlist" true) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-servmon.yaml
new file mode 100644
index 0000000..516603c
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-servmon.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "querier" "memberlist" true) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-so.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-so.yaml
new file mode 100644
index 0000000..6649ced
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-so.yaml
@@ -0,0 +1,47 @@
+{{- if .Values.querier.kedaAutoscaling.enabled }}
+{{- if not .Values.query_scheduler.enabled }}
+{{- fail "KEDA autoscaling for querier requires query scheduler to be enabled" }}
+{{- end }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "querier") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "querier") | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.querier.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ advanced:
+ horizontalPodAutoscalerConfig:
+ {{- with .Values.querier.kedaAutoscaling.behavior }}
+ behavior:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ maxReplicaCount: {{ .Values.querier.kedaAutoscaling.maxReplicaCount }}
+ minReplicaCount: {{ .Values.querier.kedaAutoscaling.minReplicaCount }}
+ pollingInterval: {{ .Values.kedaAutoscaling.pollingInterval }}
+ scaleTargetRef:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "querier") }}
+ apiVersion: apps/v1
+ kind: Deployment
+ triggers:
+ - metadata:
+ query: sum(max_over_time(cortex_query_scheduler_inflight_requests{container="query-scheduler",namespace="{{ .Release.Namespace }}",quantile="0.5"}[1m]))
+ serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
+ threshold: {{ .Values.querier.kedaAutoscaling.querySchedulerInflightRequestsThreshold | quote }}
+ {{- if .Values.kedaAutoscaling.customHeaders }}
+ customHeaders: {{ (include "mimir.lib.mapToCSVString" (dict "map" .Values.kedaAutoscaling.customHeaders)) | quote }}
+ {{- end }}
+ name: cortex_querier_hpa_default
+ type: prometheus
+ - metadata:
+ query: sum(rate(cortex_querier_request_duration_seconds_sum{container="querier",namespace="{{ .Release.Namespace }}"}[1m]))
+ serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
+ threshold: {{ .Values.querier.kedaAutoscaling.querySchedulerInflightRequestsThreshold | quote }}
+ {{- if .Values.kedaAutoscaling.customHeaders }}
+ customHeaders: {{ (include "mimir.lib.mapToCSVString" (dict "map" .Values.kedaAutoscaling.customHeaders)) | quote }}
+ {{- end }}
+ name: cortex_querier_hpa_default_requests_duration
+ type: prometheus
+{{- end }}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-svc.yaml
new file mode 100644
index 0000000..1eb323e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/querier/querier-svc.yaml
@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "querier") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 4 }}
+ {{- with .Values.querier.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.querier.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 4 }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml
new file mode 100644
index 0000000..132ab1d
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml
@@ -0,0 +1,147 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-frontend") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "query-frontend") | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.query_frontend.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ {{- if or (not .Values.query_frontend.kedaAutoscaling.enabled) (.Values.query_frontend.kedaAutoscaling.preserveReplicas) }}
+ # If replicas is not number (when using values file it's float64, when using --set arg it's int64) and is false (i.e. null) don't set it
+ {{- if or (or (kindIs "int64" .Values.query_frontend.replicas) (kindIs "float64" .Values.query_frontend.replicas)) (.Values.query_frontend.replicas) }}
+ replicas: {{ .Values.query_frontend.replicas }}
+ {{- end }}
+ {{- end }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "query-frontend") | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.query_frontend.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "query-frontend") | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "query-frontend") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.query_frontend.priorityClassName }}
+ priorityClassName: {{ .Values.query_frontend.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "query-frontend") | nindent 8 }}
+ {{- with .Values.query_frontend.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end}}
+ {{- end }}
+ containers:
+ - name: query-frontend
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=query-frontend"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ # Reduce the likelihood of queries hitting terminated query-frontends.
+ - "-server.grpc.keepalive.max-connection-age=30s"
+ - "-shutdown-delay=90s"
+ {{- range $key, $value := .Values.query_frontend.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.query_frontend.extraVolumeMounts }}
+ {{ toYaml .Values.query_frontend.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ - name: storage
+ mountPath: /data
+ {{- if .Values.query_frontend.persistence.subPath }}
+ subPath: {{ .Values.query_frontend.persistence.subPath }}
+ {{- end }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.query_frontend.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.query_frontend.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.query_frontend.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.query_frontend.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.query_frontend.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.query_frontend }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.query_frontend.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+{{- if .Values.query_frontend.extraContainers }}
+{{ toYaml .Values.query_frontend.extraContainers | indent 8}}
+{{- end }}
+ {{- with .Values.query_frontend.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.query_frontend.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "query-frontend") | nindent 6 }}
+ {{- with .Values.query_frontend.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.query_frontend.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+{{- if .Values.query_frontend.extraVolumes }}
+{{ toYaml .Values.query_frontend.extraVolumes | indent 8}}
+{{- end }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-pdb.yaml
new file mode 100644
index 0000000..e6cb8e1
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-pdb.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "query-frontend") }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-servmon.yaml
new file mode 100644
index 0000000..0b2585f
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-servmon.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "query-frontend") }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-so.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-so.yaml
new file mode 100644
index 0000000..2ed9502
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-so.yaml
@@ -0,0 +1,44 @@
+{{- if .Values.query_frontend.kedaAutoscaling.enabled }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-frontend") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "query-frontend") | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.query_frontend.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ advanced:
+ horizontalPodAutoscalerConfig:
+ {{- with .Values.query_frontend.kedaAutoscaling.behavior }}
+ behavior:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ maxReplicaCount: {{ .Values.query_frontend.kedaAutoscaling.maxReplicaCount }}
+ minReplicaCount: {{ .Values.query_frontend.kedaAutoscaling.minReplicaCount }}
+ pollingInterval: {{ .Values.kedaAutoscaling.pollingInterval }}
+ scaleTargetRef:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-frontend") }}
+ apiVersion: apps/v1
+ kind: Deployment
+ triggers:
+ - metadata:
+ query: max_over_time(sum(sum by (pod) (rate(container_cpu_usage_seconds_total{container="query-frontend",namespace="{{ .Release.Namespace }}"}[5m])) and max by (pod) (up{container="query-frontend",namespace="{{ .Release.Namespace }}"}) > 0)[15m:]) * 1000
+ serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
+ {{- $cpu_request := dig "requests" "cpu" nil .Values.query_frontend.resources }}
+ threshold: {{ mulf (include "mimir.cpuToMilliCPU" (dict "value" $cpu_request)) (divf .Values.query_frontend.kedaAutoscaling.targetCPUUtilizationPercentage 100) | floor | int64 | quote }}
+ {{- if .Values.kedaAutoscaling.customHeaders }}
+ customHeaders: {{ (include "mimir.lib.mapToCSVString" (dict "map" .Values.kedaAutoscaling.customHeaders)) | quote }}
+ {{- end }}
+ type: prometheus
+ - metadata:
+ query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="query-frontend",namespace="{{ .Release.Namespace }}"}) and max by (pod) (up{container="query-frontend",namespace="{{ .Release.Namespace }}"}) > 0) or vector(0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="query-frontend",namespace="{{ .Release.Namespace }}", resource="memory"}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="query-frontend",namespace="{{ .Release.Namespace }}"}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="query-frontend",namespace="{{ .Release.Namespace }}", reason="OOMKilled"}) or vector(0))
+ serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
+ {{- $mem_request := dig "requests" "memory" nil .Values.query_frontend.resources }}
+ threshold: {{ mulf (include "mimir.siToBytes" (dict "value" $mem_request)) (divf .Values.query_frontend.kedaAutoscaling.targetMemoryUtilizationPercentage 100) | floor | int64 | quote }}
+ {{- if .Values.kedaAutoscaling.customHeaders }}
+ customHeaders: {{ (include "mimir.lib.mapToCSVString" (dict "map" .Values.kedaAutoscaling.customHeaders)) | quote }}
+ {{- end }}
+ type: prometheus
+{{- end }}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml
new file mode 100644
index 0000000..b30e7f8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml
@@ -0,0 +1,30 @@
+{{- if not .Values.query_scheduler.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-frontend") }}-headless
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "query-frontend") | nindent 4 }}
+ prometheus.io/service-monitor: "false"
+ {{- with .Values.query_frontend.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.query_frontend.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ publishNotReadyAddresses: true
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "query-frontend") | nindent 4 }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml
new file mode 100644
index 0000000..3eec5d6
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml
@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-frontend") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "query-frontend") | nindent 4 }}
+ {{- with .Values.query_frontend.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.query_frontend.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "query-frontend") | nindent 4 }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml
new file mode 100644
index 0000000..e763019
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml
@@ -0,0 +1,140 @@
+{{- if .Values.query_scheduler.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-scheduler") }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "query-scheduler") | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.query_scheduler.annotations | nindent 4 }}
+spec:
+ replicas: {{ .Values.query_scheduler.replicas }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "query-scheduler") | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.query_scheduler.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "query-scheduler") | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "query-scheduler") | nindent 8 }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.query_scheduler.priorityClassName }}
+ priorityClassName: {{ .Values.query_scheduler.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "query-scheduler") | nindent 8 }}
+ {{- with .Values.query_scheduler.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end}}
+ {{- end }}
+ containers:
+ - name: query-scheduler
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=query-scheduler"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- range $key, $value := .Values.query_scheduler.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.query_scheduler.extraVolumeMounts }}
+ {{ toYaml .Values.query_scheduler.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ - name: storage
+ mountPath: /data
+ {{- if .Values.query_scheduler.persistence.subPath }}
+ subPath: {{ .Values.query_scheduler.persistence.subPath }}
+ {{- end }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.query_scheduler.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.query_scheduler.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.query_scheduler.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.query_scheduler.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.query_scheduler.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.query_scheduler }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.query_scheduler.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+{{- if .Values.query_scheduler.extraContainers }}
+{{ toYaml .Values.query_scheduler.extraContainers | indent 8}}
+{{- end }}
+ {{- with .Values.query_scheduler.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.query_scheduler.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "query-scheduler") | nindent 6 }}
+ {{- with .Values.query_scheduler.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.query_scheduler.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+{{- if .Values.query_scheduler.extraVolumes }}
+{{ toYaml .Values.query_scheduler.extraVolumes | indent 8}}
+{{- end }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-pdb.yaml
new file mode 100644
index 0000000..f38e16f
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.query_scheduler.podDisruptionBudget -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "query-scheduler") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-servmon.yaml
new file mode 100644
index 0000000..f20a68a
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.query_scheduler.enabled }}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "query-scheduler") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-svc-headless.yaml
new file mode 100644
index 0000000..c02dba7
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-svc-headless.yaml
@@ -0,0 +1,30 @@
+{{- if .Values.query_scheduler.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-scheduler") }}-headless
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "query-scheduler") | nindent 4 }}
+ prometheus.io/service-monitor: "false"
+ {{- with .Values.query_scheduler.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.query_scheduler.service.annotations | nindent 4 }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ publishNotReadyAddresses: true
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "query-scheduler") | nindent 4 }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-svc.yaml
new file mode 100644
index 0000000..abef45e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/query-scheduler/query-scheduler-svc.yaml
@@ -0,0 +1,27 @@
+{{- if .Values.query_scheduler.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-scheduler") }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "query-scheduler") | nindent 4 }}
+ {{- with .Values.query_scheduler.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.query_scheduler.service.annotations | nindent 4 }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "query-scheduler") | nindent 4 }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-pdb.yaml
new file mode 100644
index 0000000..097cff9
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if index .Values "results-cache" "enabled" }}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "results-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-servmon.yaml
new file mode 100644
index 0000000..6b14392
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if index .Values "results-cache" "enabled" }}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "results-cache") }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml
new file mode 100644
index 0000000..8689c85
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml
@@ -0,0 +1 @@
+{{- include "mimir.memcached.statefulSet" (dict "ctx" $ "component" "results-cache" ) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-svc-headless.yaml
new file mode 100644
index 0000000..1cdec33
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/results-cache/results-cache-svc-headless.yaml
@@ -0,0 +1 @@
+{{- include "mimir.memcached.service" (dict "ctx" $ "component" "results-cache" ) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/role.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/role.yaml
new file mode 100644
index 0000000..8e1eff7
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/role.yaml
@@ -0,0 +1,28 @@
+{{- $usePSP := (eq (include "mimir.rbac.usePodSecurityPolicy" .) "true") }}
+{{- $useSCC := (eq (include "mimir.rbac.useSecurityContextConstraints" .) "true") }}
+{{- if or $usePSP $useSCC }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" .) }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+rules:
+{{- if $usePSP }}
+- apiGroups: ['extensions']
+ resources: ['podsecuritypolicies']
+ verbs: ['use']
+ resourceNames: [{{ include "mimir.resourceName" (dict "ctx" .) }}]
+{{- end }}
+{{- if $useSCC }}
+- apiGroups:
+ - security.openshift.io
+ resources:
+ - securitycontextconstraints
+ verbs:
+ - use
+ resourceNames:
+ - {{ include "mimir.resourceName" (dict "ctx" .) }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/rolebinding.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/rolebinding.yaml
new file mode 100644
index 0000000..7450540
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/rolebinding.yaml
@@ -0,0 +1,26 @@
+{{- $usePSP := (eq (include "mimir.rbac.usePodSecurityPolicy" .) "true") }}
+{{- $useSCC := (eq (include "mimir.rbac.useSecurityContextConstraints" .) "true") }}
+{{- if or $usePSP $useSCC }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" .) }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ include "mimir.resourceName" (dict "ctx" .) }}
+subjects:
+- kind: ServiceAccount
+ name: {{ template "mimir.serviceAccountName" . }}
+{{- if .Values.rollout_operator.enabled }}
+- kind: ServiceAccount
+ name: {{ include "rollout-operator.serviceAccountName" . }}
+{{- end }}
+{{- if .Values.ruler.serviceAccount.create }}
+- kind: ServiceAccount
+ name: {{ template "mimir.ruler.serviceAccountName" . }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-dep.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-dep.yaml
new file mode 100644
index 0000000..7169196
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -0,0 +1,159 @@
+{{- if .Values.ruler.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ruler") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.ruler.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ {{- if or (not .Values.ruler.kedaAutoscaling.enabled) (.Values.ruler.kedaAutoscaling.preserveReplicas) }}
+ replicas: {{ .Values.ruler.replicas }}
+ {{- end }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 6 }}
+ strategy:
+ {{- toYaml .Values.ruler.strategy | nindent 4 }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "ruler") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ template "mimir.ruler.serviceAccountName" . }}
+ {{- if .Values.ruler.priorityClassName }}
+ priorityClassName: {{ .Values.ruler.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "ruler") | nindent 8 }}
+ {{- with .Values.ruler.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end}}
+ {{- end }}
+ containers:
+ - name: ruler
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=ruler"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ - "-distributor.remote-timeout=10s"
+ {{- if .Values.ingester.zoneAwareReplication.migration.enabled }}
+ {{- if not .Values.ingester.zoneAwareReplication.migration.writePath }}
+ - "-ingester.ring.zone-awareness-enabled=false"
+ {{- end }}
+ {{- if .Values.ingester.zoneAwareReplication.migration.excludeDefaultZone }}
+ - "-ingester.ring.excluded-zones=zone-default"
+ {{- end }}
+ {{- end }}
+ {{- if .Values.store_gateway.zoneAwareReplication.migration.enabled }}
+ {{- if not .Values.store_gateway.zoneAwareReplication.migration.readPath }}
+ - "-store-gateway.sharding-ring.prefix=collectors/"
+ - "-store-gateway.sharding-ring.zone-awareness-enabled=false"
+ {{- end }}
+ {{- end }}
+ {{- range $key, $value := .Values.ruler.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.ruler.extraVolumeMounts }}
+ {{ toYaml .Values.ruler.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: storage
+ mountPath: "/data"
+ subPath: {{ .Values.ruler.persistence.subPath }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ - name: memberlist
+ containerPort: {{ include "mimir.memberlistBindPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.ruler.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.ruler.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.ruler.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.ruler.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.ruler.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.ruler }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.ruler.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+{{- if .Values.ruler.extraContainers }}
+{{ toYaml .Values.ruler.extraContainers | indent 8}}
+{{- end }}
+ {{- with .Values.ruler.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.ruler.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "ruler") | nindent 6 }}
+ {{- with .Values.ruler.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.ruler.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+{{- if .Values.ruler.extraVolumes }}
+{{ toYaml .Values.ruler.extraVolumes | indent 8}}
+{{- end }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-pdb.yaml
new file mode 100644
index 0000000..4693d45
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-pdb.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.ruler.enabled -}}
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "ruler" "memberlist" true) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-sa.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-sa.yaml
new file mode 100644
index 0000000..8993cf1
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-sa.yaml
@@ -0,0 +1,14 @@
+{{- if .Values.ruler.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ template "mimir.ruler.serviceAccountName" . }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ {{- with .Values.ruler.serviceAccount.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.ruler.serviceAccount.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-servmon.yaml
new file mode 100644
index 0000000..67d2142
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-servmon.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.ruler.enabled -}}
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "ruler" "memberlist" true) }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-so.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-so.yaml
new file mode 100644
index 0000000..670037e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-so.yaml
@@ -0,0 +1,44 @@
+{{- if .Values.ruler.kedaAutoscaling.enabled }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ruler") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "ruler") | nindent 4 }}
+ annotations:
+ {{- toYaml .Values.ruler.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ advanced:
+ horizontalPodAutoscalerConfig:
+ {{- with .Values.ruler.kedaAutoscaling.behavior }}
+ behavior:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ maxReplicaCount: {{ .Values.ruler.kedaAutoscaling.maxReplicaCount }}
+ minReplicaCount: {{ .Values.ruler.kedaAutoscaling.minReplicaCount }}
+ pollingInterval: {{ .Values.kedaAutoscaling.pollingInterval }}
+ scaleTargetRef:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ruler") }}
+ apiVersion: apps/v1
+ kind: Deployment
+ triggers:
+ - metadata:
+ query: max_over_time(sum(sum by (pod) (rate(container_cpu_usage_seconds_total{container="ruler",namespace="{{ .Release.Namespace }}"}[5m])) and max by (pod) (up{container="ruler",namespace="{{ .Release.Namespace }}"}) > 0)[15m:]) * 1000
+ serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
+ {{- $cpu_request := dig "requests" "cpu" nil .Values.ruler.resources }}
+ threshold: {{ mulf (include "mimir.cpuToMilliCPU" (dict "value" $cpu_request)) (divf .Values.ruler.kedaAutoscaling.targetCPUUtilizationPercentage 100) | floor | int64 | quote }}
+ {{- if .Values.kedaAutoscaling.customHeaders }}
+ customHeaders: {{ (include "mimir.lib.mapToCSVString" (dict "map" .Values.kedaAutoscaling.customHeaders)) | quote }}
+ {{- end }}
+ type: prometheus
+ - metadata:
+ query: max_over_time(sum((sum by (pod) (container_memory_working_set_bytes{container="ruler",namespace="{{ .Release.Namespace }}"}) and max by (pod) (up{container="ruler",namespace="{{ .Release.Namespace }}"}) > 0) or vector(0))[15m:]) + sum(sum by (pod) (max_over_time(kube_pod_container_resource_requests{container="ruler",namespace="{{ .Release.Namespace }}", resource="memory"}[15m])) and max by (pod) (changes(kube_pod_container_status_restarts_total{container="ruler",namespace="{{ .Release.Namespace }}"}[15m]) > 0) and max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler",namespace="{{ .Release.Namespace }}", reason="OOMKilled"}) or vector(0))
+ serverAddress: {{ include "mimir.kedaPrometheusAddress" (dict "ctx" $) }}
+ {{- $mem_request := dig "requests" "memory" nil .Values.ruler.resources }}
+ threshold: {{ mulf (include "mimir.siToBytes" (dict "value" $mem_request)) (divf .Values.ruler.kedaAutoscaling.targetMemoryUtilizationPercentage 100) | floor | int64 | quote }}
+ {{- if .Values.kedaAutoscaling.customHeaders }}
+ customHeaders: {{ (include "mimir.lib.mapToCSVString" (dict "map" .Values.kedaAutoscaling.customHeaders)) | quote }}
+ {{- end }}
+ type: prometheus
+{{- end }}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-svc.yaml
new file mode 100644
index 0000000..477f626
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/ruler/ruler-svc.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.ruler.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ruler") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 4 }}
+ {{- with .Values.ruler.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.ruler.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 4 }}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/runtime-configmap.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/runtime-configmap.yaml
new file mode 100644
index 0000000..bdca1c8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/runtime-configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "runtime") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+data:
+ runtime.yaml: |
+ {{ tpl (toYaml .Values.runtimeConfig) . | nindent 4 }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/securitycontextconstraints.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/securitycontextconstraints.yaml
new file mode 100644
index 0000000..dcb3758
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/securitycontextconstraints.yaml
@@ -0,0 +1,42 @@
+{{- if eq (include "mimir.rbac.useSecurityContextConstraints" .) "true" }}
+apiVersion: security.openshift.io/v1
+kind: SecurityContextConstraints
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" .) }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+allowHostDirVolumePlugin: true
+allowHostIPC: false
+allowHostNetwork: false
+allowHostPID: false
+allowHostPorts: false
+allowPrivilegeEscalation: true
+allowPrivilegedContainer: false
+allowedCapabilities: null
+defaultAddCapabilities: null
+fsGroup:
+ type: RunAsAny
+groups: []
+priority: 10
+readOnlyRootFilesystem: false
+requiredDropCapabilities:
+ - MKNOD
+runAsUser:
+ type: RunAsAny
+seLinuxContext:
+ type: MustRunAs
+supplementalGroups:
+ type: RunAsAny
+users:
+ - system:serviceaccount:openshift-infra:pv-recycler-controller
+volumes:
+ - configMap
+ - downwardAPI
+ - emptyDir
+ - hostPath
+ - nfs
+ - persistentVolumeClaim
+ - projected
+ - secret
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/serviceaccount.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/serviceaccount.yaml
new file mode 100644
index 0000000..732a68b
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/serviceaccount.yaml
@@ -0,0 +1,14 @@
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ template "mimir.serviceAccountName" . }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }}
+ {{- with .Values.serviceAccount.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.serviceAccount.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/smoke-test/smoke-test-job.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/smoke-test/smoke-test-job.yaml
new file mode 100644
index 0000000..323c0d2
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/smoke-test/smoke-test-job.yaml
@@ -0,0 +1,76 @@
+{{ if not .Values.enterprise.enabled -}}
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "smoke-test") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "smoke-test") | nindent 4 }}
+ annotations:
+ {{- if .Values.smoke_test.annotations }}
+ {{- toYaml .Values.smoke_test.annotations | nindent 4 }}
+ {{- end }}
+ "helm.sh/hook": test
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ backoffLimit: 5
+ completions: 1
+ parallelism: 1
+ selector:
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "smoke-test") | nindent 8 }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.smoke_test.priorityClassName }}
+ priorityClassName: {{ .Values.smoke_test.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "smoke-test") | nindent 8 }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ initContainers:
+ {{- toYaml .Values.smoke_test.initContainers | nindent 8 }}
+ containers:
+ - name: smoke-test
+ image: "{{ .Values.smoke_test.image.repository }}:{{ .Values.smoke_test.image.tag }}"
+ imagePullPolicy: {{ .Values.smoke_test.image.pullPolicy }}
+ args:
+ - "-tests.smoke-test"
+ - "-tests.write-endpoint={{ template "mimir.gatewayUrl" . }}"
+ - "-tests.read-endpoint={{ template "mimir.gatewayUrl" . }}/prometheus"
+ - "-tests.tenant-id={{ .Values.smoke_test.tenantId }}"
+ - "-tests.write-read-series-test.num-series=1000"
+ - "-tests.write-read-series-test.max-query-age=48h"
+ - "-server.metrics-port={{ include "mimir.serverHttpListenPort" . }}"
+ {{- range $key, $value := .Values.smoke_test.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.smoke_test.extraVolumeMounts }}
+ {{ toYaml .Values.smoke_test.extraVolumeMounts | nindent 12 }}
+ {{- end }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.smoke_test.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.smoke_test.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ restartPolicy: OnFailure
+ volumes:
+ {{- if .Values.smoke_test.extraVolumes }}
+ {{ toYaml .Values.smoke_test.extraVolumes | nindent 8 }}
+ {{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml
new file mode 100644
index 0000000..29ab26f
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.podDisruptionBudget" (dict "ctx" $ "component" "store-gateway" "memberlist" true) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-servmon.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-servmon.yaml
new file mode 100644
index 0000000..39da841
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-servmon.yaml
@@ -0,0 +1 @@
+{{- include "mimir.lib.serviceMonitor" (dict "ctx" $ "component" "store-gateway" "memberlist" true) }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
new file mode 100644
index 0000000..4da75e1
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -0,0 +1,210 @@
+{{- $args := dict "ctx" $ "component" "store-gateway" "memberlist" true -}}
+{{- $zonesMap := include "mimir.zoneAwareReplicationMap" $args | fromYaml -}}
+{{- range $zoneName, $rolloutZone := $zonesMap }}
+{{- with $ -}}
+{{- $_ := set $args "rolloutZoneName" $zoneName -}}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: {{ include "mimir.resourceName" $args }}
+ labels:
+ {{- include "mimir.labels" $args | nindent 4 }}
+ annotations:
+ {{- include "mimir.componentAnnotations" $args | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ podManagementPolicy: {{ .Values.store_gateway.podManagementPolicy }}
+ replicas: {{ $rolloutZone.replicas }}
+ {{- if and (semverCompare ">= 1.23-0" (include "mimir.kubeVersion" .)) (.Values.store_gateway.persistentVolume.enableRetentionPolicy) }}
+ persistentVolumeClaimRetentionPolicy:
+ whenDeleted: {{ .Values.store_gateway.persistentVolume.whenDeleted }}
+ whenScaled: {{ .Values.store_gateway.persistentVolume.whenScaled }}
+ {{- end }}
+ selector:
+ matchLabels:
+ {{- include "mimir.selectorLabels" $args | nindent 6 }}
+ updateStrategy:
+ {{- if $zoneName }}
+ type: OnDelete
+ {{- else }}
+ {{- toYaml .Values.store_gateway.strategy | nindent 4 }}
+ {{- end }}
+ serviceName: {{ template "mimir.fullname" . }}-store-gateway{{- if not .Values.enterprise.legacyLabels -}}-headless{{- end -}}
+ {{- if .Values.store_gateway.persistentVolume.enabled }}
+ volumeClaimTemplates:
+ - metadata:
+ name: storage
+ {{- if .Values.store_gateway.persistentVolume.annotations }}
+ annotations:
+ {{- toYaml .Values.store_gateway.persistentVolume.annotations | nindent 10 }}
+ {{- end }}
+ spec:
+ {{- $storageClass := default .Values.store_gateway.persistentVolume.storageClass $rolloutZone.storageClass }}
+ {{- if $storageClass }}
+ {{- if (eq "-" $storageClass) }}
+ storageClassName: ""
+ {{- else }}
+ storageClassName: {{ $storageClass }}
+ {{- end }}
+ {{- end }}
+ accessModes:
+ {{- toYaml .Values.store_gateway.persistentVolume.accessModes | nindent 10 }}
+ resources:
+ requests:
+ storage: "{{ .Values.store_gateway.persistentVolume.size }}"
+ {{- end }}
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" $args | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" $args | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ {{- with .Values.store_gateway.schedulerName }}
+ schedulerName: {{ . | quote }}
+ {{- end }}
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.store_gateway.priorityClassName }}
+ priorityClassName: {{ .Values.store_gateway.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" $args | nindent 8 }}
+ {{- with .Values.store_gateway.initContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ {{- with $rolloutZone.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with $rolloutZone.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- include "mimir.lib.topologySpreadConstraints" (dict "ctx" . "component" "store-gateway") | nindent 6 }}
+ {{- with .Values.store_gateway.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: {{ .Values.store_gateway.terminationGracePeriodSeconds }}
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ {{- end }}
+ - name: runtime-config
+ configMap:
+ name: {{ template "mimir.fullname" . }}-runtime
+ {{- if not .Values.store_gateway.persistentVolume.enabled }}
+ - name: storage
+ emptyDir: {}
+ {{- end }}
+ {{- if .Values.store_gateway.extraVolumes }}
+ {{ toYaml .Values.store_gateway.extraVolumes | nindent 8 }}
+ {{- end }}
+ - name: active-queries
+ emptyDir: {}
+ containers:
+ {{- if .Values.store_gateway.extraContainers }}
+ {{ toYaml .Values.store_gateway.extraContainers | nindent 8 }}
+ {{- end }}
+ - name: store-gateway
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=store-gateway"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- if $zoneName }}
+ - "-store-gateway.sharding-ring.instance-availability-zone={{ $zoneName }}"
+ {{- else }}
+ {{- if .Values.store_gateway.zoneAwareReplication.migration.enabled }}
+ - "-store-gateway.sharding-ring.prefix=collectors/"
+ - "-store-gateway.sharding-ring.zone-awareness-enabled=false"
+ {{- end }}
+ {{- end }}
+ {{- range $key, $value := .Values.store_gateway.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.store_gateway.extraVolumeMounts }}
+ {{ toYaml .Values.store_gateway.extraVolumeMounts | nindent 12}}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ {{- if .Values.enterprise.enabled }}
+ - name: license
+ mountPath: /license
+ {{- end }}
+ - name: runtime-config
+ mountPath: /var/{{ include "mimir.name" . }}
+ - name: storage
+ mountPath: "/data"
+ {{- if .Values.store_gateway.persistentVolume.subPath }}
+ subPath: {{ .Values.store_gateway.persistentVolume.subPath }}
+ {{- end }}
+ - name: active-queries
+ mountPath: /active-query-tracker
+ ports:
+ - name: http-metrics
+ containerPort: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ - name: grpc
+ containerPort: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ - name: memberlist
+ containerPort: {{ include "mimir.memberlistBindPort" . }}
+ protocol: TCP
+ livenessProbe:
+ {{- toYaml .Values.store_gateway.livenessProbe | nindent 12 }}
+ readinessProbe:
+ {{- toYaml .Values.store_gateway.readinessProbe | nindent 12 }}
+ resources:
+ {{- toYaml .Values.store_gateway.resources | nindent 12 }}
+ securityContext:
+ {{- toYaml .Values.store_gateway.containerSecurityContext | nindent 12 }}
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.store_gateway.env }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- $cpu_request := dig "requests" "cpu" nil .Values.store_gateway.resources }}
+ {{- if $cpu_request }}
+ {{- $cpu_request_doubled := include "mimir.parseCPU" (dict "value" $cpu_request) | float64 | mulf 2 | ceil }}
+ {{- $cpu_request_plus_four := include "mimir.parseCPU" (dict "value" $cpu_request) | float64 | addf 4 | ceil }}
+ - name: "GOMAXPROCS"
+ value: {{ max $cpu_request_doubled $cpu_request_plus_four | toString | toYaml }}
+ {{- end }}
+ {{- $mem_request := dig "requests" "memory" nil .Values.store_gateway.resources }}
+ {{- if $mem_request }}
+ - name: "GOMEMLIMIT"
+ value: {{include "mimir.siToBytes" (dict "value" $mem_request) | toString | toYaml }}
+ {{- end }}
+ {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.store_gateway }}
+ {{- if $jaeger_queue_size }}
+ - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+ value: {{$jaeger_queue_size | toString | toYaml }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.store_gateway.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+
+---
+{{ end }}
+{{ end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml
new file mode 100644
index 0000000..2fb878e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "store-gateway") }}-headless
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 4 }}
+ prometheus.io/service-monitor: "false"
+ {{- with .Values.store_gateway.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.store_gateway.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" .}}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 4 }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml
new file mode 100644
index 0000000..d637a07
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml
@@ -0,0 +1,38 @@
+{{- $args := dict "ctx" $ "component" "store-gateway" "memberlist" true -}}
+{{- $zonesMap := include "mimir.zoneAwareReplicationMap" $args | fromYaml -}}
+{{- range $zoneName, $rolloutZone := $zonesMap }}
+{{- with $ -}}
+{{- $_ := set $args "rolloutZoneName" $zoneName -}}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "mimir.resourceName" $args }}
+ labels:
+ {{- include "mimir.labels" $args | nindent 4 }}
+ {{- if and $zoneName .Values.store_gateway.zoneAwareReplication.migration.enabled }}
+ # Prevent scraping PODs via this service during migration as the original non zone-aware service already scrapes all PODs and you get duplicate metrics.
+ prometheus.io/service-monitor: "false"
+ {{- end }}
+ {{- with .Values.store_gateway.service.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- toYaml .Values.store_gateway.service.annotations | nindent 4 }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ type: ClusterIP
+ ports:
+ - port: {{ include "mimir.serverHttpListenPort" . }}
+ protocol: TCP
+ name: http-metrics
+ targetPort: http-metrics
+ - port: {{ include "mimir.serverGrpcListenPort" . }}
+ protocol: TCP
+ name: grpc
+ targetPort: grpc
+ selector:
+ {{- include "mimir.selectorLabels" $args | nindent 4 }}
+
+---
+{{ end }}
+{{ end }}
\ No newline at end of file
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/tokengen/tokengen-job.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/tokengen/tokengen-job.yaml
new file mode 100644
index 0000000..7af1ba5
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/tokengen/tokengen-job.yaml
@@ -0,0 +1,94 @@
+{{- if .Values.enterprise.enabled -}}
+{{ if .Values.tokengenJob.enable }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ include "mimir.resourceName" (dict "ctx" . "component" "tokengen") }}
+ labels:
+ {{- include "mimir.labels" (dict "ctx" . "component" "tokengen") | nindent 4 }}
+ annotations:
+ {{- if .Values.tokengenJob.annotations }}
+ {{- toYaml .Values.tokengenJob.annotations | nindent 4 }}
+ {{- end }}
+ "helm.sh/hook": post-install
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ backoffLimit: 6
+ completions: 1
+ parallelism: 1
+ selector:
+ template:
+ metadata:
+ labels:
+ {{- include "mimir.podLabels" (dict "ctx" . "component" "tokengen") | nindent 8 }}
+ annotations:
+ {{- include "mimir.podAnnotations" (dict "ctx" . "component" "tokengen") | nindent 8 }}
+ namespace: {{ .Release.Namespace | quote }}
+ spec:
+ serviceAccountName: {{ template "mimir.serviceAccountName" . }}
+ {{- if .Values.tokengenJob.priorityClassName }}
+ priorityClassName: {{ .Values.tokengenJob.priorityClassName }}
+ {{- end }}
+ securityContext:
+ {{- include "mimir.lib.podSecurityContext" (dict "ctx" . "component" "tokengen") | nindent 8 }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ {{- range .Values.image.pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+ initContainers:
+ {{- toYaml .Values.tokengenJob.initContainers | nindent 8 }}
+ containers:
+ - name: tokengen
+ image: "{{ include "mimir.imageReference" . }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - "-target=tokengen"
+ - "-config.expand-env=true"
+ - "-config.file=/etc/mimir/mimir.yaml"
+ {{- range $key, $value := .Values.tokengenJob.extraArgs }}
+ - "-{{ $key }}={{ $value }}"
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.tokengenJob.extraVolumeMounts }}
+ {{ toYaml .Values.tokengenJob.extraVolumeMounts | nindent 12 }}
+ {{- end }}
+ - name: config
+ mountPath: /etc/mimir
+ - name: license
+ mountPath: /license
+ - name: active-queries
+ mountPath: /active-query-tracker
+ env:
+ {{- with .Values.global.extraEnv }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.tokengenJob.env }}
+ {{ toYaml . | nindent 12 }}
+ {{- end }}
+ envFrom:
+ {{- with .Values.global.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.tokengenJob.extraEnvFrom }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ securityContext:
+ {{- toYaml .Values.tokengenJob.containerSecurityContext | nindent 12 }}
+ restartPolicy: OnFailure
+ volumes:
+ - name: config
+ {{- include "mimir.configVolume" . | nindent 10 }}
+ {{- if .Values.tokengenJob.extraVolumes }}
+ {{ toYaml .Values.tokengenJob.extraVolumes | nindent 8 }}
+ {{- end }}
+ - name: license
+ secret:
+ secretName: {{ tpl .Values.license.secretName . }}
+ - name: storage
+ emptyDir: {}
+ - name: active-queries
+ emptyDir: {}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/validate.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/validate.yaml
new file mode 100644
index 0000000..f2dd5e9
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/validate.yaml
@@ -0,0 +1,266 @@
+{{- $calculatedConfig := include "mimir.calculatedConfig" . | fromYaml }}
+{{- $unstructuredConfig := include "mimir.unstructuredConfig" . | fromYaml }}
+
+{{- if .Values.config }}
+{{- fail "Top level 'config' is not allowed. In 'mimir.config', provide the configuration as a string that can contain template expressions. Alternatively, you can provide the configuration as an external secret." }}
+{{- end }}
+
+{{- if .Values.podDisruptionBudget }}
+{{- fail "Top level 'podDisruptionBudget' is removed in favour of per component podDisruptionBudget" }}
+{{- end }}
+
+{{- if not ($calculatedConfig.activity_tracker).filepath }}
+{{- fail "The 'activity_tracker.filepath' in 'mimir.config' should be set to a path that is on a writeable volume. For example: '/active-query-tracker/activity.log'." }}
+{{- end }}
+
+{{- if ($unstructuredConfig.server).http_listen_port }}
+{{- fail "The setting 'server.http_listen_port' is not allowed in 'mimir.config' as it leads to a circular dependency. Set it in 'mimir.structuredConfig' instead." }}
+{{- end }}
+
+{{- if ($unstructuredConfig.server).grpc_listen_port }}
+{{- fail "The setting 'server.grpc_listen_port' is not allowed in 'mimir.config' as it leads to a circular dependency. Set it in 'mimir.structuredConfig' instead." }}
+{{- end }}
+
+{{- if ($unstructuredConfig.memberlist).bind_port }}
+{{- fail "The setting 'memberlist.bind_port' is not allowed in 'mimir.config' as it leads to a circular dependency. Set it in 'mimir.structuredConfig' instead." }}
+{{- end }}
+
+{{- if not (has .Values.configStorageType (list "Secret" "ConfigMap")) }}
+{{- fail "The setting 'configStorageType' should be one of 'Secret' or 'ConfigMap'." }}
+{{- end }}
+
+{{- with .Values.metaMonitoring.grafanaAgent }}
+{{/*
+ !servmon.Enabled
+ && agent.Enabled
+ && (remote.url != "" || (len(additionalConfigs) > 0 && additionalConfigs.url != ""))
+*/}}
+{{- if and
+ (not ((($.Values).metaMonitoring).serviceMonitor).enabled)
+ .enabled
+ (or
+ (not (empty ((.metrics).remote).url))
+ (and
+ (not (empty (.metrics).additionalRemoteWriteConfigs))
+ (not (empty (first (.metrics).additionalRemoteWriteConfigs).url))
+ )
+ )
+ }}
+{{- fail "metaMonitoring.grafanaAgent.remote.url is set, but metaMonitoring.serviceMonitor is disabled; you will not see any metrics, so enable the metaMonitoring.serviceMonitor or remove the remote configuration" }}
+{{- end }}
+{{- end }}
+
+{{- with ($calculatedConfig.blocks_storage).bucket_store }}
+ {{- with .chunks_cache }}
+ {{- if and
+ (index $.Values "chunks-cache" "enabled")
+ (eq .backend "memcached")
+ (ne .memcached.addresses (include "mimir.chunksCacheAddress" $))
+ }}
+ {{- fail (cat
+ "you set chunks-cache.enabled, but the address of the chunks cache in the Mimir configuration in 'blocks_storage.bucket_store.chunks_cache.memcached.addresses'"
+ " is different from the address of the memcached instance created by the chart. (actual: "
+ .memcached.addresses
+ ", expected: "
+ (include "mimir.chunksCacheAddress" $)
+ "). If you want to use your own memcached cluster, disable chunks-cache.enabled and keep the Mimir configuration in mimir.config or mimir.structuredConfig")
+ }}
+ {{- end }}
+ {{- end }}
+ {{- with .metadata_cache }}
+ {{- if and
+ (index $.Values "metadata-cache" "enabled")
+ (eq .backend "memcached")
+ (ne .memcached.addresses (include "mimir.metadataCacheAddress" $))
+ }}
+ {{- fail (cat
+ "you set metadata-cache.enabled, but the address of the metadata cache in the Mimir configuration in 'blocks_storage.bucket_store.metadata_cache.memcached.addresses'"
+ " is different from the address of the memcached instance created by the chart. (actual: "
+ .memcached.addresses
+ ", expected: "
+ (include "mimir.metadataCacheAddress" $)
+ "). If you want to use your own memcached cluster, disable metadata-cache.enabled and keep the Mimir configuration in mimir.config or mimir.structuredConfig")
+ }}
+ {{- end }}
+ {{- end }}
+ {{- with .index_cache }}
+ {{- if and
+ (index $.Values "index-cache" "enabled")
+ (eq .backend "memcached")
+ (ne .memcached.addresses (include "mimir.indexCacheAddress" $))
+ }}
+ {{- fail (cat
+ "you set index-cache.enabled, but the address of the index cache in the Mimir configuration in 'blocks_storage.bucket_store.index_cache.memcached.addresses'"
+ " is different from the address of the memcached instance created by the chart. (actual: "
+ .memcached.addresses
+ ", expected: "
+ (include "mimir.indexCacheAddress" $)
+ "). If you want to use your own memcached cluster, disable index-cache.enabled and keep the Mimir configuration in mimir.config or mimir.structuredConfig")
+ }}
+ {{- end }}
+ {{- end }}
+{{- end }}
+
+{{- with ($calculatedConfig.frontend) }}
+ {{- with .results_cache }}
+ {{- if and
+ (index $.Values "results-cache" "enabled")
+ (eq .backend "memcached")
+ (ne .memcached.addresses (include "mimir.resultsCacheAddress" $))
+ }}
+ {{- fail (cat
+ "you set results-cache.enabled, but the address of the results cache in the Mimir configuration in 'frontend.results_cache.memcached.addresses'"
+ " is different from the address of the memcached instance created by the chart. (actual: "
+ .memcached.addresses
+ ", expected: "
+ (include "mimir.resultsCacheAddress" $)
+ "). If you want to use your own memcached cluster, disable results-cache.enabled and keep the Mimir configuration in mimir.config or mimir.structuredConfig")
+ }}
+ {{- end }}
+ {{- end }}
+{{- end }}
+
+{{- with .Values.memcached }}
+ {{- if .enabled }}
+ {{- fail "memcached.enabled has been renamed to chunks-cache.enabled" }}
+ {{- end }}
+{{- end }}
+
+{{- with (index .Values "memcached-metadata") }}
+ {{- if .enabled }}
+ {{- fail "memcached-metadata.enabled has been renamed to metadata-cache.enabled" }}
+ {{- end }}
+{{- end }}
+
+{{- with (index .Values "memcached-queries") }}
+ {{- if .enabled }}
+ {{- fail "memcached-queries.enabled has been renamed to index-cache.enabled" }}
+ {{- end }}
+{{- end }}
+
+{{- with (index .Values "memcached-results") }}
+ {{- if .enabled }}
+ {{- fail "memcached-results.enabled has been renamed to results-cache.enabled" }}
+ {{- end }}
+{{- end }}
+
+{{- if not .Values.ingester.statefulSet.enabled -}}
+{{- fail "You have set ingester.statefulSet.enabled=false. Ingesters are stateful, kubernetes Deployment workloads are no longer supported. Please refer to the documentation for more details." }}
+{{- end -}}
+
+
+{{- if and .Values.ingester.zoneAwareReplication.enabled (not ((($calculatedConfig).ingester).ring).zone_awareness_enabled) -}}
+{{- fail "You have set ingester.zoneAwareReplication.enabled=true. The mimir.config must include ingester.ring.zone_awareness_enabled=true. Please merge the latest mimir.config from the chart with your mimir.config."}}
+{{- end -}}
+
+{{- if and .Values.ingester.zoneAwareReplication.migration.enabled (not .Values.ingester.zoneAwareReplication.enabled) -}}
+{{- fail "You have set ingester.zoneAwareReplication.migration.enabled=true, but ingester.zoneAwareReplication.enabled=false. Please consult the Migrating from single zone with Helm document."}}
+{{- end -}}
+
+{{- with $.Values.ingester.zoneAwareReplication.migration }}
+{{- if and .replicas (not .enabled) }}
+{{- fail "You have set ingester.zoneAwareReplication.migration.replicas, but ingester.zoneAwareReplication.migration.enabled=false. Please consult the Migrating from single zone with Helm document."}}
+{{- end -}}
+{{/* Check that if a migration step is enabled, every previous step is also enabled. */}}
+{{- $steps := list "enabled" "writePath" "readPath" "excludeDefaultZone" "scaleDownDefaultZone" }}
+{{- range $i, $item := rest $steps -}}
+{{- range $j, $mustHave := slice $steps 0 $i }}
+{{- with $.Values.ingester.zoneAwareReplication.migration }}
+{{- if and (index . $item) (not (index . $mustHave)) -}}
+{{- printf "You have set ingester.zoneAwareReplication.migration.%s=true, but ingester.zoneAwareReplication.migration.%s=false. Please consult the Migrating from single zone with Helm document." $item $mustHave | fail }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+
+{{- if and .Values.store_gateway.zoneAwareReplication.enabled (not ((($calculatedConfig).store_gateway).sharding_ring).zone_awareness_enabled) -}}
+{{- fail "You have set store_gateway.zoneAwareReplication.enabled=true. The mimir.config must include store_gateway.sharding_ring.zone_awareness_enabled=true. Please merge the latest mimir.config from the chart with your mimir.config."}}
+{{- end -}}
+
+{{- if and .Values.store_gateway.zoneAwareReplication.migration.enabled (ne (((($calculatedConfig).store_gateway).sharding_ring).kvstore).prefix "multi-zone/") -}}
+{{- fail "You have set store_gateway.zoneAwareReplication.migration.enabled=true. The mimir.config must include store_gateway.sharding_ring.kvstore.prefix=multi-zone/. Please merge the latest mimir.config from the chart with your mimir.config."}}
+{{- end -}}
+
+{{- if and .Values.store_gateway.zoneAwareReplication.migration.enabled (not .Values.store_gateway.zoneAwareReplication.enabled) -}}
+{{- fail "You have set store_gateway.zoneAwareReplication.migration.enabled=true, but store_gateway.zoneAwareReplication.enabled=false. Please consult the Migrating from single zone with Helm document."}}
+{{- end -}}
+
+{{- with $.Values.store_gateway.zoneAwareReplication.migration }}
+{{/* Check that if a migration step is enabled, every previous step is also enabled. */}}
+{{- $steps := list "enabled" "readPath" }}
+{{- range $i, $item := rest $steps -}}
+{{- range $j, $mustHave := slice $steps 0 $i }}
+{{- with $.Values.store_gateway.zoneAwareReplication.migration }}
+{{- if and (index . $item) (not (index . $mustHave)) -}}
+{{- printf "You have set store_gateway.zoneAwareReplication.migration.%s=true, but store_gateway.zoneAwareReplication.migration.%s=false. Please consult the Migrating from single zone with Helm document." $item $mustHave | fail }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- if and .Values.alertmanager.zoneAwareReplication.enabled (not .Values.alertmanager.statefulSet.enabled) -}}
+{{- fail "You have set alertmanager.zoneAwareReplication.enabled=true, but alertmanager.statefulSet.enabled=false. This is not a supported configuration." -}}
+{{- end -}}
+
+{{- if and .Values.alertmanager.zoneAwareReplication.enabled (not ((($calculatedConfig).alertmanager).sharding_ring).zone_awareness_enabled) -}}
+{{- fail "You have set alertmanager.zoneAwareReplication.enabled=true. The mimir.config must include alertmanager.sharding_ring.zone_awareness_enabled=true. Please merge the latest mimir.config from the chart with your mimir.config."}}
+{{- end -}}
+
+{{- if and .Values.alertmanager.zoneAwareReplication.migration.enabled (not .Values.alertmanager.zoneAwareReplication.enabled) -}}
+{{- fail "You have set alertmanager.zoneAwareReplication.migration.enabled=true, but alertmanager.zoneAwareReplication.enabled=false. Please consult the Migrating from single zone with Helm document."}}
+{{- end -}}
+
+{{- with $.Values.alertmanager.zoneAwareReplication.migration }}
+{{/* Check that if a migration step is enabled, every previous step is also enabled. */}}
+{{- $steps := list "enabled" "writePath" }}
+{{- range $i, $item := rest $steps -}}
+{{- range $j, $mustHave := slice $steps 0 $i }}
+{{- with $.Values.alertmanager.zoneAwareReplication.migration }}
+{{- if and (index . $item) (not (index . $mustHave)) -}}
+{{- printf "You have set alertmanager.zoneAwareReplication.migration.%s=true, but alertmanager.zoneAwareReplication.migration.%s=false. Please consult the Migrating from single zone with Helm document." $item $mustHave | fail }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- with .Values.nginx -}}
+{{- if and .autoscaling.enabled .autoscaling.targetCPUUtilizationPercentage (not ((.resources).requests).cpu) -}}
+{{- fail "You have enabled nginx.autoscaling.targetCPUUtilizationPercentage, you must also set nginx.resources.requests.cpu." -}}
+{{- end -}}
+{{- if and .autoscaling.enabled .autoscaling.targetMemoryUtilizationPercentage (not ((.resources).requests).memory) -}}
+{{- fail "You have enabled nginx.autoscaling.targetMemoryUtilizationPercentage, you must also set nginx.resources.requests.memory." -}}
+{{- end -}}
+{{- end -}}
+
+{{- with .Values.gateway -}}
+{{- if and .autoscaling.enabled .autoscaling.targetCPUUtilizationPercentage (not ((.resources).requests).cpu) -}}
+{{- fail "You have enabled gateway.autoscaling.targetCPUUtilizationPercentage, you must also set gateway.resources.requests.cpu." -}}
+{{- end -}}
+{{- if and .autoscaling.enabled .autoscaling.targetMemoryUtilizationPercentage (not ((.resources).requests).memory) -}}
+{{- fail "You have enabled gateway.autoscaling.targetMemoryUtilizationPercentage, you must also set gateway.resources.requests.memory." -}}
+{{- end -}}
+{{- end -}}
+
+{{- if eq .Values.rbac.type "scc" -}}
+{{- with .Values.rbac.podSecurityContext -}}
+{{- if or (hasKey . "fsGroup") (hasKey . "runAsGroup") (hasKey . "runAsUser") -}}
+{{- fail "You have selected rbac.type=scc, you must set the values fsGroup, runAsGroup, runAsUser to null in rbac.podSecurityContext" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- if eq .Values.rbac.type "scc" -}}
+{{- with .Values.rollout_operator.podSecurityContext -}}
+{{- if or (hasKey . "fsGroup") (hasKey . "runAsGroup") (hasKey . "runAsUser") -}}
+{{- fail "You have selected rbac.type=scc, you must set the values fsGroup, runAsGroup, runAsUser to null in rollout_operator.podSecurityContext" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- if and .Values.ingress.enabled .Values.gateway.ingress.enabled -}}
+{{- fail "You have selected both ingress.enabled and gateway.ingress.enabled, you must select either ingress or gateway but not both" -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/templates/values-for-rego-tests.yaml b/observability/lgtm-distributed/charts/mimir-distributed/templates/values-for-rego-tests.yaml
new file mode 100644
index 0000000..abf8678
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/templates/values-for-rego-tests.yaml
@@ -0,0 +1,6 @@
+{{- if .Values.regoTestGenerateValues }}
+apiVersion: v1
+kind: Values
+spec:
+ {{- .Values | toYaml | nindent 2 }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/mimir-distributed/values.yaml b/observability/lgtm-distributed/charts/mimir-distributed/values.yaml
new file mode 100644
index 0000000..e037731
--- /dev/null
+++ b/observability/lgtm-distributed/charts/mimir-distributed/values.yaml
@@ -0,0 +1,3978 @@
+# Grafana Enterprise Metrics specific values and features are found after the `enterprise:` key
+#
+# The default values specified in this file are enough to deploy all of the
+# Grafana Mimir or Grafana Enterprise Metrics microservices but are not suitable for
+# production load.
+# To configure the resources for production load, refer to the small.yaml or
+# large.yaml values files.
+
+# Note: The values in this file are not backward compatible. Copying and pasting the values is discouraged, but if you do so,
+# make sure to use the values.yaml from the branch or tag that matches the mimir-distributed Helm chart version that you
+# want to install. You also can see values.yaml for a specific version by running
+# `helm show values grafana/mimir-distributed --version `
+
+# If you want to get the values file from Github, build the URL as follows because we git tag every release:
+# `https://github.com/grafana/mimir/blob/mimir-distributed-/operations/helm/charts/mimir-distributed/values.yaml`.
+# For example, https://github.com/grafana/mimir/blob/mimir-distributed-3.1.0/operations/helm/charts/mimir-distributed/values.yaml.
+
+# -- Overrides the version used to determine compatibility of resources with the target Kubernetes cluster.
+# This is useful when using `helm template`, because then helm will use the client version of kubectl as the Kubernetes version,
+# which may or may not match your cluster's server version. Example: 'v1.24.4'. Set to null to use the version that helm
+# devises.
+kubeVersionOverride: null
+
+# -- Overrides the chart's name. Used to change mimir/enterprise-metrics infix in the resource names. E.g. myRelease-mimir-ingester-1 to myRelease-nameOverride-ingester-1.
+# This option is used to align resource names with Cortex, when doing a migration from Cortex to Grafana Mimir.
+# Note: Grafana provided dashboards rely on the default naming and will need changes.
+nameOverride: null
+
+# -- Overrides the chart's computed fullname. Used to change the full prefix of resource names. E.g. myRelease-mimir-ingester-1 to fullnameOverride-ingester-1.
+# Note: Grafana provided dashboards rely on the default naming and will need changes.
+fullnameOverride: null
+
+image:
+ # -- Grafana Mimir container image repository. Note: for Grafana Enterprise Metrics use the value 'enterprise.image.repository'
+ repository: grafana/mimir
+ # -- Grafana Mimir container image tag. Note: for Grafana Enterprise Metrics use the value 'enterprise.image.tag'
+ tag: 2.12.0
+ # -- Container pull policy - shared between Grafana Mimir and Grafana Enterprise Metrics
+ pullPolicy: IfNotPresent
+ # -- Optionally specify an array of imagePullSecrets - shared between Grafana Mimir and Grafana Enterprise Metrics
+ # Secrets must be manually created in the namespace.
+ # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+ # pullSecrets:
+ # - myRegistryKeySecretName
+
+global:
+ # -- Definitions to set up nginx resolver
+ dnsService: kube-dns
+ dnsNamespace: kube-system
+ clusterDomain: cluster.local.
+
+ # -- Common environment variables to add to all pods directly managed by this chart.
+ # scope: admin-api, alertmanager, compactor, distributor, gateway, ingester, memcached, nginx, overrides-exporter, querier, query-frontend, query-scheduler, ruler, store-gateway, smoke-test, tokengen
+ extraEnv: []
+
+ # -- Common source of environment injections to add to all pods directly managed by this chart.
+ # scope: admin-api, alertmanager, compactor, distributor, gateway, ingester, memcached, nginx, overrides-exporter, querier, query-frontend, query-scheduler, ruler, store-gateway, smoke-test, tokengen
+ # For example to inject values from a Secret, use:
+ # extraEnvFrom:
+ # - secretRef:
+ # name: mysecret
+ extraEnvFrom: []
+
+ # -- Pod annotations for all pods directly managed by this chart. Usable for example to associate a version to 'global.extraEnv' and 'global.extraEnvFrom' and trigger a restart of the affected services.
+ # scope: admin-api, alertmanager, compactor, distributor, gateway, ingester, memcached, nginx, overrides-exporter, querier, query-frontend, query-scheduler, ruler, store-gateway, tokengen
+ podAnnotations: {}
+
+ # -- Pod labels for all pods directly managed by this chart.
+ # scope: admin-api, alertmanager, compactor, distributor, gateway, ingester, memcached, nginx, overrides-exporter, querier, query-frontend, query-scheduler, ruler, store-gateway, tokengen
+ podLabels: {}
+
+serviceAccount:
+ # -- Whether to create a service account or not. In case 'create' is false, do set 'name' to an existing service account name.
+ create: true
+ # -- Override for the generated service account name.
+ name:
+ annotations: {}
+ labels: {}
+
+# -- Configuration is loaded from the secret called 'externalConfigSecretName'. If 'useExternalConfig' is true, then the configuration is not generated, just consumed.
+useExternalConfig: false
+
+# -- Defines what kind of object stores the configuration, a ConfigMap or a Secret.
+# In order to move sensitive information (such as credentials) from the ConfigMap/Secret to a more secure location (e.g. vault), it is possible to use [environment variables in the configuration](https://grafana.com/docs/mimir/latest/reference-configuration-parameters/#use-environment-variables-in-the-configuration).
+# Such environment variables can be then stored in a separate Secret and injected via the global.extraEnvFrom value. For details about environment injection from a Secret please see [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/#use-case-as-container-environment-variables).
+configStorageType: ConfigMap
+
+# -- Name of the Secret or ConfigMap that contains the configuration (used for naming even if config is internal).
+externalConfigSecretName: '{{ include "mimir.resourceName" (dict "ctx" . "component" "config") }}'
+
+# -- When 'useExternalConfig' is true, then changing 'externalConfigVersion' triggers restart of services - otherwise changes to the configuration cause a restart.
+externalConfigVersion: "0"
+
+# --Vault Agent config to mount secrets to TLS configurable components. This requires Vault and Vault Agent to already be running.
+vaultAgent:
+ enabled: false
+ # -- Vault Kubernetes Authentication role
+ roleName: ""
+ # -- Path to client certificate in Vault
+ clientCertPath: ""
+ # -- Path to client key in Vault
+ clientKeyPath: ""
+ # -- Path to server certificate in Vault
+ serverCertPath: ""
+ # -- Path to server key in Vault
+ serverKeyPath: ""
+ # -- Path to client CA certificate in Vault
+ caCertPath: ""
+
+mimir:
+ # -- Base config file for Grafana Mimir and Grafana Enterprise Metrics. Contains Helm templates that are evaulated at install/upgrade.
+ # To modify the resulting configuration, either copy and alter 'mimir.config' as a whole or use the 'mimir.structuredConfig' to add and modify certain YAML elements.
+ config: |
+ usage_stats:
+ installation_mode: helm
+
+ activity_tracker:
+ filepath: /active-query-tracker/activity.log
+
+ {{- if .Values.enterprise.enabled }}
+ admin_api:
+ leader_election:
+ enabled: true
+ ring:
+ kvstore:
+ store: "memberlist"
+
+ admin_client:
+ storage:
+ {{- if .Values.minio.enabled }}
+ type: s3
+ s3:
+ access_key_id: {{ .Values.minio.rootUser }}
+ bucket_name: enterprise-metrics-admin
+ endpoint: {{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000
+ insecure: true
+ secret_access_key: {{ .Values.minio.rootPassword }}
+ {{- end }}
+ {{- if (index .Values "admin-cache" "enabled") }}
+ cache:
+ backend: memcached
+ memcached:
+ addresses: {{ include "mimir.adminCacheAddress" . }}
+ max_item_size: {{ mul (index .Values "admin-cache").maxItemMemory 1024 1024 }}
+ {{- end }}
+ {{- end }}
+
+ alertmanager:
+ data_dir: /data
+ enable_api: true
+ external_url: /alertmanager
+ {{- if .Values.alertmanager.zoneAwareReplication.enabled }}
+ sharding_ring:
+ zone_awareness_enabled: true
+ {{- end }}
+ {{- if .Values.alertmanager.fallbackConfig }}
+ fallback_config_file: /configs/alertmanager_fallback_config.yaml
+ {{- end }}
+
+ {{- if .Values.minio.enabled }}
+ alertmanager_storage:
+ backend: s3
+ s3:
+ access_key_id: {{ .Values.minio.rootUser }}
+ bucket_name: {{ include "mimir.minioBucketPrefix" . }}-ruler
+ endpoint: {{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000
+ insecure: true
+ secret_access_key: {{ .Values.minio.rootPassword }}
+ {{- end }}
+
+ {{- if .Values.enterprise.enabled }}
+ auth:
+ type: enterprise
+ {{- end }}
+
+ # This configures how the store-gateway synchronizes blocks stored in the bucket. It uses Minio by default for getting started (configured via flags) but this should be changed for production deployments.
+ blocks_storage:
+ backend: s3
+ bucket_store:
+ {{- if index .Values "chunks-cache" "enabled" }}
+ chunks_cache:
+ backend: memcached
+ memcached:
+ addresses: {{ include "mimir.chunksCacheAddress" . }}
+ max_item_size: {{ mul (index .Values "chunks-cache").maxItemMemory 1024 1024 }}
+ timeout: 750ms
+ max_idle_connections: 150
+ {{- end }}
+ {{- if index .Values "index-cache" "enabled" }}
+ index_cache:
+ backend: memcached
+ memcached:
+ addresses: {{ include "mimir.indexCacheAddress" . }}
+ max_item_size: {{ mul (index .Values "index-cache").maxItemMemory 1024 1024 }}
+ timeout: 750ms
+ max_idle_connections: 150
+ {{- end }}
+ {{- if index .Values "metadata-cache" "enabled" }}
+ metadata_cache:
+ backend: memcached
+ memcached:
+ addresses: {{ include "mimir.metadataCacheAddress" . }}
+ max_item_size: {{ mul (index .Values "metadata-cache").maxItemMemory 1024 1024 }}
+ max_idle_connections: 150
+ {{- end }}
+ sync_dir: /data/tsdb-sync
+ {{- if .Values.minio.enabled }}
+ s3:
+ access_key_id: {{ .Values.minio.rootUser }}
+ bucket_name: {{ include "mimir.minioBucketPrefix" . }}-tsdb
+ endpoint: {{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000
+ insecure: true
+ secret_access_key: {{ .Values.minio.rootPassword }}
+ {{- end }}
+ tsdb:
+ dir: /data/tsdb
+ head_compaction_interval: 15m
+ wal_replay_concurrency: 3
+
+ {{- if .Values.enterprise.enabled }}
+ cluster_name: "{{ .Release.Name }}"
+ {{- end }}
+
+ compactor:
+ compaction_interval: 30m
+ deletion_delay: 2h
+ max_closing_blocks_concurrency: 2
+ max_opening_blocks_concurrency: 4
+ symbols_flushers_concurrency: 4
+ first_level_compaction_wait_period: 25m
+ data_dir: "/data"
+ sharding_ring:
+ wait_stability_min_duration: 1m
+ heartbeat_period: 1m
+ heartbeat_timeout: 4m
+
+ distributor:
+ ring:
+ heartbeat_period: 1m
+ heartbeat_timeout: 4m
+
+ frontend:
+ parallelize_shardable_queries: true
+ {{- if index .Values "results-cache" "enabled" }}
+ results_cache:
+ backend: memcached
+ memcached:
+ timeout: 500ms
+ addresses: {{ include "mimir.resultsCacheAddress" . }}
+ max_item_size: {{ mul (index .Values "results-cache").maxItemMemory 1024 1024 }}
+ cache_results: true
+ query_sharding_target_series_per_shard: 2500
+ {{- end }}
+ {{- if .Values.query_scheduler.enabled }}
+ scheduler_address: {{ template "mimir.fullname" . }}-query-scheduler-headless.{{ .Release.Namespace }}.svc:{{ include "mimir.serverGrpcListenPort" . }}
+ {{- end }}
+
+ frontend_worker:
+ grpc_client_config:
+ max_send_msg_size: 419430400 # 400MiB
+ {{- if .Values.query_scheduler.enabled }}
+ scheduler_address: {{ template "mimir.fullname" . }}-query-scheduler-headless.{{ .Release.Namespace }}.svc:{{ include "mimir.serverGrpcListenPort" . }}
+ {{- else }}
+ frontend_address: {{ template "mimir.fullname" . }}-query-frontend-headless.{{ .Release.Namespace }}.svc:{{ include "mimir.serverGrpcListenPort" . }}
+ {{- end }}
+
+ {{- if and .Values.enterprise.enabled }}
+ gateway:
+ proxy:
+ admin_api:
+ url: http://{{ template "mimir.fullname" . }}-admin-api.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}
+ alertmanager:
+ url: http://{{ template "mimir.fullname" . }}-alertmanager-headless.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}
+ compactor:
+ url: http://{{ template "mimir.fullname" . }}-compactor.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}
+ default:
+ url: http://{{ template "mimir.fullname" . }}-admin-api.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}
+ distributor:
+ url: dns:///{{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverGrpcListenPort" . }}
+ ingester:
+ url: http://{{ template "mimir.fullname" . }}-ingester-headless.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}
+ query_frontend:
+ url: http://{{ template "mimir.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}
+ ruler:
+ url: http://{{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}
+ store_gateway:
+ url: http://{{ template "mimir.fullname" . }}-store-gateway-headless.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}
+ {{- if and .Values.enterprise.enabled .Values.graphite.enabled }}
+ graphite_write_proxy:
+ url: http://{{ template "mimir.fullname" . }}-graphite-write-proxy.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}
+ graphite_querier:
+ url: http://{{ template "mimir.fullname" . }}-graphite-querier.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}
+ {{- end}}
+ {{- end }}
+
+ ingester:
+ ring:
+ final_sleep: 0s
+ num_tokens: 512
+ tokens_file_path: /data/tokens
+ unregister_on_shutdown: false
+ heartbeat_period: 2m
+ heartbeat_timeout: 10m
+ {{- if .Values.ingester.zoneAwareReplication.enabled }}
+ zone_awareness_enabled: true
+ {{- end }}
+
+ ingester_client:
+ grpc_client_config:
+ max_recv_msg_size: 104857600
+ max_send_msg_size: 104857600
+
+ {{- if .Values.enterprise.enabled }}
+ instrumentation:
+ enabled: true
+ distributor_client:
+ address: dns:///{{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverGrpcListenPort" . }}
+
+ license:
+ path: "/license/license.jwt"
+ {{- end }}
+
+ limits:
+ # Limit queries to 500 days. You can override this on a per-tenant basis.
+ max_total_query_length: 12000h
+ # Adjust max query parallelism to 16x sharding, without sharding we can run 15d queries fully in parallel.
+ # With sharding we can further shard each day another 16 times. 15 days * 16 shards = 240 subqueries.
+ max_query_parallelism: 240
+ # Avoid caching results newer than 10m because some samples can be delayed
+ # This presents caching incomplete results
+ max_cache_freshness: 10m
+
+ memberlist:
+ abort_if_cluster_join_fails: false
+ compression_enabled: false
+ join_members:
+ - dns+{{ include "mimir.fullname" . }}-gossip-ring.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.memberlistBindPort" . }}
+
+ querier:
+ # With query sharding we run more but smaller queries. We must strike a balance
+ # which allows us to process more sharded queries in parallel when requested, but not overload
+ # queriers during non-sharded queries.
+ max_concurrent: 16
+
+ query_scheduler:
+ # Increase from default of 100 to account for queries created by query sharding
+ max_outstanding_requests_per_tenant: 800
+
+ ruler:
+ alertmanager_url: dnssrvnoa+http://_http-metrics._tcp.{{ template "mimir.fullname" . }}-alertmanager-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}/alertmanager
+ enable_api: true
+ rule_path: /data
+
+ {{- if or (.Values.minio.enabled) (index .Values "metadata-cache" "enabled") }}
+ ruler_storage:
+ {{- if .Values.minio.enabled }}
+ backend: s3
+ s3:
+ endpoint: {{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000
+ bucket_name: {{ include "mimir.minioBucketPrefix" . }}-ruler
+ access_key_id: {{ .Values.minio.rootUser }}
+ secret_access_key: {{ .Values.minio.rootPassword }}
+ insecure: true
+ {{- end }}
+ {{- if index .Values "metadata-cache" "enabled" }}
+ cache:
+ backend: memcached
+ memcached:
+ addresses: {{ include "mimir.metadataCacheAddress" . }}
+ max_item_size: {{ mul (index .Values "metadata-cache").maxItemMemory 1024 1024 }}
+ {{- end }}
+ {{- end }}
+
+ runtime_config:
+ file: /var/{{ include "mimir.name" . }}/runtime.yaml
+
+ store_gateway:
+ sharding_ring:
+ heartbeat_period: 1m
+ heartbeat_timeout: 4m
+ wait_stability_min_duration: 1m
+ {{- if .Values.store_gateway.zoneAwareReplication.enabled }}
+ kvstore:
+ prefix: multi-zone/
+ {{- end }}
+ tokens_file_path: /data/tokens
+ unregister_on_shutdown: false
+ {{- if .Values.store_gateway.zoneAwareReplication.enabled }}
+ zone_awareness_enabled: true
+ {{- end }}
+
+ {{- if and .Values.enterprise.enabled .Values.graphite.enabled }}
+ graphite:
+ enabled: true
+
+ write_proxy:
+ distributor_client:
+ address: dns:///{{ template "mimir.fullname" . }}-distributor.{{ .Release.Namespace }}.svc:{{ include "mimir.serverGrpcListenPort" . }}
+
+ querier:
+ remote_read:
+ query_address: http://{{ template "mimir.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }}/prometheus
+ proxy_bad_requests: false
+
+ schemas:
+ default_storage_schemas_file: /etc/graphite-proxy/storage-schemas.conf
+ default_storage_aggregations_file: /etc/graphite-proxy/storage-aggregations.conf
+ aggregation_cache:
+ memcached:
+ addresses: dnssrvnoa+{{ template "mimir.fullname" . }}-gr-aggr-cache.{{ .Release.Namespace}}.svc:11211
+ timeout: 1s
+ metric_name_cache:
+ memcached:
+ addresses: dnssrvnoa+{{ template "mimir.fullname" . }}-gr-metricname-cache.{{ .Release.Namespace}}.svc:11211
+ timeout: 1s
+ {{- end}}
+
+ # -- Additional structured values on top of the text based 'mimir.config'. Applied after the text based config is evaluated for templates. Enables adding and modifying YAML elements in the evaulated 'mimir.config'.
+ #
+ # Additionally, consider the optional "insecure_skip_verify" key below, it allows you to skip_verify_false in case the s3_endpoint certificate is not trusted.
+ # For more information see https://grafana.com/docs/mimir/latest/references/configuration-parameters/
+ #
+ # Example:
+ #
+ # structuredConfig:
+ # common:
+ # storage:
+ # backend: s3
+ # s3:
+ # bucket_name: "${BUCKET_NAME}"
+ # endpoint: "${BUCKET_HOST}:${BUCKET_PORT}"
+ # access_key_id: "${AWS_ACCESS_KEY_ID}" # This is a secret injected via an environment variable
+ # secret_access_key: "${AWS_SECRET_ACCESS_KEY}" # This is a secret injected via an environment variable
+ # http:
+ # insecure_skip_verify: true
+ structuredConfig: {}
+
+# -- runtimeConfig provides a reloadable runtime configuration. Changing the runtimeConfig doesn't require a restart of all components.
+# For more infromation see https://grafana.com/docs/mimir/latest/configure/about-runtime-configuration/
+#
+# Example:
+#
+# runtimeConfig:
+# ingester_limits: # limits that each ingester replica enforces
+# max_ingestion_rate: 20000
+# max_series: 1500000
+# max_tenants: 1000
+# max_inflight_push_requests: 30000
+# distributor_limits: # limits that each distributor replica enforces
+# max_ingestion_rate: 20000
+# max_inflight_push_requests: 30000
+# max_inflight_push_requests_bytes: 50000000
+# overrides:
+# tenant-1: # limits for tenant-1 that the whole cluster enforces
+# ingestion_tenant_shard_size: 9
+# max_global_series_per_user: 1500000
+# max_fetched_series_per_query: 100000
+runtimeConfig: {}
+
+# RBAC configuration
+rbac:
+ # -- If true, PodSecurityPolicy will be rendered by the chart on Kuberentes 1.24.
+ # By default the PodSecurityPolicy is not rendered on version 1.24.
+ create: true
+ # -- PSP configuration
+ podSecurityPolicy:
+ seccompProfile: runtime/default
+ privileged: false
+ allowPrivilegeEscalation: false
+ hostNetwork: false
+ hostIPC: false
+ hostPID: false
+ readOnlyRootFilesystem: true
+ runAsUser:
+ rule: "MustRunAsNonRoot"
+ seLinux:
+ rule: "RunAsAny"
+ supplementalGroups:
+ rule: "MustRunAs"
+ ranges:
+ - min: 1
+ max: 65535
+ fsGroup:
+ rule: "MustRunAs"
+ ranges:
+ - min: 1
+ max: 65535
+ additionalVolumes: []
+ forcePSPOnKubernetes124: false
+ # -- For GKE/EKS/AKS use 'type: psp'. For OpenShift use 'type: scc'
+ type: psp
+ # -- podSecurityContext is the default pod security context for Mimir, GEM, gateway, and cache components.
+ # When installing on OpenShift, override podSecurityContext settings with
+ #
+ # rbac:
+ # podSecurityContext:
+ # fsGroup: null
+ # runAsGroup: null
+ # runAsUser: null
+ podSecurityContext:
+ fsGroup: 10001
+ runAsGroup: 10001
+ runAsNonRoot: true
+ runAsUser: 10001
+ seccompProfile:
+ type: RuntimeDefault
+
+# -- KEDA Autoscaling configuration
+kedaAutoscaling:
+ # -- A Prometheus-compatible URL. Cadvisor and Mimir metrics for the Mimir pods are expected in this server.
+ # For more information on the required metrics see [Monitor system health](https://grafana.com/docs/helm-charts/mimir-distributed/latest/run-production-environment-with-helm/monitor-system-health/).
+ # If empty, the helm chart uses the metamonitoring URL from metaMonitoring.grafanaAgent.metrics.remote.url.
+ # If that is empty, then the Mimir cluster is used.
+ prometheusAddress: ""
+ customHeaders: {}
+ pollingInterval: 10
+
+alertmanager:
+ enabled: true
+ # -- Total number of replicas for the alertmanager across all availability zones
+ # If alertmanager.zoneAwareReplication.enabled=false, this number is taken as is.
+ # Otherwise each zone starts `ceil(replicas / number_of_zones)` number of pods.
+ # E.g. if 'replicas' is set to 4 and there are 3 zones, then 4/3=1.33 and after rounding up it means 2 pods per zone are started.
+ replicas: 1
+
+ statefulSet:
+ enabled: true
+
+ service:
+ annotations: {}
+ labels: {}
+
+ # -- Optionally set the scheduler for pods of the alertmanager
+ schedulerName: ""
+
+ resources:
+ requests:
+ cpu: 10m
+ memory: 32Mi
+
+ # -- Fallback config for alertmanager.
+ # When a tenant doesn't have an Alertmanager configuration, the Grafana Mimir Alertmanager uses the fallback configuration.
+ fallbackConfig: |
+ receivers:
+ - name: default-receiver
+ route:
+ receiver: default-receiver
+
+ extraArgs: {}
+
+ # Pod Labels
+ podLabels: {}
+
+ # Pod Annotations
+ podAnnotations: {}
+
+ # -- Pod Disruption Budget for alertmanager, this will be applied across availability zones to prevent losing redundancy
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ # -- The name of the PriorityClass for alertmanager pods
+ priorityClassName: null
+
+ # -- NodeSelector to pin alertmanager pods to certain set of nodes. This is ignored when alertmanager.zoneAwareReplication.enabled=true.
+ nodeSelector: {}
+ # -- Pod affinity settings for the alertmanager. This is ignored when alertmanager.zoneAwareReplication.enabled=true.
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ annotations: {}
+ persistence:
+ # SubPath in emptyDir for persistence, only enabled if alertmanager.statefulSet.enabled is false
+ subPath:
+
+ persistentVolume:
+ # If true and alertmanager.statefulSet.enabled is true,
+ # Alertmanager will create/use a Persistent Volume Claim
+ # If false, use emptyDir
+ enabled: true
+
+ # Alertmanager data Persistent Volume Claim annotations
+ #
+ annotations: {}
+
+ # Alertmanager data Persistent Volume access modes
+ # Must match those of existing PV or dynamic provisioner
+ # Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
+ #
+ accessModes:
+ - ReadWriteOnce
+
+ # Alertmanager data Persistent Volume size
+ #
+ size: 1Gi
+
+ # Subdirectory of Alertmanager data Persistent Volume to mount
+ # Useful if the volume's root directory is not empty
+ #
+ subPath: ""
+
+ # Alertmanager data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", storageClassName: "", which disables dynamic provisioning
+ # If undefined (the default) or set to null, no storageClassName spec is
+ # set, choosing the default provisioner.
+ #
+ # A per-zone storageClass configuration in `alertmanager.zoneAwareReplication.zones[*].storageClass` takes precedence over this field.
+ # storageClass: "-"
+
+ # -- Enable StatefulSetAutoDeletePVC feature
+ # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention
+ enableRetentionPolicy: false
+ whenDeleted: Retain
+ whenScaled: Retain
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+ # -- SecurityContext override for alermeneger pods
+ securityContext: {}
+
+ # -- The SecurityContext for alertmanager containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ # Tolerations for pod assignment
+ # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+ tolerations: []
+
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 0
+ maxUnavailable: 1
+ # -- updateStrategy of the alertmanager statefulset. This is ignored when alertmanager.zoneAwareReplication.enabled=true.
+ statefulStrategy:
+ type: RollingUpdate
+
+ terminationGracePeriodSeconds: 60
+
+ initContainers: []
+ # Init containers to be added to the alertmanager pod.
+ # - name: my-init-container
+ # image: busybox:latest
+ # command: ['sh', '-c', 'echo hello']
+
+ extraContainers: []
+ # Additional containers to be added to the alertmanager pod.
+ # - name: reverse-proxy
+ # image: angelbarrera92/basic-auth-reverse-proxy:dev
+ # args:
+ # - "serve"
+ # - "--upstream=http://localhost:3100"
+ # - "--auth-config=/etc/reverse-proxy-conf/authn.yaml"
+ # ports:
+ # - name: http
+ # containerPort: 11811
+ # protocol: TCP
+ # volumeMounts:
+ # - name: reverse-proxy-auth-config
+ # mountPath: /etc/reverse-proxy-conf
+
+ extraVolumes: []
+ # Additional volumes to the alertmanager pod.
+ # - name: reverse-proxy-auth-config
+ # secret:
+ # secretName: reverse-proxy-auth-config
+
+ # Extra volume mounts that will be added to the alertmanager container
+ extraVolumeMounts: []
+
+ # Extra env variables to pass to the alertmanager container
+ env: []
+ extraEnvFrom: []
+
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: null
+
+ # -- Options to configure zone-aware replication for alertmanager
+ # Example configuration with full geographical redundancy:
+ # rollout_operator:
+ # enabled: true
+ # alertmanager:
+ # zoneAwareReplication:
+ # enabled: true
+ # topologyKey: 'kubernetes.io/hostname' # This generates default anti-affinity rules
+ # zones: # Zone list has to be fully redefined for modification. Update with you actual zones or skip to use logical zones only.
+ # - name: zone-a
+ # nodeSelector:
+ # topology.kubernetes.io/zone: us-central1-a
+ # - name: zone-a
+ # nodeSelector:
+ # topology.kubernetes.io/zone: us-central1-b
+ # - name: zone-c
+ # nodeSelector:
+ # topology.kubernetes.io/zone: us-central1-c
+ #
+ zoneAwareReplication:
+ # -- Enable zone-aware replication for alertmanager
+ enabled: false
+ # -- Maximum number of alertmanagers that can be unavailable per zone during rollout
+ maxUnavailable: 2
+ # -- topologyKey to use in pod anti-affinity. If unset, no anti-affinity rules are generated. If set, the generated anti-affinity rule makes sure that pods from different zones do not mix.
+ # E.g.: topologyKey: 'kubernetes.io/hostname'
+ topologyKey: null
+ # -- Auxiliary values for migration, see https://grafana.com/docs/helm-charts/mimir-distributed/latest/migration-guides/migrate-from-single-zone-with-helm/
+ migration:
+ # -- Indicate if migration is ongoing for multi zone alertmanager
+ enabled: false
+ # -- Start zone-aware alertmanagers
+ writePath: false
+ # -- Zone definitions for alertmanager zones. Note: you have to redefine the whole list to change parts as YAML does not allow to modify parts of a list.
+ zones:
+ # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+ - name: zone-a
+ # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
+ # nodeSelector:
+ # topology.kubernetes.io/zone: zone-a
+ nodeSelector: null
+ # -- extraAffinity adds user defined custom affinity rules (merged with generated rules)
+ extraAffinity: {}
+ # -- Alertmanager data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", then use `storageClassName: ""`, which disables dynamic provisioning
+ # If undefined or set to null (the default), then fall back to the value of `alertmanager.persistentVolume.storageClass`.
+ storageClass: null
+ # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+ - name: zone-b
+ # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
+ # nodeSelector:
+ # topology.kubernetes.io/zone: zone-b
+ nodeSelector: null
+ # -- extraAffinity adds user defined custom affinity rules (merged with generated rules)
+ extraAffinity: {}
+ # -- Alertmanager data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", then use `storageClassName: ""`, which disables dynamic provisioning
+ # If undefined or set to null (the default), then fall back to the value of `alertmanager.persistentVolume.storageClass`.
+ storageClass: null
+ # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+ - name: zone-c
+ # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
+ # nodeSelector:
+ # topology.kubernetes.io/zone: zone-c
+ nodeSelector: null
+ # -- extraAffinity adds user defined custom affinity rules (merged with generated rules)
+ extraAffinity: {}
+ # -- Alertmanager data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", then use `storageClassName: ""`, which disables dynamic provisioning
+ # If undefined or set to null (the default), then fall back to the value of `alertmanager.persistentVolume.storageClass`.
+ storageClass: null
+
+distributor:
+ # Setting it to null will produce a deployment without replicas set, allowing you to use autoscaling with the deployment
+ replicas: 1
+
+ # -- [Experimental] Configure autoscaling via KEDA (https://keda.sh). This requires having
+ # KEDA already installed in the Kubernetes cluster. The metrics for scaling are read
+ # according to top-level kedaAutoscaling.prometheusAddress (defaulting to metamonitoring remote-write destination).
+ # Basic auth and extra HTTP headers from metaMonitoring are ignored, please use customHeaders.
+ # The remote URL is used even if metamonitoring is disabled.
+ # See https://github.com/grafana/mimir/issues/7367 for more details on how to migrate to autoscaled resources without disruptions.
+ kedaAutoscaling:
+ enabled: false
+ # -- preserveReplicas gives you the option to migrate from non-autoscaled to autoscaled deployments without losing replicas. When set to true, the replica fields in the component will be left intact.
+ # For futher details see [helm: autoscaling migration procedure](https://github.com/grafana/mimir/issues/7367)
+ preserveReplicas: false
+ minReplicaCount: 1
+ maxReplicaCount: 10
+ targetCPUUtilizationPercentage: 100
+ targetMemoryUtilizationPercentage: 100
+ behavior:
+ scaleDown:
+ policies:
+ - periodSeconds: 600
+ type: Percent
+ value: 10
+
+ service:
+ annotations: {}
+ labels: {}
+
+ resources:
+ requests:
+ cpu: 100m
+ memory: 512Mi
+
+ # Additional distributor container arguments, e.g. log level (debug, info, warn, error)
+ extraArgs: {}
+
+ # Pod Labels
+ podLabels: {}
+
+ # Pod Annotations
+ podAnnotations: {}
+
+ # Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ # -- The name of the PriorityClass for distributor pods
+ priorityClassName: null
+
+ nodeSelector: {}
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ annotations: {}
+ persistence:
+ subPath:
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+ # -- SecurityContext override for distributor pods
+ securityContext: {}
+
+ # -- The SecurityContext for distributor containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 15%
+
+ # Keep the termination grace period higher than the -shutdown-delay configured on the distributor.
+ terminationGracePeriodSeconds: 100
+
+ tolerations: []
+ initContainers: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+ env: []
+ extraEnvFrom: []
+
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: 1000
+
+ingester:
+ # -- Total number of replicas for the ingester across all availability zones
+ # If ingester.zoneAwareReplication.enabled=false, this number is taken as is.
+ # Otherwise each zone starts `ceil(replicas / number_of_zones)` number of pods.
+ # E.g. if 'replicas' is set to 4 and there are 3 zones, then 4/3=1.33 and after rounding up it means 2 pods per zone are started.
+ replicas: 3
+
+ statefulSet:
+ enabled: true
+
+ service:
+ annotations: {}
+ labels: {}
+
+ # -- Optionally set the scheduler for pods of the ingester
+ schedulerName: ""
+
+ resources:
+ requests:
+ cpu: 100m
+ memory: 512Mi
+
+ # Additional ingester container arguments, e.g. log level (debug, info, warn, error)
+ extraArgs: {}
+ # Pod Labels
+ podLabels: {}
+
+ # Pod Annotations
+ podAnnotations: {}
+
+ # -- The name of the PriorityClass for ingester pods
+ priorityClassName: null
+
+ # -- Pod Disruption Budget for ingester, this will be applied across availability zones to prevent losing redundancy
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ podManagementPolicy: Parallel
+
+ # -- NodeSelector to pin ingester pods to certain set of nodes. This is ignored when ingester.zoneAwareReplication.enabled=true.
+ nodeSelector: {}
+ # -- Pod affinity settings for the ingester. This is ignored when ingester.zoneAwareReplication.enabled=true.
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ # It is recommended to replace this with requiredDuringSchedulingIgnoredDuringExecution podAntiAffinity rules when
+ # deploying to production.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ annotations: {}
+
+ persistentVolume:
+ # If true and ingester.statefulSet.enabled is true,
+ # Ingester will create/use a Persistent Volume Claim
+ # If false, use emptyDir
+ #
+ enabled: true
+
+ # Ingester data Persistent Volume Claim annotations
+ #
+ annotations: {}
+
+ # Ingester data Persistent Volume access modes
+ # Must match those of existing PV or dynamic provisioner
+ # Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
+ accessModes:
+ - ReadWriteOnce
+
+ # Ingester data Persistent Volume size
+ size: 2Gi
+
+ # Subdirectory of Ingester data Persistent Volume to mount
+ # Useful if the volume's root directory is not empty
+ subPath: ""
+
+ # -- Enable StatefulSetAutoDeletePVC feature
+ # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention
+ enableRetentionPolicy: false
+ whenDeleted: Retain
+ whenScaled: Retain
+
+ # Ingester data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", storageClassName: "", which disables dynamic provisioning
+ # If undefined (the default) or set to null, no storageClassName spec is
+ # set, choosing the default provisioner.
+ #
+ # A per-zone storageClass configuration in `ingester.zoneAwareReplication.zones[*].storageClass` takes precedence over this field.
+ #
+ # storageClass: "-"
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 60
+
+ # -- SecurityContext override for ingester pods
+ securityContext: {}
+
+ # -- The SecurityContext for ingester containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ # -- updateStrategy of the ingester statefulset. This is ignored when ingester.zoneAwareReplication.enabled=true.
+ statefulStrategy:
+ type: RollingUpdate
+
+ terminationGracePeriodSeconds: 240
+
+ tolerations: []
+ initContainers: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+ env: []
+ extraEnvFrom: []
+
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: 1000
+
+ # -- Options to configure zone-aware replication for ingester
+ # Example configuration with full geographical redundancy:
+ # rollout_operator:
+ # enabled: true
+ # ingester:
+ # zoneAwareReplication:
+ # enabled: true
+ # topologyKey: 'kubernetes.io/hostname' # This generates default anti-affinity rules
+ # zones: # Zone list has to be fully redefined for modification. Update with you actual zones or skip to use logical zones only.
+ # - name: zone-a
+ # nodeSelector:
+ # topology.kubernetes.io/zone: us-central1-a
+ # storageClass: storage-class-us-central1-a
+ # - name: zone-a
+ # nodeSelector:
+ # topology.kubernetes.io/zone: us-central1-b
+ # storageClass: storage-class-us-central1-b
+ # - name: zone-c
+ # nodeSelector:
+ # topology.kubernetes.io/zone: us-central1-c
+ # storageClass: storage-class-us-central1-c
+ #
+ zoneAwareReplication:
+ # -- Enable zone-aware replication for ingester
+ enabled: true
+ # -- Maximum number of ingesters that can be unavailable per zone during rollout
+ maxUnavailable: 50
+ # -- topologyKey to use in pod anti-affinity. If unset, no anti-affinity rules are generated. If set, the generated anti-affinity rule makes sure that pods from different zones do not mix.
+ # E.g.: topologyKey: 'kubernetes.io/hostname'
+ topologyKey: null
+ # -- Auxiliary values for migration, see https://grafana.com/docs/helm-charts/mimir-distributed/latest/migration-guides/migrate-from-single-zone-with-helm/
+ migration:
+ # -- Indicate if migration is ongoing for multi zone ingester
+ enabled: false
+ # -- Exclude default zone on write path
+ excludeDefaultZone: false
+ # -- Enable zone-awareness, read path only
+ readPath: false
+ # -- Total number of replicas to start in availability zones when migration is enabled
+ replicas: 0
+ # -- Scale default zone ingesters to 0
+ scaleDownDefaultZone: false
+ # -- Enable zone-awareness, write path only
+ writePath: false
+ # -- Zone definitions for ingester zones. Note: you have to redefine the whole list to change parts as YAML does not allow to modify parts of a list.
+ zones:
+ # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+ - name: zone-a
+ # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
+ # nodeSelector:
+ # topology.kubernetes.io/zone: zone-a
+ nodeSelector: null
+ # -- extraAffinity adds user defined custom affinity rules (merged with generated rules)
+ extraAffinity: {}
+ # -- Ingester data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", then use `storageClassName: ""`, which disables dynamic provisioning
+ # If undefined or set to null (the default), then fall back to the value of `ingester.persistentVolume.storageClass`.
+ storageClass: null
+ # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+ - name: zone-b
+ # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
+ # nodeSelector:
+ # topology.kubernetes.io/zone: zone-b
+ nodeSelector: null
+ # -- extraAffinity adds user defined custom affinity rules (merged with generated rules)
+ extraAffinity: {}
+ # -- Ingester data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", then use `storageClassName: ""`, which disables dynamic provisioning
+ # If undefined or set to null (the default), then fall back to the value of `ingester.persistentVolume.storageClass`.
+ storageClass: null
+ # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+ - name: zone-c
+ # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
+ # nodeSelector:
+ # topology.kubernetes.io/zone: zone-c
+ nodeSelector: null
+ # -- extraAffinity adds user defined custom affinity rules (merged with generated rules)
+ extraAffinity: {}
+ # -- Ingester data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", then use `storageClassName: ""`, which disables dynamic provisioning
+ # If undefined or set to null (the default), then fall back to the value of `ingester.persistentVolume.storageClass`.
+ storageClass: null
+
+overrides_exporter:
+ enabled: true
+ replicas: 1
+
+ annotations: {}
+
+ initContainers: []
+
+ service:
+ annotations: {}
+ labels: {}
+
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 15%
+
+ podLabels: {}
+ podAnnotations: {}
+ # Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ # -- The name of the PriorityClass for overrides-exporter pods
+ priorityClassName: null
+
+ nodeSelector: {}
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints: {}
+ # maxSkew: 1
+ # topologyKey: kubernetes.io/hostname
+ # whenUnsatisfiable: ScheduleAnyway
+
+ # -- SecurityContext override for overrides-exporter pods
+ securityContext: {}
+
+ # -- The SecurityContext for overrides-exporter containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ extraArgs: {}
+
+ persistence:
+ subPath:
+
+ livenessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+
+ terminationGracePeriodSeconds: 60
+
+ tolerations: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+ env: []
+ extraEnvFrom: []
+
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: null
+
+ruler:
+ enabled: true
+ replicas: 1
+
+ # -- [Experimental] Configure autoscaling via KEDA (https://keda.sh). This requires having
+ # KEDA already installed in the Kubernetes cluster. The metrics for scaling are read
+ # according to top-level kedaAutoscaling.prometheusAddress (defaulting to metamonitoring remote-write destination).
+ # Basic auth and extra HTTP headers from metaMonitoring are ignored, please use customHeaders.
+ # The remote URL is used even if metamonitoring is disabled.
+ # See https://github.com/grafana/mimir/issues/7367 for more details on how to migrate to autoscaled resources without disruptions.
+ kedaAutoscaling:
+ enabled: false
+ # -- preserveReplicas gives you the option to migrate from non-autoscaled to autoscaled deployments without losing replicas. When set to true, the replica fields in the component will be left intact.
+ # For futher details see [helm: autoscaling migration procedure](https://github.com/grafana/mimir/issues/7367)
+ preserveReplicas: false
+ minReplicaCount: 1
+ maxReplicaCount: 10
+ targetCPUUtilizationPercentage: 100
+ targetMemoryUtilizationPercentage: 100
+ behavior:
+ scaleDown:
+ policies:
+ - periodSeconds: 600
+ type: Percent
+ value: 10
+
+ service:
+ annotations: {}
+ labels: {}
+
+ # -- Dedicated service account for ruler pods.
+ # If not set, the default service account defined at the begining of this file will be used.
+ # This service account can be used even if the default one is not set.
+ serviceAccount:
+ create: false
+ # -- Ruler specific service account name. If not set and create is set to true, the default
+ # name will be the default mimir service account's name with the "-ruler" suffix.
+ name: ""
+ annotations: {}
+ labels: {}
+
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+
+ # Additional ruler container arguments, e.g. log level (debug, info, warn, error)
+ extraArgs: {}
+ # log.level: debug
+
+ # Pod Labels
+ podLabels: {}
+
+ # Pod Annotations
+ podAnnotations: {}
+
+ # Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ nodeSelector: {}
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ annotations: {}
+ persistence:
+ subPath:
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+ # -- SecurityContext override for ruler pods
+ securityContext: {}
+
+ # -- The SecurityContext for ruler containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 50%
+ maxUnavailable: 0
+
+ terminationGracePeriodSeconds: 180
+
+ tolerations: []
+ initContainers: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+ env: []
+ extraEnvFrom: []
+
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: 1000
+
+querier:
+ replicas: 2
+
+ # -- [Experimental] Configure autoscaling via KEDA (https://keda.sh). This requires having
+ # KEDA already installed in the Kubernetes cluster. The metrics for scaling are read
+ # according to top-level kedaAutoscaling.prometheusAddress (defaulting to metamonitoring remote-write destination).
+ # Basic auth and extra HTTP headers from metaMonitoring are ignored, please use customHeaders.
+ # The remote URL is used even if metamonitoring is disabled.
+ # See https://github.com/grafana/mimir/issues/7367 for more details on how to migrate to autoscaled resources without disruptions.
+ kedaAutoscaling:
+ enabled: false
+ # -- preserveReplicas gives you the option to migrate from non-autoscaled to autoscaled deployments without losing replicas. When set to true, the replica fields in the component will be left intact.
+ # For futher details see [helm: autoscaling migration procedure](https://github.com/grafana/mimir/issues/7367)
+ preserveReplicas: false
+ minReplicaCount: 1
+ maxReplicaCount: 10
+ querySchedulerInflightRequestsThreshold: 12
+ behavior:
+ scaleDown:
+ policies:
+ - periodSeconds: 120
+ type: Percent
+ value: 10
+ stabilizationWindowSeconds: 600
+ scaleUp:
+ policies:
+ - periodSeconds: 120
+ type: Percent
+ value: 50
+ - periodSeconds: 120
+ type: Pods
+ value: 15
+ stabilizationWindowSeconds: 60
+
+ service:
+ annotations: {}
+ labels: {}
+
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+
+ # Additional querier container arguments, e.g. log level (debug, info, warn, error)
+ extraArgs: {}
+
+ # Pod Labels
+ podLabels: {}
+
+ # Pod Annotations
+ podAnnotations: {}
+
+ # Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ # -- The name of the PriorityClass for querier pods
+ priorityClassName: null
+
+ nodeSelector: {}
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ annotations: {}
+ persistence:
+ subPath:
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+ # -- SecurityContext override for querier pods
+ securityContext: {}
+
+ # -- The SecurityContext for querier containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 15%
+
+ terminationGracePeriodSeconds: 180
+
+ tolerations: []
+ initContainers: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+ env: []
+ extraEnvFrom: []
+
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: 5000
+
+query_frontend:
+ # Setting it to null will produce a deployment without replicas set, allowing you to use autoscaling with the deployment
+ replicas: 1
+
+ # -- [Experimental] Configure autoscaling via KEDA (https://keda.sh). This requires having
+ # KEDA already installed in the Kubernetes cluster. The metrics for scaling are read
+ # according to top-level kedaAutoscaling.prometheusAddress (defaulting to metamonitoring remote-write destination).
+ # Basic auth and extra HTTP headers from metaMonitoring are ignored, please use customHeaders.
+ # The remote URL is used even if metamonitoring is disabled.
+ # See https://github.com/grafana/mimir/issues/7367 for more details on how to migrate to autoscaled resources without disruptions.
+ kedaAutoscaling:
+ enabled: false
+ # -- preserveReplicas gives you the option to migrate from non-autoscaled to autoscaled deployments without losing replicas. When set to true, the replica fields in the component will be left intact.
+ # For futher details see [helm: autoscaling migration procedure](https://github.com/grafana/mimir/issues/7367)
+ preserveReplicas: false
+ minReplicaCount: 1
+ maxReplicaCount: 10
+ targetCPUUtilizationPercentage: 75
+ targetMemoryUtilizationPercentage: 100
+ behavior:
+ scaleDown:
+ policies:
+ - periodSeconds: 60
+ type: Percent
+ value: 10
+
+ service:
+ annotations: {}
+ labels: {}
+
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+
+ # Additional query-frontend container arguments, e.g. log level (debug, info, warn, error)
+ extraArgs: {}
+
+ # Pod Labels
+ podLabels: {}
+
+ # Pod Annotations
+ podAnnotations: {}
+
+ # Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ # -- The name of the PriorityClass for query-frontend pods
+ priorityClassName: null
+
+ nodeSelector: {}
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ annotations: {}
+ persistence:
+ subPath:
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+ # -- SecurityContext override for query-fronted pods
+ securityContext: {}
+
+ # -- The SecurityContext for query-frontend containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 15%
+
+ terminationGracePeriodSeconds: 390
+
+ tolerations: []
+ initContainers: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+ env: []
+ extraEnvFrom: []
+
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: 5000
+
+query_scheduler:
+ enabled: true
+ replicas: 2
+
+ service:
+ annotations: {}
+ labels: {}
+
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+
+ # Additional query-scheduler container arguments, e.g. log level (debug, info, warn, error)
+ extraArgs: {}
+
+ # Pod Labels
+ podLabels: {}
+
+ # Pod Annotations
+ podAnnotations: {}
+
+ # Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ # -- The name of the PriorityClass for query-scheduler pods
+ priorityClassName: null
+
+ nodeSelector: {}
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ annotations: {}
+ persistence:
+ subPath:
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+ # -- SecurityContext override for query-scheduler pods
+ securityContext: {}
+
+ # -- The SecurityContext for query-scheduler containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 1
+
+ terminationGracePeriodSeconds: 180
+
+ tolerations: []
+ initContainers: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+ env: []
+ extraEnvFrom: []
+
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: null
+
+store_gateway:
+ # -- Total number of replicas for the store-gateway across all availability zones
+ # If store_gateway.zoneAwareReplication.enabled=false, this number is taken as is.
+ # Otherwise each zone starts `ceil(replicas / number_of_zones)` number of pods.
+ # E.g. if 'replicas' is set to 4 and there are 3 zones, then 4/3=1.33 and after rounding up it means 2 pods per zone are started.
+ replicas: 1
+
+ service:
+ annotations: {}
+ labels: {}
+
+ # -- Optionally set the scheduler for pods of the store-gateway
+ schedulerName: ""
+
+ resources:
+ requests:
+ cpu: 100m
+ memory: 512Mi
+
+ # Additional store-gateway container arguments, e.g. log level (debug, info, warn, error)
+ extraArgs: {}
+
+ # Pod Labels
+ podLabels: {}
+
+ # Pod Annotations
+ podAnnotations: {}
+
+ # -- Management policy for store-gateway pods
+ # New variable introduced with Helm chart version 5.1.0. For backwards compatibility it is set to `OrderedReady`
+ # On new deployments it is highly recommended to switch it to `Parallel` as this will be the new default from 6.0.0
+ podManagementPolicy: OrderedReady
+
+ # -- Pod Disruption Budget for store-gateway, this will be applied across availability zones to prevent losing redundancy
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ # -- The name of the PriorityClass for store-gateway pods
+ priorityClassName: null
+
+ # -- NodeSelector to pin store-gateway pods to certain set of nodes. This is ignored when store_gateway.zoneAwareReplication.enabled=true.
+ nodeSelector: {}
+ # -- Pod affinity settings for the store_gateway. This is ignored when store_gateway.zoneAwareReplication.enabled=true.
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ # It is recommended to replace this with requiredDuringSchedulingIgnoredDuringExecution podAntiAffinity rules when
+ # deploying to production.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ annotations: {}
+
+ persistentVolume:
+ # If true Store-gateway will create/use a Persistent Volume Claim
+ # If false, use emptyDir
+ #
+ enabled: true
+
+ # Store-gateway data Persistent Volume Claim annotations
+ #
+ annotations: {}
+
+ # Store-gateway data Persistent Volume access modes
+ # Must match those of existing PV or dynamic provisioner
+ # Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
+ #
+ accessModes:
+ - ReadWriteOnce
+
+ # Store-gateway data Persistent Volume size
+ #
+ size: 2Gi
+
+ # Subdirectory of Store-gateway data Persistent Volume to mount
+ # Useful if the volume's root directory is not empty
+ #
+ subPath: ""
+
+ # Store-gateway data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", storageClassName: "", which disables dynamic provisioning
+ # If undefined (the default) or set to null, no storageClassName spec is
+ # set, choosing the default provisioner.
+ #
+ # A per-zone storageClass configuration in `store_gateway.zoneAwareReplication.zones[*].storageClass` takes precedence over this field.
+ # storageClass: "-"
+
+ # -- Enable StatefulSetAutoDeletePVC feature
+ # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention
+ enableRetentionPolicy: false
+ whenDeleted: Retain
+ whenScaled: Retain
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 60
+
+ # -- SecurityContext override for store-gateway pods
+ securityContext: {}
+
+ # -- The SecurityContext for store-gateway containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ # -- updateStrategy of the store-gateway statefulset. This is ignored when store_gateway.zoneAwareReplication.enabled=true.
+ strategy:
+ type: RollingUpdate
+
+ terminationGracePeriodSeconds: 240
+
+ tolerations: []
+ initContainers: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+ env: []
+ extraEnvFrom: []
+
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: 1000
+
+ # -- Options to configure zone-aware replication for store-gateway
+ # Example configuration with full geographical redundancy:
+ # rollout_operator:
+ # enabled: true
+ # store_gateway:
+ # zoneAwareReplication:
+ # enabled: true
+ # topologyKey: 'kubernetes.io/hostname' # This generates default anti-affinity rules
+ # zones: # Zone list has to be fully redefined for modification. Update with you actual zones or skip to use logical zones only.
+ # - name: zone-a
+ # nodeSelector:
+ # topology.kubernetes.io/zone: us-central1-a
+ # - name: zone-a
+ # nodeSelector:
+ # topology.kubernetes.io/zone: us-central1-b
+ # - name: zone-c
+ # nodeSelector:
+ # topology.kubernetes.io/zone: us-central1-c
+ #
+ zoneAwareReplication:
+ # -- Enable zone-aware replication for store-gateway
+ enabled: true
+ # -- Maximum number of store-gateways that can be unavailable per zone during rollout
+ maxUnavailable: 50
+ # -- topologyKey to use in pod anti-affinity. If unset, no anti-affinity rules are generated. If set, the generated anti-affinity rule makes sure that pods from different zones do not mix.
+ # E.g.: topologyKey: 'kubernetes.io/hostname'
+ topologyKey: null
+ # -- Auxiliary values for migration, see https://grafana.com/docs/helm-charts/mimir-distributed/latest/migration-guides/migrate-from-single-zone-with-helm/
+ migration:
+ # -- Indicate if migration is ongoing for multi zone store-gateway
+ enabled: false
+ # -- Enable zone-awareness on the readPath
+ readPath: false
+ # -- Zone definitions for store-gateway zones. Note: you have to redefine the whole list to change parts as YAML does not allow to modify parts of a list.
+ zones:
+ # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+ - name: zone-a
+ # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
+ # nodeSelector:
+ # topology.kubernetes.io/zone: zone-a
+ nodeSelector: null
+ # -- extraAffinity adds user defined custom affinity rules (merged with generated rules)
+ extraAffinity: {}
+ # -- StoreGateway data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", then use `storageClassName: ""`, which disables dynamic provisioning
+ # If undefined or set to null (the default), then fall back to the value of `store_gateway.persistentVolume.storageClass`.
+ storageClass: null
+ # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+ - name: zone-b
+ # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
+ # nodeSelector:
+ # topology.kubernetes.io/zone: zone-b
+ nodeSelector: null
+ # -- extraAffinity adds user defined custom affinity rules (merged with generated rules)
+ extraAffinity: {}
+ # -- StoreGateway data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", then use `storageClassName: ""`, which disables dynamic provisioning
+ # If undefined or set to null (the default), then fall back to the value of `store_gateway.persistentVolume.storageClass`.
+ storageClass: null
+ # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+ - name: zone-c
+ # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
+ # nodeSelector:
+ # topology.kubernetes.io/zone: zone-c
+ nodeSelector: null
+ # -- extraAffinity adds user defined custom affinity rules (merged with generated rules)
+ extraAffinity: {}
+ # -- StoreGateway data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", then use `storageClassName: ""`, which disables dynamic provisioning
+ # If undefined or set to null (the default), then fall back to the value of `store_gateway.persistentVolume.storageClass`.
+ storageClass: null
+
+compactor:
+ replicas: 1
+
+ service:
+ annotations: {}
+ labels: {}
+
+ # -- Optionally set the scheduler for pods of the compactor
+ schedulerName: ""
+
+ resources:
+ requests:
+ cpu: 100m
+ memory: 512Mi
+
+ # Additional compactor container arguments, e.g. log level (debug, info, warn, error)
+ extraArgs: {}
+
+ # Pod Labels
+ podLabels: {}
+
+ # Pod Annotations
+ podAnnotations: {}
+
+ # Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ podManagementPolicy: OrderedReady
+
+ # -- The name of the PriorityClass for compactor pods
+ priorityClassName: null
+
+ nodeSelector: {}
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ annotations: {}
+
+ persistentVolume:
+ # If true compactor will create/use a Persistent Volume Claim
+ # If false, use emptyDir
+ #
+ enabled: true
+
+ # compactor data Persistent Volume Claim annotations
+ #
+ annotations: {}
+
+ # compactor data Persistent Volume access modes
+ # Must match those of existing PV or dynamic provisioner
+ # Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
+ #
+ accessModes:
+ - ReadWriteOnce
+
+ # compactor data Persistent Volume size
+ #
+ size: 2Gi
+
+ # Subdirectory of compactor data Persistent Volume to mount
+ # Useful if the volume's root directory is not empty
+ #
+ subPath: ""
+
+ # compactor data Persistent Volume Storage Class
+ # If defined, storageClassName:
+ # If set to "-", storageClassName: "", which disables dynamic provisioning
+ # If undefined (the default) or set to null, no storageClassName spec is
+ # set, choosing the default provisioner.
+ #
+ # storageClass: "-"
+
+ # -- Enable StatefulSetAutoDeletePVC feature
+ # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention
+ enableRetentionPolicy: false
+ whenDeleted: Retain
+ whenScaled: Retain
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 60
+
+ # -- SecurityContext override for compactor pods
+ securityContext: {}
+
+ # -- The SecurityContext for compactor containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ strategy:
+ type: RollingUpdate
+
+ terminationGracePeriodSeconds: 240
+
+ tolerations: []
+ initContainers: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+ env: []
+ extraEnvFrom: []
+
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: null
+
+memcached:
+ image:
+ # -- Memcached Docker image repository
+ repository: memcached
+ # -- Memcached Docker image tag
+ tag: 1.6.25-alpine
+ # -- Memcached Docker image pull policy
+ pullPolicy: IfNotPresent
+
+ # -- The SecurityContext override for memcached pods
+ podSecurityContext: {}
+
+ # -- The name of the PriorityClass for memcached pods
+ priorityClassName: null
+
+ # -- The SecurityContext for memcached containers
+ containerSecurityContext:
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+ allowPrivilegeEscalation: false
+
+memcachedExporter:
+ # -- Whether memcached metrics should be exported
+ enabled: true
+
+ image:
+ repository: prom/memcached-exporter
+ tag: v0.14.3
+ pullPolicy: IfNotPresent
+
+ resources:
+ requests: {}
+ limits: {}
+
+ # -- The SecurityContext for memcached exporter containers
+ containerSecurityContext:
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+ allowPrivilegeEscalation: false
+
+ # -- Extra args to add to the exporter container.
+ # Example:
+ # extraArgs:
+ # memcached.tls.enable: true
+ # memcached.tls.cert-file: /certs/cert.crt
+ # memcached.tls.key-file: /certs/cert.key
+ # memcached.tls.ca-file: /certs/ca.crt
+ # memcached.tls.insecure-skip-verify: false
+ # memcached.tls.server-name: memcached
+ extraArgs: {}
+
+chunks-cache:
+ # -- Specifies whether memcached based chunks-cache should be enabled
+ enabled: false
+
+ # -- Total number of chunks-cache replicas
+ replicas: 1
+
+ # -- Port of the chunks-cache service
+ port: 11211
+
+ # -- Amount of memory allocated to chunks-cache for object storage (in MB).
+ allocatedMemory: 8192
+
+ # -- Maximum item memory for chunks-cache (in MB).
+ maxItemMemory: 1
+
+ # -- Maximum number of connections allowed
+ connectionLimit: 16384
+
+ # -- Extra init containers for chunks-cache pods
+ initContainers: []
+
+ # -- Annotations for the chunks-cache pods
+ annotations: {}
+ # -- Node selector for chunks-cache pods
+ nodeSelector: {}
+ # -- Affinity for chunks-cache pods
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints: {}
+ # maxSkew: 1
+ # topologyKey: kubernetes.io/hostname
+ # whenUnsatisfiable: ScheduleAnyway
+
+ # -- Tolerations for chunks-cache pods
+ tolerations: []
+ # -- Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+ # -- The name of the PriorityClass for chunks-cache pods
+ priorityClassName: null
+ # -- Labels for chunks-cache pods
+ podLabels: {}
+ # -- Annotations for chunks-cache pods
+ podAnnotations: {}
+ # -- Management policy for chunks-cache pods
+ podManagementPolicy: Parallel
+ # -- Grace period to allow the chunks-cache to shutdown before it is killed
+ terminationGracePeriodSeconds: 60
+
+ # -- Stateful chunks-cache strategy
+ statefulStrategy:
+ type: RollingUpdate
+
+ # -- Add extended options for chunks-cache memcached container. The format is the same as for the memcached -o/--extend flag.
+ # Example:
+ # extraExtendedOptions: 'tls,no_hashexpand'
+ extraExtendedOptions: ""
+
+ # -- Additional CLI args for chunks-cache
+ extraArgs: {}
+
+ # -- Additional containers to be added to the chunks-cache pod.
+ extraContainers: []
+
+ # -- Additional volumes to be added to the chunks-cache pod (applies to both memcached and exporter containers).
+ # Example:
+ # extraVolumes:
+ # - name: extra-volume
+ # secret:
+ # secretName: extra-volume-secret
+ extraVolumes: []
+
+ # -- Additional volume mounts to be added to the chunks-cache pod (applies to both memcached and exporter containers).
+ # Example:
+ # extraVolumeMounts:
+ # - name: extra-volume
+ # mountPath: /etc/extra-volume
+ # readOnly: true
+ extraVolumeMounts: []
+
+ # -- Resource requests and limits for the chunks-cache
+ # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+ resources: null
+
+ # -- Service annotations and labels
+ service:
+ annotations: {}
+ labels: {}
+
+index-cache:
+ # -- Specifies whether memcached based index-cache should be enabled
+ enabled: false
+
+ # -- Total number of index-cache replicas
+ replicas: 1
+
+ # -- Port of the index-cache service
+ port: 11211
+
+ # -- Amount of memory allocated to index-cache for object storage (in MB).
+ allocatedMemory: 2048
+
+ # -- Maximum item index-cache for memcached (in MB).
+ maxItemMemory: 5
+
+ # -- Maximum number of connections allowed
+ connectionLimit: 16384
+
+ # -- Extra init containers for index-cache pods
+ initContainers: []
+
+ # -- Annotations for the index-cache pods
+ annotations: {}
+ # -- Node selector for index-cache pods
+ nodeSelector: {}
+ # -- Affinity for index-cache pods
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints: {}
+ # maxSkew: 1
+ # topologyKey: kubernetes.io/hostname
+ # whenUnsatisfiable: ScheduleAnyway
+
+ # -- Tolerations for index-cache pods
+ tolerations: []
+ # -- Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+ # -- The name of the PriorityClass for index-cache pods
+ priorityClassName: null
+ # -- Labels for index-cache pods
+ podLabels: {}
+ # -- Annotations for index-cache pods
+ podAnnotations: {}
+ # -- Management policy for index-cache pods
+ podManagementPolicy: Parallel
+ # -- Grace period to allow the index-cache to shutdown before it is killed
+ terminationGracePeriodSeconds: 60
+
+ # -- Stateful index-cache strategy
+ statefulStrategy:
+ type: RollingUpdate
+
+ # -- Add extended options for index-cache memcached container. The format is the same as for the memcached -o/--extend flag.
+ # Example:
+ # extraExtendedOptions: 'tls,modern,track_sizes'
+ extraExtendedOptions: ""
+
+ # -- Additional CLI args for index-cache
+ extraArgs: {}
+
+ # -- Additional containers to be added to the index-cache pod.
+ extraContainers: []
+
+ # -- Additional volumes to be added to the index-cache pod (applies to both memcached and exporter containers).
+ # Example:
+ # extraVolumes:
+ # - name: extra-volume
+ # secret:
+ # secretName: extra-volume-secret
+ extraVolumes: []
+
+ # -- Additional volume mounts to be added to the index-cache pod (applies to both memcached and exporter containers).
+ # Example:
+ # extraVolumeMounts:
+ # - name: extra-volume
+ # mountPath: /etc/extra-volume
+ # readOnly: true
+ extraVolumeMounts: []
+
+ # -- Resource requests and limits for the index-cache
+ # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+ resources: null
+
+ # -- Service annotations and labels
+ service:
+ annotations: {}
+ labels: {}
+
+metadata-cache:
+ # -- Specifies whether memcached based metadata-cache should be enabled
+ enabled: false
+
+ # -- Total number of metadata-cache replicas
+ replicas: 1
+
+ # -- Port of the metadata-cache service
+ port: 11211
+
+ # -- Amount of memory allocated to metadata-cache for object storage (in MB).
+ allocatedMemory: 512
+
+ # -- Maximum item metadata-cache for memcached (in MB).
+ maxItemMemory: 1
+
+ # -- Maximum number of connections allowed
+ connectionLimit: 16384
+
+ # -- Extra init containers for metadata-cache pods
+ initContainers: []
+
+ # -- Annotations for the metadata-cache pods
+ annotations: {}
+ # -- Node selector for metadata-cache pods
+ nodeSelector: {}
+ # -- Affinity for metadata-cache pods
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints: {}
+ # maxSkew: 1
+ # topologyKey: kubernetes.io/hostname
+ # whenUnsatisfiable: ScheduleAnyway
+
+ # -- Tolerations for metadata-cache pods
+ tolerations: []
+ # -- Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+ # -- The name of the PriorityClass for metadata-cache pods
+ priorityClassName: null
+ # -- Labels for metadata-cache pods
+ podLabels: {}
+ # -- Annotations for metadata-cache pods
+ podAnnotations: {}
+ # -- Management policy for metadata-cache pods
+ podManagementPolicy: Parallel
+ # -- Grace period to allow the metadata-cache to shutdown before it is killed
+ terminationGracePeriodSeconds: 60
+
+ # -- Stateful metadata-cache strategy
+ statefulStrategy:
+ type: RollingUpdate
+
+ # -- Add extended options for metadata-cache memcached container. The format is the same as for the memcached -o/--extend flag.
+ # Example:
+ # extraExtendedOptions: 'tls,modern,track_sizes'
+ extraExtendedOptions: ""
+
+ # -- Additional CLI args for metadata-cache
+ extraArgs: {}
+
+ # -- Additional containers to be added to the metadata-cache pod.
+ extraContainers: []
+
+ # -- Additional volumes to be added to the metadata-cache pod (applies to both memcached and exporter containers).
+ # Example:
+ # extraVolumes:
+ # - name: extra-volume
+ # secret:
+ # secretName: extra-volume-secret
+ extraVolumes: []
+
+ # -- Additional volume mounts to be added to the metadata-cache pod (applies to both memcached and exporter containers).
+ # Example:
+ # extraVolumeMounts:
+ # - name: extra-volume
+ # mountPath: /etc/extra-volume
+ # readOnly: true
+ extraVolumeMounts: []
+
+ # -- Resource requests and limits for the metadata-cache
+ # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+ resources: null
+
+ # -- Service annotations and labels
+ service:
+ annotations: {}
+ labels: {}
+
+results-cache:
+ # -- Specifies whether memcached based results-cache should be enabled
+ enabled: false
+
+ # -- Total number of results-cache replicas
+ replicas: 1
+
+ # -- Port of the results-cache service
+ port: 11211
+
+ # -- Amount of memory allocated to results-cache for object storage (in MB).
+ allocatedMemory: 512
+
+ # -- Maximum item results-cache for memcached (in MB).
+ maxItemMemory: 5
+
+ # -- Maximum number of connections allowed
+ connectionLimit: 16384
+
+ # -- Extra init containers for results-cache pods
+ initContainers: []
+
+ # -- Annotations for the results-cache pods
+ annotations: {}
+ # -- Node selector for results-cache pods
+ nodeSelector: {}
+ # -- Affinity for results-cache pods
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints: {}
+ # maxSkew: 1
+ # topologyKey: kubernetes.io/hostname
+ # whenUnsatisfiable: ScheduleAnyway
+
+ # -- Tolerations for results-cache pods
+ tolerations: []
+ # -- Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+ # -- The name of the PriorityClass for results-cache pods
+ priorityClassName: null
+ # -- Labels for results-cache pods
+ podLabels: {}
+ # -- Annotations for results-cache pods
+ podAnnotations: {}
+ # -- Management policy for results-cache pods
+ podManagementPolicy: Parallel
+ # -- Grace period to allow the results-cache to shutdown before it is killed
+ terminationGracePeriodSeconds: 60
+
+ # -- Stateful results-cache strategy
+ statefulStrategy:
+ type: RollingUpdate
+
+ # -- Add extended options for results-cache memcached container. The format is the same as for the memcached -o/--extend flag.
+ # Example:
+ # extraExtendedOptions: 'tls,modern,track_sizes'
+ extraExtendedOptions: ""
+
+ # -- Additional CLI args for results-cache
+ extraArgs: {}
+
+ # -- Additional containers to be added to the results-cache pod.
+ extraContainers: []
+
+ # -- Additional volumes to be added to the results-cache pod (applies to both memcached and exporter containers).
+ # Example:
+ # extraVolumes:
+ # - name: extra-volume
+ # secret:
+ # secretName: extra-volume-secret
+ extraVolumes: []
+
+ # -- Additional volume mounts to be added to the results-cache pod (applies to both memcached and exporter containers).
+ # Example:
+ # extraVolumeMounts:
+ # - name: extra-volume
+ # mountPath: /etc/extra-volume
+ # readOnly: true
+ extraVolumeMounts: []
+
+ # -- Resource requests and limits for the results-cache
+ # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+ resources: null
+
+ # -- Service annotations and labels
+ service:
+ annotations: {}
+ labels: {}
+
+# -- Setting for the Grafana Rollout Operator https://github.com/grafana/helm-charts/tree/main/charts/rollout-operator
+rollout_operator:
+ enabled: true
+
+ # -- podSecurityContext is the pod security context for the rollout operator.
+ # When installing on OpenShift, override podSecurityContext settings with
+ #
+ # rollout_operator:
+ # podSecurityContext:
+ # fsGroup: null
+ # runAsGroup: null
+ # runAsUser: null
+ podSecurityContext:
+ fsGroup: 10001
+ runAsGroup: 10001
+ runAsNonRoot: true
+ runAsUser: 10001
+ seccompProfile:
+ type: RuntimeDefault
+
+ # Set the container security context
+ securityContext:
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+ allowPrivilegeEscalation: false
+
+minio:
+ enabled: true
+ mode: standalone
+ rootUser: grafana-mimir
+ buckets:
+ - name: mimir-tsdb
+ policy: none
+ purge: false
+ - name: mimir-ruler
+ policy: none
+ purge: false
+ - name: enterprise-metrics-tsdb
+ policy: none
+ purge: false
+ - name: enterprise-metrics-admin
+ policy: none
+ purge: false
+ - name: enterprise-metrics-ruler
+ policy: none
+ purge: false
+ persistence:
+ enabled: true
+ size: 5Gi
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+ rootPassword: supersecret
+ # Changed the mc config path to '/tmp' from '/etc' as '/etc' is only writable by root and OpenShift will not permit this.
+ configPathmc: "/tmp/minio/mc/"
+
+# -- DEPRECATED: use the 'gateway' section instead. For a migration guide refer to
+# https://grafana.com/docs/helm-charts/mimir-distributed/latest/migration-guides/migrate-to-unified-proxy-deployment/
+#
+# Configuration for nginx gateway.
+# Can only be enabled when 'enterprise.enabled' is false.
+nginx:
+ # -- Specifies whether nginx should be enabled
+ enabled: true
+ # -- Number of replicas for nginx
+ replicas: 1
+ # -- Enable logging of 2xx and 3xx HTTP requests
+ verboseLogging: true
+ autoscaling:
+ # -- Enable autoscaling for nginx
+ enabled: false
+ # -- Minimum autoscaling replicas for nginx
+ minReplicas: 1
+ # -- Maximum autoscaling replicas for nginx
+ maxReplicas: 3
+ # -- Target CPU utilisation percentage for nginx
+ targetCPUUtilizationPercentage: 60
+ # -- Target memory utilisation percentage for nginx
+ targetMemoryUtilizationPercentage:
+ # -- See `kubectl explain deployment.spec.strategy` for more,
+ # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
+ deploymentStrategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 15%
+ image:
+ # -- The Docker registry for nginx image
+ registry: docker.io
+ # -- The nginx image repository
+ repository: nginxinc/nginx-unprivileged
+ # -- The nginx image tag
+ tag: 1.25-alpine
+ # -- The nginx image pull policy
+ pullPolicy: IfNotPresent
+ # -- The name of the PriorityClass for nginx pods
+ priorityClassName: null
+ # -- Labels for nginx pods
+ podLabels: {}
+ # -- Annotations for nginx pods
+ podAnnotations: {}
+ # -- Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+ # -- Additional CLI args for nginx
+ extraArgs: {}
+ # -- Environment variables to add to the nginx pods
+ extraEnv: []
+ # -- Environment variables from secrets or configmaps to add to the nginx pods
+ extraEnvFrom: []
+ # -- Volumes to add to the nginx pods
+ extraVolumes: []
+ # -- Volume mounts to add to the nginx pods
+ extraVolumeMounts: []
+ # -- The SecurityContext override for nginx containers
+ podSecurityContext: {}
+
+ # -- The SecurityContext for nginx containers
+ containerSecurityContext:
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+ allowPrivilegeEscalation: false
+ # -- Resource requests and limits for the nginx
+ resources: {}
+ # -- Grace period to allow the nginx to shutdown before it is killed
+ terminationGracePeriodSeconds: 30
+ # -- Affinity for nginx pods. Passed through `tpl` and, thus, to be configured as string
+ # @default -- Hard node and soft zone anti-affinity
+ affinity: ""
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ annotations: {}
+
+ # -- Node selector for nginx pods
+ nodeSelector: {}
+ # -- Tolerations for nginx pods
+ tolerations: []
+ # Nginx service configuration
+ service:
+ # -- Port of the nginx service
+ port: 80
+ # -- Type of the nginx service
+ type: ClusterIP
+ # -- ClusterIP of the nginx service
+ clusterIP: null
+ # -- Node port if service type is NodePort
+ nodePort: null
+ # -- Load balancer IP address if service type is LoadBalancer
+ loadBalancerIP: null
+ # -- Annotations for the nginx service
+ annotations: {}
+ # -- Labels for nginx service
+ labels: {}
+ # Ingress configuration
+ ingress:
+ # -- Specifies whether an ingress for the nginx should be created
+ enabled: false
+ # -- Ingress Class Name. MAY be required for Kubernetes versions >= 1.18
+ # ingressClassName: nginx
+ # -- Annotations for the nginx ingress
+ annotations: {}
+ # -- Hosts configuration for the nginx ingress
+ hosts:
+ - host: nginx.mimir.example.com
+ paths:
+ - path: /
+ # -- pathType (e.g. ImplementationSpecific, Prefix, .. etc.) might also be required by some Ingress Controllers
+ # pathType: Prefix
+ # -- TLS configuration for the nginx ingress
+ tls:
+ - secretName: mimir-nginx-tls
+ hosts:
+ - nginx.mimir.example.com
+ # -- Route configuration (for OpenShift only)
+ route:
+ # -- Specifies whether an OpenShift route for the nginx should be created
+ enabled: false
+ # -- Annotations for the nginx route
+ annotations: {}
+ # -- Hostname configuration
+ host: nginx.mimir.example.com
+ # -- TLS configuration for OpenShift Route
+ tls:
+ # -- More details about TLS configuration and termination types: https://docs.openshift.com/container-platform/3.11/architecture/networking/routes.html#secured-routes
+ # For OpenShift 4: https://docs.openshift.com/container-platform/4.11/networking/routes/secured-routes.html
+ termination: edge
+ # Basic auth configuration
+ basicAuth:
+ # -- Enables basic authentication for nginx
+ enabled: false
+ # -- The basic auth username for nginx
+ username: null
+ # -- The basic auth password for nginx
+ password: null
+ # -- Uses the specified username and password to compute a htpasswd using Sprig's `htpasswd` function.
+ # The value is templated using `tpl`. Override this to use a custom htpasswd, e.g. in case the default causes
+ # high CPU load.
+ htpasswd: >-
+ {{ htpasswd (required "'nginx.basicAuth.username' is required" .Values.nginx.basicAuth.username) (required "'nginx.basicAuth.password' is required" .Values.nginx.basicAuth.password) }}
+ # -- Existing basic auth secret to use. Must contain '.htpasswd'
+ existingSecret: null
+ # Configures the readiness probe for nginx
+ readinessProbe:
+ httpGet:
+ path: /
+ port: http-metric
+ initialDelaySeconds: 15
+ timeoutSeconds: 1
+
+ # -- Additional containers to be added to the nginx pod.
+ extraContainers: []
+ # - name: dnsmasq
+ # image: "janeczku/go-dnsmasq:release-1.0.7"
+ # imagePullPolicy: IfNotPresent
+ # args:
+ # - --listen
+ # - "127.0.0.1:8053"
+ # - --hostsfile=/etc/hosts
+ # - --enable-search
+ # - --verbose
+
+ nginxConfig:
+ # -- NGINX log format
+ logFormat: |-
+ main '$remote_addr - $remote_user [$time_local] $status '
+ '"$request" $body_bytes_sent "$http_referer" '
+ '"$http_user_agent" "$http_x_forwarded_for"';
+ # -- Sets the log level of the NGINX error log. One of `debug`, `info`, `notice`, `warn`, `error`, `crit`, `alert`, or `emerg`
+ errorLogLevel: error
+ # -- Enables NGINX access logs
+ accessLogEnabled: true
+ # -- Allows appending custom configuration to the server block
+ serverSnippet: ""
+ # -- Allows appending custom configuration to the http block
+ httpSnippet: ""
+ # -- Allows to set a custom resolver
+ resolver: null
+ # -- Config file contents for Nginx. Passed through the `tpl` function to allow templating
+ # @default -- See values.yaml
+ file: |
+ worker_processes 5; ## Default: 1
+ error_log /dev/stderr {{ .Values.nginx.nginxConfig.errorLogLevel }};
+ pid /tmp/nginx.pid;
+ worker_rlimit_nofile 8192;
+
+ events {
+ worker_connections 4096; ## Default: 1024
+ }
+
+ http {
+ client_body_temp_path /tmp/client_temp;
+ proxy_temp_path /tmp/proxy_temp_path;
+ fastcgi_temp_path /tmp/fastcgi_temp;
+ uwsgi_temp_path /tmp/uwsgi_temp;
+ scgi_temp_path /tmp/scgi_temp;
+
+ default_type application/octet-stream;
+ log_format {{ .Values.nginx.nginxConfig.logFormat }}
+
+ {{- if .Values.nginx.verboseLogging }}
+ access_log /dev/stderr main;
+ {{- else }}
+
+ map $status $loggable {
+ ~^[23] 0;
+ default 1;
+ }
+ access_log {{ .Values.nginx.nginxConfig.accessLogEnabled | ternary "/dev/stderr main if=$loggable;" "off;" }}
+ {{- end }}
+
+ sendfile on;
+ tcp_nopush on;
+ proxy_http_version 1.1;
+
+ {{- if .Values.nginx.nginxConfig.resolver }}
+ resolver {{ .Values.nginx.nginxConfig.resolver }};
+ {{- else }}
+ resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }};
+ {{- end }}
+
+ {{- with .Values.nginx.nginxConfig.httpSnippet }}
+ {{ . | nindent 2 }}
+ {{- end }}
+
+ # Ensure that X-Scope-OrgID is always present, default to the no_auth_tenant for backwards compatibility when multi-tenancy was turned off.
+ map $http_x_scope_orgid $ensured_x_scope_orgid {
+ default $http_x_scope_orgid;
+ "" "{{ include "mimir.noAuthTenant" . }}";
+ }
+
+ map $http_x_scope_orgid $has_multiple_orgid_headers {
+ default 0;
+ "~^.+,.+$" 1;
+ }
+
+ proxy_read_timeout 300;
+ server {
+ listen 8080;
+ listen [::]:8080;
+
+ {{- if .Values.nginx.basicAuth.enabled }}
+ auth_basic "Mimir";
+ auth_basic_user_file /etc/nginx/secrets/.htpasswd;
+ {{- end }}
+
+ if ($has_multiple_orgid_headers = 1) {
+ return 400 'Sending multiple X-Scope-OrgID headers is not allowed. Use a single header with | as separator instead.';
+ }
+
+ location = / {
+ return 200 'OK';
+ auth_basic off;
+ }
+
+ proxy_set_header X-Scope-OrgID $ensured_x_scope_orgid;
+
+ # Distributor endpoints
+ location /distributor {
+ set $distributor {{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$distributor:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location = /api/v1/push {
+ set $distributor {{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$distributor:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location /otlp/v1/metrics {
+ set $distributor {{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$distributor:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ # Alertmanager endpoints
+ location {{ template "mimir.alertmanagerHttpPrefix" . }} {
+ set $alertmanager {{ template "mimir.fullname" . }}-alertmanager-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$alertmanager:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location = /multitenant_alertmanager/status {
+ set $alertmanager {{ template "mimir.fullname" . }}-alertmanager-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$alertmanager:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location = /api/v1/alerts {
+ set $alertmanager {{ template "mimir.fullname" . }}-alertmanager-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$alertmanager:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ # Ruler endpoints
+ location {{ template "mimir.prometheusHttpPrefix" . }}/config/v1/rules {
+ set $ruler {{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$ruler:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location {{ template "mimir.prometheusHttpPrefix" . }}/api/v1/rules {
+ set $ruler {{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$ruler:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ location {{ template "mimir.prometheusHttpPrefix" . }}/api/v1/alerts {
+ set $ruler {{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$ruler:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location = /ruler/ring {
+ set $ruler {{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$ruler:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ # Rest of {{ template "mimir.prometheusHttpPrefix" . }} goes to the query frontend
+ location {{ template "mimir.prometheusHttpPrefix" . }} {
+ set $query_frontend {{ template "mimir.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$query_frontend:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ # Buildinfo endpoint can go to any component
+ location = /api/v1/status/buildinfo {
+ set $query_frontend {{ template "mimir.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$query_frontend:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ # Compactor endpoint for uploading blocks
+ location /api/v1/upload/block/ {
+ set $compactor {{ template "mimir.fullname" . }}-compactor.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$compactor:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ {{- with .Values.nginx.nginxConfig.serverSnippet }}
+ {{ . | nindent 4 }}
+ {{- end }}
+ }
+ }
+
+# -- Use either this ingress or the gateway, but not both at once.
+# If you enable this, make sure to disable the gateway's ingress.
+ingress:
+ enabled: false
+ # ingressClassName: nginx
+ annotations: {}
+ paths:
+ distributor-headless:
+ - path: /distributor
+ # -- pathType (e.g. ImplementationSpecific, Prefix, .. etc.) might also be required by some Ingress Controllers
+ # pathType: Prefix
+ - path: /api/v1/push
+ - path: /otlp/v1/metrics
+ alertmanager-headless:
+ - path: /alertmanager
+ - path: /multitenant_alertmanager/status
+ - path: /api/v1/alerts
+ ruler:
+ - path: /prometheus/config/v1/rules
+ - path: /prometheus/api/v1/rules
+ - path: /prometheus/api/v1/alerts
+ query-frontend:
+ - path: /prometheus
+ - path: /api/v1/status/buildinfo
+ compactor:
+ - path: /api/v1/upload/block/
+ hosts:
+ - mimir.example.com
+ # tls:
+ # - secretName: mimir-distributed-tls
+ # hosts:
+ # - mimir.example.com
+
+# -- A reverse proxy deployment that is meant to receive traffic for Mimir or GEM.
+# When enterprise.enabled is true the GEM gateway is deployed. Otherwise, it is an nginx.
+# Options except those under gateway.nginx apply to both versions - nginx and GEM gateway.
+gateway:
+ # -- The gateway is deployed by default for enterprise installations (enterprise.enabled=true).
+ # Toggle this to have it deployed for non-enterprise installations too.
+ enabledNonEnterprise: false
+
+ # -- Number of replicas for the Deployment
+ replicas: 1
+
+ # -- HorizontalPodAutoscaler
+ autoscaling:
+ enabled: false
+ minReplicas: 1
+ maxReplicas: 3
+ targetCPUUtilizationPercentage: 70
+ targetMemoryUtilizationPercentage: 70
+
+ # -- Deployment strategy. See `kubectl explain deployment.spec.strategy` for more,
+ # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 15%
+
+ # -- The name of the PriorityClass
+ priorityClassName: null
+ # -- Labels for Deployment Pods
+ podLabels: {}
+ # -- Annotations Deployment Pods
+ podAnnotations: {}
+ # -- PodDisruptionBudget https://kubernetes.io/docs/tasks/run-application/configure-pdb/
+ podDisruptionBudget:
+ maxUnavailable: 1
+ # -- Additional CLI args for the container
+ extraArgs: {}
+ # -- Environment variables to add to the Pods. https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/
+ env: []
+ # -- Environment variables from secrets or configmaps to add to the Pods.
+ extraEnvFrom: []
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: null
+ # -- Volumes to add to the Pods
+ extraVolumes: []
+ # -- Volume mounts to add to the Pods
+ extraVolumeMounts: []
+ # -- Additional containers to be added to the Pods.
+ extraContainers: []
+ # - name: dnsmasq
+ # image: "janeczku/go-dnsmasq:release-1.0.7"
+ # imagePullPolicy: IfNotPresent
+ # args:
+ # - --listen
+ # - "127.0.0.1:8053"
+ # - --hostsfile=/etc/hosts
+ # - --enable-search
+ # - --verbose
+
+ # -- Init containers https://kubernetes.io/docs/concepts/workloads/pods/init-containers/
+ initContainers: []
+
+ # -- SecurityContext override for gateway pods
+ securityContext: {}
+ # -- The SecurityContext for gateway containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ # -- Resource requests and limits for the container
+ resources: {}
+ # -- Grace period to allow the gateway container to shut down before it is killed
+ terminationGracePeriodSeconds: 30
+
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ # Annotations for the Deployment
+ annotations: {}
+
+ # -- Node selector for Deployment Pods
+ nodeSelector: {}
+ # -- Tolerations for Deployment Pods
+ tolerations: []
+ # -- Gateway Service configuration
+ service:
+ # -- Port on which the Service listens
+ port: 80
+ # -- Type of the Service
+ type: ClusterIP
+ # -- ClusterIP of the Service
+ clusterIP: null
+ # -- Node port if service type is NodePort
+ nodePort: null
+ # -- Load balancer IP address if service type is LoadBalancer
+ loadBalancerIP: null
+ # -- Annotations for the Service
+ annotations: {}
+ # -- Labels for the Service
+ labels: {}
+ # -- DEPRECATED Legacy compatibility port the GEM gateway service listens on, set to 'null' to disable
+ legacyPort: 8080
+ # -- Overrides the name of the Service. Useful if you are switching from the deprecated nginx or
+ # GEM gateway configuration and want to use the same in-cluster address for Mimir/GEM.
+ # By using the same name as the nginx/GEM gateway Service, Helm will not delete the Service Resource.
+ # Instead, it will update the existing one in place.
+ # If left as an empty string, a name is generated.
+ nameOverride: ""
+
+ ingress:
+ enabled: false
+ # -- Overrides the name of the Ingress. Useful if you are switching from the deprecated nginx or
+ # GEM gateway configuration and you Ingress Controller needs time to reconcile a new Ingress resource.
+ # By using the same name as the nginx/GEM gateway Ingress, Helm will not delete the Ingress Resource.
+ # Instead, it will update the existing one in place.
+ # If left as an empty string, a name is generated.
+ nameOverride: ""
+ # -- Ingress Class Name. MAY be required for Kubernetes versions >= 1.18
+ ingressClassName: ""
+ # -- Annotations for the Ingress
+ annotations: {}
+ # -- Hosts configuration for the Ingress
+ hosts:
+ # -- Passed through the `tpl` function to allow templating.
+ - host: "{{ .Release.Name }}.mimir.example.com"
+ paths:
+ - path: /
+ # -- pathType (e.g. ImplementationSpecific, Prefix, .. etc.) might also be required by some Ingress Controllers
+ # pathType: Prefix
+ # -- TLS configuration for the nginx ingress
+ tls:
+ - secretName: mimir-tls
+ # -- Hosts included in the tls certificate. Passed through the `tpl` function to allow templating.
+ hosts:
+ - "{{ .Release.Name }}.mimir.example.com"
+
+ # -- OpenShift Route configuration
+ route:
+ enabled: false
+ # -- Annotations for the Route
+ annotations: {}
+ # -- Passed through the `tpl` function to allow templating.
+ host: "{{ .Release.Name }}.mimir.example.com"
+
+ tls:
+ # -- More details about TLS configuration and termination types: https://docs.openshift.com/container-platform/3.11/architecture/networking/routes.html#secured-routes
+ # For OpenShift 4: https://docs.openshift.com/container-platform/4.11/networking/routes/secured-routes.html
+ termination: edge
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 15
+ timeoutSeconds: 1
+
+ nginx:
+ # -- Enable logging of 2xx and 3xx HTTP requests
+ verboseLogging: true
+
+ # -- Image for the nginx. pullPolicy and optional pullSecrets are set in toplevel 'image' section, not here.
+ image:
+ # -- The Docker registry for nginx image
+ registry: docker.io
+ # -- The nginx image repository
+ repository: nginxinc/nginx-unprivileged
+ # -- The nginx image tag
+ tag: 1.25-alpine
+
+ # -- Basic auth configuration
+ basicAuth:
+ # -- Enables basic authentication for nginx
+ enabled: false
+ # -- The basic auth username for nginx
+ username: null
+ # -- The basic auth password for nginx
+ password: null
+ # -- Uses the specified username and password to compute a htpasswd using Sprig's `htpasswd` function.
+ # The value is templated using `tpl`. Override this to use a custom htpasswd, e.g. in case the default causes
+ # high CPU load.
+ htpasswd: >-
+ {{ htpasswd (required "'gateway.nginx.basicAuth.username' is required" .Values.gateway.nginx.basicAuth.username) (required "'gateway.nginx.basicAuth.password' is required" .Values.gateway.nginx.basicAuth.password) }}
+ # -- Name of an existing basic auth secret to use instead of gateway.nginx.basicAuth.htpasswd. Must contain '.htpasswd' key
+ existingSecret: null
+
+ config:
+ # -- NGINX log format
+ logFormat: |-
+ main '$remote_addr - $remote_user [$time_local] $status '
+ '"$request" $body_bytes_sent "$http_referer" '
+ '"$http_user_agent" "$http_x_forwarded_for"';
+ # -- Sets the log level of the NGINX error log. One of `debug`, `info`, `notice`, `warn`, `error`, `crit`, `alert`, or `emerg`
+ errorLogLevel: error
+ # -- Enables NGINX access logs
+ accessLogEnabled: true
+ # -- Allows appending custom configuration to the server block
+ serverSnippet: ""
+ # -- Allows appending custom configuration to the http block
+ httpSnippet: ""
+ # -- Allows to set a custom resolver
+ resolver: null
+ # -- Configures whether or not NGINX bind IPv6
+ enableIPv6: true
+ # -- Config file contents for Nginx. Passed through the `tpl` function to allow templating.
+ file: |
+ worker_processes 5; ## Default: 1
+ error_log /dev/stderr {{ .Values.gateway.nginx.config.errorLogLevel }};
+ pid /tmp/nginx.pid;
+ worker_rlimit_nofile 8192;
+
+ events {
+ worker_connections 4096; ## Default: 1024
+ }
+
+ http {
+ client_body_temp_path /tmp/client_temp;
+ proxy_temp_path /tmp/proxy_temp_path;
+ fastcgi_temp_path /tmp/fastcgi_temp;
+ uwsgi_temp_path /tmp/uwsgi_temp;
+ scgi_temp_path /tmp/scgi_temp;
+
+ default_type application/octet-stream;
+ log_format {{ .Values.gateway.nginx.config.logFormat }}
+
+ {{- if .Values.gateway.nginx.verboseLogging }}
+ access_log /dev/stderr main;
+ {{- else }}
+
+ map $status $loggable {
+ ~^[23] 0;
+ default 1;
+ }
+ access_log {{ .Values.gateway.nginx.config.accessLogEnabled | ternary "/dev/stderr main if=$loggable;" "off;" }}
+ {{- end }}
+
+ sendfile on;
+ tcp_nopush on;
+ proxy_http_version 1.1;
+
+ {{- if .Values.gateway.nginx.config.resolver }}
+ resolver {{ .Values.gateway.nginx.config.resolver }};
+ {{- else }}
+ resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }};
+ {{- end }}
+
+ {{- with .Values.gateway.nginx.config.httpSnippet }}
+ {{ . | nindent 2 }}
+ {{- end }}
+
+ # Ensure that X-Scope-OrgID is always present, default to the no_auth_tenant for backwards compatibility when multi-tenancy was turned off.
+ map $http_x_scope_orgid $ensured_x_scope_orgid {
+ default $http_x_scope_orgid;
+ "" "{{ include "mimir.noAuthTenant" . }}";
+ }
+
+ map $http_x_scope_orgid $has_multiple_orgid_headers {
+ default 0;
+ "~^.+,.+$" 1;
+ }
+
+ proxy_read_timeout 300;
+ server {
+ listen {{ include "mimir.serverHttpListenPort" . }};
+ {{- if .Values.gateway.nginx.config.enableIPv6 }}
+ listen [::]:{{ include "mimir.serverHttpListenPort" . }};
+ {{- end }}
+
+ {{- if .Values.gateway.nginx.basicAuth.enabled }}
+ auth_basic "Mimir";
+ auth_basic_user_file /etc/nginx/secrets/.htpasswd;
+ {{- end }}
+
+ if ($has_multiple_orgid_headers = 1) {
+ return 400 'Sending multiple X-Scope-OrgID headers is not allowed. Use a single header with | as separator instead.';
+ }
+
+ location = / {
+ return 200 'OK';
+ auth_basic off;
+ }
+
+ location = /ready {
+ return 200 'OK';
+ auth_basic off;
+ }
+
+ proxy_set_header X-Scope-OrgID $ensured_x_scope_orgid;
+
+ # Distributor endpoints
+ location /distributor {
+ set $distributor {{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$distributor:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location = /api/v1/push {
+ set $distributor {{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$distributor:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location /otlp/v1/metrics {
+ set $distributor {{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$distributor:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ # Alertmanager endpoints
+ location {{ template "mimir.alertmanagerHttpPrefix" . }} {
+ set $alertmanager {{ template "mimir.fullname" . }}-alertmanager-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$alertmanager:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location = /multitenant_alertmanager/status {
+ set $alertmanager {{ template "mimir.fullname" . }}-alertmanager-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$alertmanager:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location = /api/v1/alerts {
+ set $alertmanager {{ template "mimir.fullname" . }}-alertmanager-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$alertmanager:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ # Ruler endpoints
+ location {{ template "mimir.prometheusHttpPrefix" . }}/config/v1/rules {
+ set $ruler {{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$ruler:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location {{ template "mimir.prometheusHttpPrefix" . }}/api/v1/rules {
+ set $ruler {{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$ruler:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ location {{ template "mimir.prometheusHttpPrefix" . }}/api/v1/alerts {
+ set $ruler {{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$ruler:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+ location = /ruler/ring {
+ set $ruler {{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$ruler:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ # Rest of {{ template "mimir.prometheusHttpPrefix" . }} goes to the query frontend
+ location {{ template "mimir.prometheusHttpPrefix" . }} {
+ set $query_frontend {{ template "mimir.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$query_frontend:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ # Buildinfo endpoint can go to any component
+ location = /api/v1/status/buildinfo {
+ set $query_frontend {{ template "mimir.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$query_frontend:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ # Compactor endpoint for uploading blocks
+ location /api/v1/upload/block/ {
+ set $compactor {{ template "mimir.fullname" . }}-compactor.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }};
+ proxy_pass http://$compactor:{{ include "mimir.serverHttpListenPort" . }}$request_uri;
+ }
+
+ {{- with .Values.gateway.nginx.config.serverSnippet }}
+ {{ . | nindent 4 }}
+ {{- end }}
+ }
+ }
+
+metaMonitoring:
+ # Dashboard configuration for deploying Grafana dashboards for Mimir
+ dashboards:
+ # -- If enabled, Grafana dashboards are deployed
+ enabled: false
+ # -- Annotations to add to the Grafana dashboard ConfigMap
+ annotations:
+ k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
+ # -- Labels to add to the Grafana dashboard ConfigMap
+ labels:
+ grafana_dashboard: "1"
+
+ # ServiceMonitor configuration for monitoring Kubernetes Services with Prometheus Operator and/or Grafana Agent
+ serviceMonitor:
+ # -- If enabled, ServiceMonitor resources for Prometheus Operator are created
+ enabled: false
+ # -- To disable setting a 'cluster' label in metrics, set to 'null'.
+ # To overwrite the 'cluster' label with your own value, set to a non-empty string.
+ # Keep empty string "" to have the default value in the 'cluster' label, which is the helm release name for Mimir and the actual cluster name for Enterprise Metrics.
+ clusterLabel: ""
+ # -- Alternative namespace for ServiceMonitor resources
+ # If left unset, the default is to install the ServiceMonitor resources in the namespace where the chart is installed, i.e. the namespace specified for the helm command.
+ namespace: null
+ # -- Namespace selector for ServiceMonitor resources
+ # If left unset, the default is to select the namespace where the chart is installed, i.e. the namespace specified for the helm command.
+ namespaceSelector: null
+ # -- ServiceMonitor annotations
+ annotations: {}
+ # -- Additional ServiceMonitor labels
+ labels: {}
+ # -- ServiceMonitor scrape interval
+ interval: null
+ # -- ServiceMonitor scrape timeout in Go duration format (e.g. 15s)
+ scrapeTimeout: null
+ # -- ServiceMonitor relabel configs to apply to targets before scraping
+ # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig
+ relabelings: []
+ # -- ServiceMonitor metric relabel configs to apply to samples before ingestion
+ # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig
+ metricRelabelings: []
+ # -- ServiceMonitor will use http by default, but you can pick https as well
+ scheme: http
+ # -- ServiceMonitor will use these tlsConfig settings to make the health check requests
+ tlsConfig: null
+
+ # Rules for the Prometheus Operator
+ prometheusRule:
+ # -- If enabled, a PrometheusRule resource for Prometheus Operator is created
+ enabled: false
+ # -- Create standard Mimir alerts in Prometheus Operator via a PrometheusRule CRD
+ mimirAlerts: false
+ # -- Create standard Mimir recording rules in Prometheus Operator via a PrometheusRule CRD
+ mimirRules: false
+ # -- PrometheusRule annotations
+ annotations: {}
+ # -- Additional PrometheusRule labels. To find out what your Prometheus operator expects,
+ # see the Prometheus object and field spec.ruleSelector
+ labels: {}
+ # -- prometheusRule namespace. This should be the namespace where the Prometheus Operator is installed,
+ # unless the Prometheus Operator is set up to look for rules outside its namespace
+ namespace: null
+ # -- Contents of Prometheus rules file
+ groups: []
+ # - name: mimir_api_1
+ # rules:
+ # - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+ # by (le, cluster, job))
+ # record: cluster_job:cortex_request_duration_seconds:99quantile
+ # - expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
+ # by (le, cluster, job))
+ # record: cluster_job:cortex_request_duration_seconds:50quantile
+ # - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[1m]))
+ # by (cluster, job)
+ # record: cluster_job:cortex_request_duration_seconds:avg
+ # - expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job)
+ # record: cluster_job:cortex_request_duration_seconds_bucket:sum_rate
+ # - expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job)
+ # record: cluster_job:cortex_request_duration_seconds_sum:sum_rate
+ # - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)
+ # record: cluster_job:cortex_request_duration_seconds_count:sum_rate
+
+ # metaMonitoringAgent configures the built in Grafana Agent that can scrape metrics and logs and send them to a local or remote destination
+ grafanaAgent:
+ # -- Controls whether to create PodLogs, MetricsInstance, LogsInstance, and GrafanaAgent CRs to scrape the
+ # ServiceMonitors of the chart and ship metrics and logs to the remote endpoints below.
+ # Note that you need to configure serviceMonitor in order to have some metrics available.
+ enabled: false
+
+ # -- Controls the image repository and tag for config-reloader and grafana-agent containers in the meta-monitoring
+ # StatefulSet and DaemonSet created by the grafana-agent-operator. You can define one or both sections under imageRepo.
+ # If a section is defined, you must pass repo, image and tag keys.
+ imageRepo:
+ # configReloader:
+ # repo: quay.io
+ # image: prometheus-operator/prometheus-config-reloader
+ # tag: v0.47.0
+ # grafanaAgent:
+ # repo: docker.io
+ # image: grafana/agent
+ # tag: v0.29.0
+
+ # -- Controls whether to install the Grafana Agent Operator and its CRDs.
+ # Note that helm will not install CRDs if this flag is enabled during an upgrade.
+ # In that case install the CRDs manually from https://github.com/grafana/agent/tree/main/operations/agent-static-operator/crds
+ installOperator: false
+
+ logs:
+ # -- Controls whether to create resources PodLogs and LogsInstance resources
+ enabled: true
+
+ # -- Default destination for logs. The config here is translated to Promtail client
+ # configuration to write logs to this Loki-compatible remote. Optional.
+ remote:
+ # -- Full URL for Loki push endpoint. Usually ends in /loki/api/v1/push
+ url: ""
+
+ auth:
+ # -- Used to set X-Scope-OrgID header on requests. Usually not used in combination with username and password.
+ tenantId: ""
+
+ # -- Basic authentication username. Optional.
+ username: ""
+
+ # -- The value under key passwordSecretKey in this secret will be used as the basic authentication password. Required only if passwordSecretKey is set.
+ passwordSecretName: ""
+ # -- The value under this key in passwordSecretName will be used as the basic authentication password. Required only if passwordSecretName is set.
+ passwordSecretKey: ""
+
+ # -- Client configurations for the LogsInstance that will scrape Mimir pods. Follows the format of .remote.
+ additionalClientConfigs: []
+
+ metrics:
+ # -- Controls whether to create MetricsInstance resources and ServiceMonitor resources for scraping Kubernetes (when .scrapeK8s.enabled=true).
+ enabled: true
+
+ # -- Default destination for metrics. The config here is translated to remote_write
+ # configuration to push metrics to this Prometheus-compatible remote. Optional.
+ # Note that you need to configure serviceMonitor in order to have some metrics available.
+ #
+ # If you leave the metaMonitoring.grafanaAgent.metrics.remote.url field empty,
+ # then the chart automatically fills in the address of the GEM gateway Service
+ # or the Mimir NGINX Service.
+ #
+ # If you have deployed Mimir, and metaMonitoring.grafanaAgent.metrics.remote.url is not set,
+ # then the metamonitoring metrics are be sent to the Mimir cluster.
+ # You can query these metrics using the HTTP header X-Scope-OrgID: metamonitoring
+ #
+ # If you have deployed GEM, then there are two cases:
+ # * If are using the 'trust' authentication type (mimir.structuredConfig.auth.type: trust),
+ # then the same instructions apply as for Mimir.
+ #
+ # * If you are using the enterprise authentication type (mimir.structuredConfig.auth.type=enterprise, which is also the default when enterprise.enabled=true),
+ # then you also need to provide a Secret with the authentication token for the tenant.
+ # The token should be to an access policy with metrics:read scope.
+ # To set up the Secret, refer to https://grafana.com/docs/helm-charts/mimir-distributed/latest/run-production-environment-with-helm/monitor-system-health/
+ # Assuming you are using the GEM authentication model, the Helm chart values should look like the following example.
+ #
+ # remote:
+ # auth:
+ # username: metamonitoring
+ # passwordSecretName: gem-tokens
+ # passwordSecretKey: metamonitoring
+ remote:
+ # -- Full URL for Prometheus remote-write. Usually ends in /push.
+ # If you leave the url field empty, then the chart automatically fills in the
+ # address of the GEM gateway Service or the Mimir NGINX Service.
+ url: ""
+
+ # -- Used to add HTTP headers to remote-write requests.
+ headers: {}
+ auth:
+ # -- Basic authentication username. Optional.
+ username: ""
+
+ # -- The value under key passwordSecretKey in this secret will be used as the basic authentication password. Required only if passwordSecretKey is set.
+ passwordSecretName: ""
+ # -- The value under this key in passwordSecretName will be used as the basic authentication password. Required only if passwordSecretName is set.
+ passwordSecretKey: ""
+
+ # -- Additional remote-write for the MetricsInstance that will scrape Mimir pods. Follows the format of .remote.
+ additionalRemoteWriteConfigs: []
+
+ scrapeK8s:
+ # -- When grafanaAgent.enabled and serviceMonitor.enabled, controls whether to create ServiceMonitors CRs
+ # for cadvisor, kubelet, and kube-state-metrics. The scraped metrics are reduced to those pertaining to
+ # Mimir pods only.
+ enabled: true
+
+ # -- Controls service discovery of kube-state-metrics.
+ kubeStateMetrics:
+ namespace: kube-system
+ labelSelectors:
+ app.kubernetes.io/name: kube-state-metrics
+ service:
+ port: http-metrics
+
+ # -- The scrape interval for all ServiceMonitors.
+ scrapeInterval: 60s
+
+ # -- Sets the namespace of the resources. Leave empty or unset to use the same namespace as the Helm release.
+ namespace: ""
+
+ # -- Labels to add to all monitoring.grafana.com custom resources.
+ # Does not affect the ServiceMonitors for kubernetes metrics; use serviceMonitor.labels for that.
+ labels: {}
+
+ # -- Annotations to add to all monitoring.grafana.com custom resources.
+ # Does not affect the ServiceMonitors for kubernetes metrics; use serviceMonitor.annotations for that.
+ annotations: {}
+
+ # -- SecurityContext of Grafana Agent pods. This is different from the SecurityContext that the operator pod runs with.
+ # The operator pod SecurityContext is configured in the grafana-agent-operator.podSecurityContext value.
+ # As of mimir-distributed 4.0.0 the Agent DaemonSet that collects logs needs to run as root and be able to access the
+ # pod logs on each host. Because of that the agent subchart is incompatible with the PodSecurityPolicy of the
+ # mimir-distributed chart and with the Restricted policy of Pod Security Standards https://kubernetes.io/docs/concepts/security/pod-security-standards/
+ podSecurityContext:
+ # fsGroup: 10001
+ # runAsGroup: 10001
+ # runAsNonRoot: true
+ # runAsUser: 10001
+ # seccompProfile:
+ # type: RuntimeDefault
+
+ # -- SecurityContext of Grafana Agent containers. This is different from the SecurityContext that the operator container runs with.
+ # As of mimir-distributed 4.0.0 the agent subchart needs to have root file system write access so that the Agent pods can write temporary files where.
+ # This makes the subchart incompatible with the PodSecurityPolicy of the mimir-distributed chart.
+ containerSecurityContext:
+ # allowPrivilegeEscalation: false
+ # runAsUser: 10001
+ # capabilities:
+ # drop: [ALL]
+
+# -- Values exposed by the [Grafana agent-operator chart](https://github.com/grafana/helm-charts/blob/main/charts/agent-operator/values.yaml)
+grafana-agent-operator:
+ podSecurityContext:
+ fsGroup: 10001
+ runAsGroup: 10001
+ runAsNonRoot: true
+ runAsUser: 10001
+ seccompProfile:
+ type: RuntimeDefault
+
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+##############################################################################
+# The values in and after the `enterprise:` key configure the enterprise features
+enterprise:
+ # Enable enterprise features. License must be provided, nginx gateway is not installed, instead
+ # the enterprise gateway is used.
+ enabled: false
+
+ # Whether to generate pre-2.0 Grafana Enterprise Metrics resource labels and selectors, or generate new Kubernetes standard selectors.
+ # Rolling upgrade from version 1.7.x without downtime requires this setting to be true. Fresh installation or upgrade with downtime can set
+ # it to false.
+ legacyLabels: false
+
+ image:
+ # -- Grafana Enterprise Metrics container image repository. Note: for Grafana Mimir use the value 'image.repository'
+ repository: grafana/enterprise-metrics
+ # -- Grafana Enterprise Metrics container image tag. Note: for Grafana Mimir use the value 'image.tag'
+ tag: v2.12.0
+ # Note: pullPolicy and optional pullSecrets are set in toplevel 'image' section, not here
+
+# In order to use Grafana Enterprise Metrics features, you will need to provide the contents of your Grafana Enterprise Metrics
+# license, either by providing the contents of the license.jwt, or the name Kubernetes Secret that contains your license.jwt.
+# To set the license contents, use the flag `--set-file 'license.contents=./license.jwt'`
+# To use your own Kubernetes Secret, `--set license.external=true`.
+license:
+ contents: "NOTAVALIDLICENSE"
+ external: false
+ secretName: '{{ include "mimir.resourceName" (dict "ctx" . "component" "license") }}'
+
+# Settings for the initial admin(istrator) token generator job. Can only be enabled if
+# enterprise.enabled is true - requires license.
+tokengenJob:
+ enable: true
+ extraArgs: {}
+ env: []
+ extraEnvFrom: []
+ annotations: {}
+ initContainers: []
+
+ # -- SecurityContext override for tokengenjob pods
+ securityContext: {}
+
+ # -- The SecurityContext for tokengenjob containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ # -- The name of the PriorityClass for tokenjobgen pods
+ priorityClassName: null
+
+# Settings for the admin_api service providing authentication and authorization service.
+# Can only be enabled if enterprise.enabled is true - requires license.
+admin_api:
+ replicas: 1
+
+ annotations: {}
+ service:
+ annotations: {}
+ labels: {}
+
+ initContainers: []
+
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 15%
+
+ podLabels: {}
+ podAnnotations: {}
+
+ nodeSelector: {}
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+
+ # Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+
+ # -- The name of the PriorityClass for admin-api pods
+ priorityClassName: null
+
+ # -- SecurityContext override for admin-api pods
+ securityContext: {}
+
+ # -- The SecurityContext for admin_api containers
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ extraArgs: {}
+
+ persistence:
+ subPath:
+
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+ resources:
+ requests:
+ cpu: 10m
+ memory: 32Mi
+
+ terminationGracePeriodSeconds: 60
+
+ tolerations: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+ env: []
+ extraEnvFrom: []
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: null
+
+# -- Cache for admin bucket.
+# If this is disabled, in-memory cache will be set by default.
+# You can use Redis too for cache and set the configuration via structuredConfig.
+# See GEM documentation for Redis configuration option.
+admin-cache:
+ # -- Specifies whether admin-cache using memcached should be enabled
+ enabled: false
+
+ # -- Total number of admin-cache replicas
+ replicas: 1
+
+ # -- Port of the admin-cache service
+ port: 11211
+
+ # -- Amount of memory allocated to admin-cache for object storage (in MB).
+ allocatedMemory: 64
+
+ # -- Maximum item memory for admin-cache (in MB).
+ maxItemMemory: 1
+
+ # -- Maximum number of connections allowed
+ connectionLimit: 16384
+
+ # -- Extra init containers for admin-cache pods
+ initContainers: []
+
+ # -- Annotations for the admin-cache pods
+ annotations: {}
+ # -- Node selector for admin-cache pods
+ nodeSelector: {}
+ # -- Affinity for admin-cache pods
+ affinity: {}
+
+ # -- topologySpreadConstraints allows to customize the default topologySpreadConstraints. This can be either a single dict as shown below or a slice of topologySpreadConstraints.
+ # labelSelector is taken from the constraint itself (if it exists) or is generated by the chart using the same selectors as for services.
+ topologySpreadConstraints: {}
+ # maxSkew: 1
+ # topologyKey: kubernetes.io/hostname
+ # whenUnsatisfiable: ScheduleAnyway
+
+ # -- Tolerations for admin-cache pods
+ tolerations: []
+ # -- Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+ # -- The name of the PriorityClass for admin-cache pods
+ priorityClassName: null
+ # -- Labels for admin-cache pods
+ podLabels: {}
+ # -- Annotations for admin-cache pods
+ podAnnotations: {}
+ # -- Management policy for admin-cache pods
+ podManagementPolicy: Parallel
+ # -- Grace period to allow the admin-cache to shutdown before it is killed
+ terminationGracePeriodSeconds: 60
+
+ # -- Stateful admin-cache strategy
+ statefulStrategy:
+ type: RollingUpdate
+
+ # -- Additional CLI args for admin-cache
+ extraArgs: {}
+
+ # -- Additional containers to be added to the admin-cache pod.
+ extraContainers: []
+
+ # -- Resource requests and limits for the admin-cache
+ # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+ resources: null
+
+ # -- Service annotations and labels
+ service:
+ annotations: {}
+ labels: {}
+
+graphite:
+ # -- If true, enables graphite querier and graphite write proxy functionality.
+ # Read more in https://grafana.com/docs/enterprise-metrics/latest/graphite/
+ enabled: false
+
+ querier:
+ # Setting it to null will produce a deployment without replicas set, allowing you to use autoscaling with the deployment
+ replicas: 2
+
+ schemasConfiguration:
+ storageSchemas: |-
+ [default]
+ pattern = .*
+ intervals = 0:1s
+ retentions = 10s:8d,10min:1y
+ storageAggregations: |-
+ [default]
+ aggregationMethod = avg
+ pattern = .*
+ xFilesFactor = 0.1
+
+ service:
+ annotations: {}
+ labels: {}
+
+ # -- Resources for graphite-querier pods
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+
+ # -- Additional graphite-querier container arguments, e.g. log level (debug, info, warn, error)
+ extraArgs: {}
+ # -- The name of the PriorityClass for graphite-querier pods
+ priorityClassName: null
+ # -- Labels for graphite-querier pods
+ podLabels: {}
+ # -- Annotations for graphite-querier pods
+ podAnnotations: {}
+
+ nodeSelector: {}
+ affinity:
+ podAntiAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ podAffinityTerm:
+ labelSelector:
+ matchExpressions:
+ - key: app.kubernetes.io/component
+ operator: In
+ values:
+ - graphite-querier
+ topologyKey: "kubernetes.io/hostname"
+
+ livenessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+ annotations: {}
+ persistence:
+ subPath:
+
+ # -- SecurityContext override for graphite querier pods
+ securityContext: {}
+
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 15%
+
+ terminationGracePeriodSeconds: 180
+
+ env: []
+ extraEnvFrom: []
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: null
+ tolerations: []
+ podDisruptionBudget:
+ maxUnavailable: 1
+ initContainers: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+
+ write_proxy:
+ replicas: 2
+
+ service:
+ annotations: {}
+ labels: {}
+
+ # -- Resources for graphite-write-proxy pods
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+
+ # -- Additional graphite-write-proxy container arguments, e.g. log level (debug, info, warn, error)
+ extraArgs: {}
+ # -- The name of the PriorityClass for graphite-write-proxy pods
+ priorityClassName: null
+ # -- Labels for graphite-write-proxy pods
+ podLabels: {}
+ # -- Annotations for graphite-write-proxy pods
+ podAnnotations: {}
+ # -- Node selector for graphite-write-proxy pods
+ nodeSelector: {}
+ # -- Affinity for graphite-write-proxy pods
+ affinity:
+ podAntiAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ podAffinityTerm:
+ labelSelector:
+ matchExpressions:
+ - key: app.kubernetes.io/component
+ operator: In
+ values:
+ - graphite-write-proxy
+ topologyKey: "kubernetes.io/hostname"
+
+ livenessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ initialDelaySeconds: 45
+
+ annotations: {}
+ persistence:
+ subPath:
+
+ # -- SecurityContext override for graphite write-proxy pods
+ securityContext: {}
+
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: [ALL]
+
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 15%
+
+ # -- Grace period to allow the graphite-write-proxy to shutdown before it is killed
+ terminationGracePeriodSeconds: 180
+ env: []
+ extraEnvFrom: []
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: null
+ tolerations: []
+ podDisruptionBudget:
+ maxUnavailable: 1
+ initContainers: []
+ extraContainers: []
+ extraVolumes: []
+ extraVolumeMounts: []
+
+# Graphite's aggregation cache. If you want to know more about it please check
+# https://grafana.com/docs/enterprise-metrics/latest/graphite/graphite_querier/#aggregation-cache
+gr-aggr-cache:
+ # -- Specifies whether memcached based graphite-aggregation-cache should be enabled. Note that the cache will only appear if graphite is also enabled.
+ enabled: true
+
+ # -- Total number of graphite-aggregation-cache replicas
+ replicas: 1
+
+ # -- Port of the graphite-aggregation-cache service
+ port: 11211
+
+ # -- Amount of memory allocated to graphite-aggregation-cache for object storage (in MB).
+ allocatedMemory: 8192
+
+ # -- Maximum item memory for graphite-aggregation-cache (in MB).
+ maxItemMemory: 1
+
+ # -- Maximum number of connections allowed
+ connectionLimit: 16384
+
+ # -- Extra init containers for graphite-aggregation-cache pods
+ initContainers: []
+
+ # -- Annotations for the graphite-aggregation-cache pods
+ annotations: {}
+ # -- Node selector for graphite-aggregation-cache pods
+ nodeSelector: {}
+ # -- Affinity for graphite-aggregation-cache pods
+ affinity: {}
+ # -- Tolerations for graphite-aggregation-cache pods
+ tolerations: []
+ # -- Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+ # -- The name of the PriorityClass for graphite-aggregation-cache pods
+ priorityClassName: null
+ # -- Labels for graphite-aggregation-cache pods
+ podLabels: {}
+ # -- Annotations for graphite-aggregation-cache pods
+ podAnnotations: {}
+ # -- Management policy for graphite-aggregation-cache pods
+ podManagementPolicy: Parallel
+ # -- Grace period to allow the graphite-aggregation-cache to shutdown before it is killed
+ terminationGracePeriodSeconds: 60
+
+ # -- Stateful graphite-aggregation-cache strategy
+ statefulStrategy:
+ type: RollingUpdate
+
+ # -- Additional CLI args for graphite-aggregation-cache
+ extraArgs: {}
+
+ # -- Additional containers to be added to the graphite-aggregation-cache pod.
+ extraContainers: []
+
+ # -- Resource requests and limits for the graphite-aggregation-cache
+ # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+ resources: null
+
+ # -- Service annotations and labels
+ service:
+ annotations: {}
+ labels: {}
+
+# Graphite's metric name cache. If you want to know more about it please check
+# https://grafana.com/docs/enterprise-metrics/latest/graphite/graphite_querier/#metric-name-cache
+gr-metricname-cache:
+ # -- Specifies whether memcached based graphite-metric-name-cache should be enabled. Note that the cache will only appear if graphite is also enabled.
+ enabled: true
+
+ # -- Total number of graphite-metric-name-cache replicas
+ replicas: 1
+
+ # -- Port of the graphite-metric-name-cache service
+ port: 11211
+
+ # -- Amount of memory allocated to graphite-metric-name-cache for object storage (in MB).
+ allocatedMemory: 8192
+
+ # -- Maximum item memory for graphite-metric-name-cache (in MB).
+ maxItemMemory: 1
+
+ # -- Maximum number of connections allowed
+ connectionLimit: 16384
+
+ # -- Extra init containers for graphite-metric-name-cache pods
+ initContainers: []
+
+ # -- Annotations for the graphite-metric-name-cache pods
+ annotations: {}
+ # -- Node selector for graphite-metric-name-cache pods
+ nodeSelector: {}
+ # -- Affinity for graphite-metric-name-cache pods
+ affinity: {}
+ # -- Tolerations for graphite-metric-name-cache pods
+ tolerations: []
+ # -- Pod Disruption Budget
+ podDisruptionBudget:
+ maxUnavailable: 1
+ # -- The name of the PriorityClass for graphite-metric-name-cache pods
+ priorityClassName: null
+ # -- Labels for graphite-metric-name-cache pods
+ podLabels: {}
+ # -- Annotations for graphite-metric-name-cache pods
+ podAnnotations: {}
+ # -- Management policy for graphite-metric-name-cache pods
+ podManagementPolicy: Parallel
+ # -- Grace period to allow the graphite-metric-name-cache to shutdown before it is killed
+ terminationGracePeriodSeconds: 60
+
+ # -- Stateful graphite-metric-name-cache strategy
+ statefulStrategy:
+ type: RollingUpdate
+
+ # -- Additional CLI args for graphite-metric-name-cache
+ extraArgs: {}
+
+ # -- Additional containers to be added to the graphite-metric-name-cache pod.
+ extraContainers: []
+
+ # -- Resource requests and limits for the graphite-metric-name-cache
+ # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+ resources: null
+
+ # -- Service annotations and labels
+ service:
+ annotations: {}
+ labels: {}
+
+# -- Settings for the smoke-test job. This is meant to run as a Helm test hook
+# (`helm test RELEASE`) after installing the chart. It quickly verifies
+# that writing and reading metrics works. Currently not supported for
+# installations using GEM token-based authentication.
+smoke_test:
+ image:
+ repository: grafana/mimir-continuous-test
+ tag: 2.12.0
+ pullPolicy: IfNotPresent
+ tenantId: ""
+ extraArgs: {}
+ env: []
+ extraEnvFrom: []
+ annotations: {}
+ initContainers: []
+ # -- The name of the PriorityClass for smoke-test pods
+ priorityClassName: null
+
+# -- Settings for mimir-continuous-test.
+# This continuously writes and reads metrics from Mimir.
+# https://grafana.com/docs/mimir/latest/manage/tools/mimir-continuous-test/
+continuous_test:
+ enabled: false
+ # -- Number of replicas to start of continuous test
+ replicas: 1
+ image:
+ repository: grafana/mimir-continuous-test
+ tag: 2.12.0
+ pullPolicy: IfNotPresent
+ # Note: optional pullSecrets are set in toplevel 'image' section, not here
+
+ # -- Authentication settings of continuous test
+ auth:
+ # -- Type of authentication to use (tenantId, basicAuth, bearerToken)
+ type: tenantId
+ # -- The tenant to use for tenantId or basicAuth authentication type
+ # In case of tenantId authentication, it is injected as the X-Scope-OrgID header on requests.
+ # In case of basicAuth, it is set as the username.
+ tenant: "mimir-continuous-test"
+ # -- Password for basicAuth auth (note: can be environment variable from secret attached in extraEnvFrom, e.g. $(PASSWORD))
+ # For GEM, it should contain an access token created for an access policy that allows `metrics:read` and `metrics:write` for the tenant.
+ password: null
+ # -- Bearer token for bearerToken auth (note: can be environment variable from secret attached in extraEnvFrom, e.g. $(TOKEN))
+ bearerToken: null
+ # -- The maximum number of series to write in a single request.
+ numSeries: 1000
+ # -- How far back in the past metrics can be queried at most.
+ maxQueryAge: "48h"
+ # -- Interval between test runs
+ runInterval: "5m"
+
+ # -- Pod affinity settings for the continuous test replicas
+ affinity: {}
+ # -- Annotations for the continuous test Deployment
+ annotations: {}
+ # -- The SecurityContext for continuous test containers
+ containerSecurityContext:
+ readOnlyRootFilesystem: true
+ # -- Extra environment variables for continuous test containers
+ env: []
+ # -- Extra command line arguments for the continuous test container
+ extraArgs: {}
+ # -- Extra environment from secret/configmap for continuous test containers
+ extraEnvFrom: []
+ # -- Jaeger reporter queue size
+ # Set to 'null' to use the Jaeger client's default value
+ jaegerReporterMaxQueueSize: null
+ # -- Extra volumes for the continuous test container
+ extraVolumes: []
+ # -- Extra volume mounts for the continuous test container
+ extraVolumeMounts: []
+ # -- Extra containers for the continuous test Deployment
+ extraContainers: []
+ # -- Extra initContainers for the continuous test Deployment
+ initContainers: []
+ # -- Nodeselector of continuous test replicas
+ nodeSelector: {}
+ # -- The name of the PriorityClass for continuous test pods
+ priorityClassName: null
+ # -- Kubernetes resource requests and limits for continuous test
+ resources:
+ limits:
+ memory: 1Gi
+ requests:
+ cpu: "1"
+ memory: 512Mi
+ # -- Security context for the continuous test Deployment
+ securityContext: {}
+ # -- Service for monitoring continuous test
+ service:
+ annotations: {}
+ labels: {}
+ # -- Upgrade strategy for the continuous test Deployment
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 0
+ maxUnavailable: 1
+
+ tolerations: []
+ terminationGracePeriodSeconds: 30
+
+# -- Add dynamic manifests via values. Example:
+# extraObjects:
+# - kind: ConfigMap
+# apiVersion: v1
+# metadata:
+# name: extra-cm-{{ .Release.Name }}
+# data: |
+# extra.yml: "does-my-install-need-extra-info: true"
+extraObjects: []
diff --git a/observability/lgtm-distributed/charts/oncall/.helmignore b/observability/lgtm-distributed/charts/oncall/.helmignore
new file mode 100644
index 0000000..fd6e5fd
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/.helmignore
@@ -0,0 +1,26 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+
+# exclude helm unit tests
+tests/
diff --git a/observability/lgtm-distributed/charts/oncall/Chart.lock b/observability/lgtm-distributed/charts/oncall/Chart.lock
new file mode 100644
index 0000000..e550990
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/Chart.lock
@@ -0,0 +1,27 @@
+dependencies:
+- name: cert-manager
+ repository: https://charts.jetstack.io
+ version: v1.8.0
+- name: mariadb
+ repository: https://charts.bitnami.com/bitnami
+ version: 12.2.5
+- name: postgresql
+ repository: https://charts.bitnami.com/bitnami
+ version: 11.9.10
+- name: rabbitmq
+ repository: https://charts.bitnami.com/bitnami
+ version: 12.0.0
+- name: redis
+ repository: https://charts.bitnami.com/bitnami
+ version: 16.13.2
+- name: grafana
+ repository: https://grafana.github.io/helm-charts
+ version: 6.57.1
+- name: ingress-nginx
+ repository: https://kubernetes.github.io/ingress-nginx
+ version: 4.1.4
+- name: prometheus
+ repository: https://prometheus-community.github.io/helm-charts
+ version: 25.8.2
+digest: sha256:edc9fef449a694cd319135e37ac84f8247ac9ad0c48ac86099dae4e428beb7b7
+generated: "2024-06-20T17:56:01.397630734Z"
diff --git a/observability/lgtm-distributed/charts/oncall/Chart.yaml b/observability/lgtm-distributed/charts/oncall/Chart.yaml
new file mode 100644
index 0000000..2646403
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/Chart.yaml
@@ -0,0 +1,39 @@
+apiVersion: v2
+appVersion: v1.7.2
+dependencies:
+- condition: cert-manager.enabled
+ name: cert-manager
+ repository: https://charts.jetstack.io
+ version: v1.8.0
+- condition: mariadb.enabled
+ name: mariadb
+ repository: https://charts.bitnami.com/bitnami
+ version: 12.2.5
+- condition: postgresql.enabled
+ name: postgresql
+ repository: https://charts.bitnami.com/bitnami
+ version: 11.9.10
+- condition: rabbitmq.enabled
+ name: rabbitmq
+ repository: https://charts.bitnami.com/bitnami
+ version: 12.0.0
+- condition: redis.enabled
+ name: redis
+ repository: https://charts.bitnami.com/bitnami
+ version: 16.13.2
+- condition: grafana.enabled
+ name: grafana
+ repository: https://grafana.github.io/helm-charts
+ version: 6.57.1
+- condition: ingress-nginx.enabled
+ name: ingress-nginx
+ repository: https://kubernetes.github.io/ingress-nginx
+ version: 4.1.4
+- condition: prometheus.enabled
+ name: prometheus
+ repository: https://prometheus-community.github.io/helm-charts
+ version: 25.8.2
+description: Developer-friendly incident response with brilliant Slack integration
+name: oncall
+type: application
+version: 1.7.2
diff --git a/observability/lgtm-distributed/charts/oncall/README.md b/observability/lgtm-distributed/charts/oncall/README.md
new file mode 100644
index 0000000..993f064
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/README.md
@@ -0,0 +1,431 @@
+# Grafana OnCall Helm Chart
+
+This Grafana OnCall Chart is the best way to operate Grafana OnCall on Kubernetes.
+It will deploy Grafana OnCall engine and celery workers, along with RabbitMQ cluster, Redis Cluster, and MySQL 5.7 database.
+It will also deploy cert manager and nginx ingress controller, as Grafana OnCall backend might need to be externally available
+to receive alerts from other monitoring systems. Grafana OnCall engine acts as a backend and can be connected to the
+Grafana frontend plugin named Grafana OnCall.
+Architecture diagram can be found [here](https://raw.githubusercontent.com/grafana/oncall/dev/docs/img/architecture_diagram.png)
+
+## Production usage
+
+**Default helm chart configuration is not intended for production.**
+The helm chart includes all the services into a single release, which is not recommended for production usage.
+It is recommended to run stateful services such as MySQL and RabbitMQ separately from this release or use managed
+PaaS solutions. It will significantly reduce the overhead of managing them.
+Here are the instructions on how to set up your own [ingress](#set-up-external-access), [MySQL](#connect-external-mysql),
+[RabbitMQ](#connect-external-rabbitmq), [Redis](#connect-external-redis)
+
+### Cluster requirements
+
+- ensure you can run x86-64/amd64 workloads. arm64 architecture is currently not supported
+- kubernetes version 1.25+ is not supported, if cert-manager is enabled
+
+## Install
+
+### Prepare the repo
+
+```bash
+# Add the repository
+helm repo add grafana https://grafana.github.io/helm-charts
+helm repo update
+```
+
+### Installing the helm chart
+
+```bash
+# Install the chart
+helm install \
+ --wait \
+ --set base_url=example.com \
+ --set grafana."grafana\.ini".server.domain=example.com \
+ release-oncall \
+ grafana/oncall
+```
+
+Follow the `helm install` output to finish setting up Grafana OnCall backend and Grafana OnCall frontend plugin e.g.
+
+```bash
+đź‘‹ Your Grafana OnCall instance has been successfully deployed
+
+ âť— Set up a DNS record for your domain (use A Record and "@" to point a root domain to the IP address)
+ Get the external IP address by running the following commands and point example.com to it:
+
+ kubectl get ingress release-oncall -o jsonpath="{.status.loadBalancer.ingress[0].ip}"
+
+ Wait until the dns record got propagated.
+ NOTE: Check with the following command: nslookup example.com
+ Try reaching https://example.com/ready/ from the browser, make sure it is not cached locally
+
+ 🦎 Grafana was installed as a part of this helm release. Open https://example.com/grafana/plugins/grafana-oncall-app
+ The User is admin
+ Get password by running this command:
+
+ kubectl get secret --namespace default release-oncall-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
+
+ đź”— Connect Grafana OnCall Plugin to Grafana OnCall backend:
+
+ Fill the Grafana OnCall Backend URL:
+
+ http://release-oncall-engine:8080
+
+🎉🎉🎉 Done! 🎉🎉🎉
+```
+
+## Configuration
+
+You can edit values.yml to make changes to the helm chart configuration and re-deploy the release with the following command:
+
+```bash
+helm upgrade \
+ --install \
+ --wait \
+ --set base_url=example.com \
+ --set grafana."grafana\.ini".server.domain=example.com \
+ release-oncall \
+ grafana/oncall
+```
+
+### Passwords and external secrets
+
+As OnCall subcharts are Bitname charts, there is a common approach to secrets. Bundled charts allow specifying passwords
+in values.yaml explicitly or as K8s secret value. OnCall chart refers either to secret created in sub-chart or
+to specified external secret.
+Similarly, if component chart is disabled, the password(s) can be supplied in `external` value
+(e.g. externalMysql) explicitly or as K8s secret value. In the first case, the secret is created with the specified
+value. In the second case the external secret is used.
+
+- If `.auth.existingSecret` is non-empty, then this secret is used. Secret keys are pre-defined by chart.
+- If subchart supports password files and `.customPasswordFiles` dictionary is non-empty, then password files
+ are used. Dictionary keys are pre-defined per sub-chart. Password files are not supported by OnCall chart and should
+ not be used with bundled sub-charts.
+- Passwords are specified via `auth` section values, e.g. `auth.password`. K8s secret is created.
+ - If `.auth.forcePassword` is `true`, then passwords MUST be specified. Otherwise, missing passwords
+ are generated.
+
+If external component is used instead of the bundled one:
+
+- If existingSecret within appropriate external component values is non-empty (e.g. `externalMysql.existingSecret`) then
+ it is used together with corresponding key names, e.g. `externalMysql.passwordKey`.
+- Otherwise, corresponding password values are used, e.g. `externalMysql.password`. K8s secret is created by OnCall chart.
+
+Below is the summary for the dependent charts.
+
+MySQL/MariaDB:
+
+```yaml
+database:
+ type: "mysql" # This is default
+mariaDB:
+ enabled: true # Default
+ auth:
+ existingSecret: ""
+ forcePassword: false
+ # Secret name: `-mariadb`
+ rootPassword: "" # Secret key: mariadb-root-password
+ password: "" # Secret key: mariadb-password
+ replicationPassword: "" # Secret key: mariadb-replication-password
+externalMysql:
+ password: ""
+ existingSecret: ""
+ passwordKey: ""
+```
+
+Postgres:
+
+```yaml
+database:
+ type: postgresql
+mariadb:
+ enabled: false # Must be set to false for Postgres
+postgresql:
+ enabled: true # Must be set to true for bundled Postgres
+ auth:
+ existingSecret: ""
+ secretKeys:
+ adminPasswordKey: ""
+ userPasswordKey: "" # Not needed
+ replicationPasswordKey: "" # Not needed with disabled replication
+ # Secret name: `-postgresql`
+ postgresPassword: "" # password for admin user postgres. As non-admin user is not created, only this one is relevant.
+ password: "" # Not needed
+ replicationPassword: "" # Not needed with disabled replication
+externalPostgresql:
+ user: ""
+ password: ""
+ existingSecret: ""
+ passwordKey: ""
+```
+
+Rabbitmq:
+
+```yaml
+rabbitmq:
+ enabled: true
+ auth:
+ existingPasswordSecret: "" # Must contain `rabbitmq-password` key
+ existingErlangSecret: "" # Must contain `rabbitmq-erlang-cookie` key
+ # Secret name: `-rabbitmq`
+ password: ""
+ erlangCookie: ""
+externalRabbitmq:
+ user: ""
+ password: ""
+ existingSecret: ""
+ passwordKey: ""
+ usernameKey: ""
+```
+
+Redis:
+
+```yaml
+redis:
+ enabled: true
+ auth:
+ existingSecret: ""
+ existingSecretPasswordKey: ""
+ # Secret name: `-redis`
+ password: ""
+externalRedis:
+ password: ""
+ existingSecret: ""
+ passwordKey: ""
+```
+
+### Running split ingestion and API services
+
+You can run a detached service for handling integrations by setting up the following variables:
+
+```yaml
+detached_integrations:
+ enabled: true
+detached_integrations_service:
+ enabled: true
+```
+
+This will run an integrations-only service listening by default in port 30003.
+
+### Set up Slack and Telegram
+
+You can set up Slack connection via following variables:
+
+```yaml
+oncall:
+ slack:
+ enabled: true
+ commandName: oncall
+ clientId: ~
+ clientSecret: ~
+ signingSecret: ~
+ existingSecret: ""
+ clientIdKey: ""
+ clientSecretKey: ""
+ signingSecretKey: ""
+ redirectHost: ~
+```
+
+`oncall.slack.commandName` is used for changing default bot slash command,
+`oncall`. In slack, it could be called via `/`.
+
+To set up Telegram token and webhook url use:
+
+```yaml
+oncall:
+ telegram:
+ enabled: true
+ token: ~
+ webhookUrl: ~
+```
+
+To use Telegram long polling instead of webhook use:
+
+```yaml
+telegramPolling:
+ enabled: true
+```
+
+### Set up external access
+
+Grafana OnCall can be connected to the external monitoring systems or grafana deployed to the other cluster.
+Nginx Ingress Controller and Cert Manager charts are included in the helm chart with the default configuration.
+If you set the DNS A Record pointing to the external IP address of the installation with the Hostname matching
+base_url parameter, https will be automatically set up. If grafana is enabled in the chart values, it will also be
+available on `https:///grafana/`. See the details in `helm install` output.
+
+To use a different ingress controller or tls certificate management system, set the following values to
+false and edit ingress settings
+
+```yaml
+ingress-nginx:
+ enabled: false
+
+cert-manager:
+ enabled: false
+
+ingress:
+ enabled: true
+ annotations:
+ kubernetes.io/ingress.class: "nginx"
+ cert-manager.io/issuer: "letsencrypt-prod"
+```
+
+### Use PostgreSQL instead of MySQL
+
+It is possible to use PostgreSQL instead of MySQL. To do so, set mariadb.enabled to `false`,
+postgresql.enabled to `true` and database.type to `postgresql`.
+
+```yaml
+mariadb:
+ enabled: false
+
+postgresql:
+ enabled: true
+
+database:
+ type: postgresql
+```
+
+### Connect external MySQL
+
+It is recommended to use the managed MySQL 5.7 database provided by your cloud provider
+Make sure to create the database with the following parameters before installing this chart
+
+```sql
+CREATE DATABASE oncall CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+```
+
+To use an external MySQL instance set mariadb.enabled to `false` and configure the `externalMysql` parameters.
+
+```yaml
+mariadb:
+ enabled: false
+
+# Make sure to create the database with the following parameters:
+# CREATE DATABASE oncall CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+externalMysql:
+ host:
+ port:
+ db_name:
+ user:
+ password:
+ existingSecret: ""
+ usernameKey: username
+ passwordKey: password
+```
+
+### Connect external PostgreSQL
+
+To use an external PostgreSQL instance set mariadb.enabled to `false`,
+postgresql.enabled to `false`, database.type to `postgresql` and configure
+the `externalPostgresql` parameters.
+
+```yaml
+mariadb:
+ enabled: false
+
+postgresql:
+ enabled: false
+
+database:
+ type: postgresql
+
+# Make sure to create the database with the following parameters:
+# CREATE DATABASE oncall WITH ENCODING UTF8;
+externalPostgresql:
+ host:
+ port:
+ db_name:
+ user:
+ password:
+ existingSecret: ""
+ passwordKey: password
+```
+
+### Connect external RabbitMQ
+
+Option 1. Install RabbitMQ separately into the cluster using the [official documentation](https://www.rabbitmq.com/kubernetes/operator/operator-overview.html)
+Option 2. Use managed solution such as [CloudAMPQ](https://www.cloudamqp.com/)
+
+To use an external RabbitMQ instance set rabbitmq.enabled to `false` and configure the `externalRabbitmq` parameters.
+
+```yaml
+rabbitmq:
+ enabled: false # Disable the RabbitMQ dependency from the release
+
+externalRabbitmq:
+ host:
+ port:
+ user:
+ password:
+ protocol:
+ vhost:
+ existingSecret: ""
+ passwordKey: password
+ usernameKey: username
+```
+
+### Connect external Redis
+
+To use an external Redis instance set redis.enabled to `false` and configure the `externalRedis` parameters.
+
+```yaml
+redis:
+ enabled: false # Disable the Redis dependency from the release
+
+externalRedis:
+ host:
+ password:
+ existingSecret: ""
+ passwordKey: password
+```
+
+## Update
+
+```bash
+# Add & upgrade the repository
+helm repo add grafana https://grafana.github.io/helm-charts
+helm repo update
+
+# Re-deploy
+helm upgrade \
+ --install \
+ --wait \
+ --set base_url=example.com \
+ --set grafana."grafana\.ini".server.domain=example.com \
+ release-oncall \
+ grafana/oncall
+```
+
+After re-deploying, please also update the Grafana OnCall plugin on the plugin version page.
+See [Grafana docs](https://grafana.com/docs/grafana/latest/administration/plugin-management/#update-a-plugin) for
+more info on updating Grafana plugins.
+
+## Uninstall
+
+### Uninstalling the helm chart
+
+```bash
+helm delete release-oncall
+```
+
+### Clean up PVC's
+
+```bash
+kubectl delete pvc data-release-oncall-mariadb-0 data-release-oncall-rabbitmq-0 \
+redis-data-release-oncall-redis-master-0 redis-data-release-oncall-redis-replicas-0 \
+redis-data-release-oncall-redis-replicas-1 redis-data-release-oncall-redis-replicas-2
+```
+
+### Clean up secrets
+
+```bash
+kubectl delete secrets certificate-tls release-oncall-cert-manager-webhook-ca release-oncall-ingress-nginx-admission
+```
+
+## Troubleshooting
+
+### Issues during initial configuration
+
+In the event that you run into issues during initial configuration, it is possible that mismatching versions between
+your OnCall backend and UI is the culprit. Ensure that the versions match, and if not,
+consider updating your `helm` deployment.
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/Chart.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/Chart.yaml
new file mode 100644
index 0000000..44da745
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/Chart.yaml
@@ -0,0 +1,22 @@
+annotations:
+ artifacthub.io/prerelease: "false"
+ artifacthub.io/signKey: |
+ fingerprint: 1020CF3C033D4F35BAE1C19E1226061C665DF13E
+ url: https://cert-manager.io/public-keys/cert-manager-keyring-2021-09-20-1020CF3C033D4F35BAE1C19E1226061C665DF13E.gpg
+apiVersion: v1
+appVersion: v1.8.0
+description: A Helm chart for cert-manager
+home: https://github.com/cert-manager/cert-manager
+icon: https://raw.githubusercontent.com/jetstack/cert-manager/master/logo/logo.png
+keywords:
+- cert-manager
+- kube-lego
+- letsencrypt
+- tls
+maintainers:
+- email: cert-manager-maintainers@googlegroups.com
+ name: cert-manager-maintainers
+name: cert-manager
+sources:
+- https://github.com/cert-manager/cert-manager
+version: v1.8.0
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/README.md b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/README.md
new file mode 100644
index 0000000..5adc560
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/README.md
@@ -0,0 +1,220 @@
+# cert-manager
+
+cert-manager is a Kubernetes addon to automate the management and issuance of
+TLS certificates from various issuing sources.
+
+It will ensure certificates are valid and up to date periodically, and attempt
+to renew certificates at an appropriate time before expiry.
+
+## Prerequisites
+
+- Kubernetes 1.18+
+
+## Installing the Chart
+
+Full installation instructions, including details on how to configure extra
+functionality in cert-manager can be found in the [installation docs](https://cert-manager.io/docs/installation/kubernetes/).
+
+Before installing the chart, you must first install the cert-manager CustomResourceDefinition resources.
+This is performed in a separate step to allow you to easily uninstall and reinstall cert-manager without deleting your installed custom resources.
+
+```bash
+$ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.crds.yaml
+```
+
+To install the chart with the release name `my-release`:
+
+```console
+## Add the Jetstack Helm repository
+$ helm repo add jetstack https://charts.jetstack.io
+
+## Install the cert-manager helm chart
+$ helm install my-release --namespace cert-manager --version v1.8.0 jetstack/cert-manager
+```
+
+In order to begin issuing certificates, you will need to set up a ClusterIssuer
+or Issuer resource (for example, by creating a 'letsencrypt-staging' issuer).
+
+More information on the different types of issuers and how to configure them
+can be found in [our documentation](https://cert-manager.io/docs/configuration/).
+
+For information on how to configure cert-manager to automatically provision
+Certificates for Ingress resources, take a look at the
+[Securing Ingresses documentation](https://cert-manager.io/docs/usage/ingress/).
+
+> **Tip**: List all releases using `helm list`
+
+## Upgrading the Chart
+
+Special considerations may be required when upgrading the Helm chart, and these
+are documented in our full [upgrading guide](https://cert-manager.io/docs/installation/upgrading/).
+
+**Please check here before performing upgrades!**
+
+## Uninstalling the Chart
+
+To uninstall/delete the `my-release` deployment:
+
+```console
+$ helm delete my-release
+```
+
+The command removes all the Kubernetes components associated with the chart and deletes the release.
+
+If you want to completely uninstall cert-manager from your cluster, you will also need to
+delete the previously installed CustomResourceDefinition resources:
+
+```console
+$ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.crds.yaml
+```
+
+## Configuration
+
+The following table lists the configurable parameters of the cert-manager chart and their default values.
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `global.imagePullSecrets` | Reference to one or more secrets to be used when pulling images | `[]` |
+| `global.rbac.create` | If `true`, create and use RBAC resources (includes sub-charts) | `true` |
+| `global.priorityClassName`| Priority class name for cert-manager and webhook pods | `""` |
+| `global.podSecurityPolicy.enabled` | If `true`, create and use PodSecurityPolicy (includes sub-charts) | `false` |
+| `global.podSecurityPolicy.useAppArmor` | If `true`, use Apparmor seccomp profile in PSP | `true` |
+| `global.leaderElection.namespace` | Override the namespace used to store the ConfigMap for leader election | `kube-system` |
+| `global.leaderElection.leaseDuration` | The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate | |
+| `global.leaderElection.renewDeadline` | The interval between attempts by the acting master to renew a leadership slot before it stops leading. This must be less than or equal to the lease duration | |
+| `global.leaderElection.retryPeriod` | The duration the clients should wait between attempting acquisition and renewal of a leadership | |
+| `installCRDs` | If true, CRD resources will be installed as part of the Helm chart. If enabled, when uninstalling CRD resources will be deleted causing all installed custom resources to be DELETED | `false` |
+| `image.repository` | Image repository | `quay.io/jetstack/cert-manager-controller` |
+| `image.tag` | Image tag | `v1.8.0` |
+| `image.pullPolicy` | Image pull policy | `IfNotPresent` |
+| `replicaCount` | Number of cert-manager replicas | `1` |
+| `clusterResourceNamespace` | Override the namespace used to store DNS provider credentials etc. for ClusterIssuer resources | Same namespace as cert-manager pod |
+| `featureGates` | Set of comma-separated key=value pairs that describe feature gates on the controller. Some feature gates may also have to be enabled on other components, and can be set supplying the `feature-gate` flag to `.extraArgs` | `` |
+| `extraArgs` | Optional flags for cert-manager | `[]` |
+| `extraEnv` | Optional environment variables for cert-manager | `[]` |
+| `serviceAccount.create` | If `true`, create a new service account | `true` |
+| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the fullname template | |
+| `serviceAccount.annotations` | Annotations to add to the service account | |
+| `serviceAccount.automountServiceAccountToken` | Automount API credentials for the Service Account | `true` |
+| `volumes` | Optional volumes for cert-manager | `[]` |
+| `volumeMounts` | Optional volume mounts for cert-manager | `[]` |
+| `resources` | CPU/memory resource requests/limits | `{}` |
+| `securityContext` | Optional security context. The yaml block should adhere to the [SecurityContext spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#securitycontext-v1-core) | `{}` |
+| `securityContext.enabled` | Deprecated (use `securityContext`) - Enable security context | `false` |
+| `containerSecurityContext` | Security context to be set on the controller component container | `{}` |
+| `nodeSelector` | Node labels for pod assignment | `{}` |
+| `affinity` | Node affinity for pod assignment | `{}` |
+| `tolerations` | Node tolerations for pod assignment | `[]` |
+| `ingressShim.defaultIssuerName` | Optional default issuer to use for ingress resources | |
+| `ingressShim.defaultIssuerKind` | Optional default issuer kind to use for ingress resources | |
+| `ingressShim.defaultIssuerGroup` | Optional default issuer group to use for ingress resources | |
+| `prometheus.enabled` | Enable Prometheus monitoring | `true` |
+| `prometheus.servicemonitor.enabled` | Enable Prometheus Operator ServiceMonitor monitoring | `false` |
+| `prometheus.servicemonitor.namespace` | Define namespace where to deploy the ServiceMonitor resource | (namespace where you are deploying) |
+| `prometheus.servicemonitor.prometheusInstance` | Prometheus Instance definition | `default` |
+| `prometheus.servicemonitor.targetPort` | Prometheus scrape port | `9402` |
+| `prometheus.servicemonitor.path` | Prometheus scrape path | `/metrics` |
+| `prometheus.servicemonitor.interval` | Prometheus scrape interval | `60s` |
+| `prometheus.servicemonitor.labels` | Add custom labels to ServiceMonitor | |
+| `prometheus.servicemonitor.scrapeTimeout` | Prometheus scrape timeout | `30s` |
+| `prometheus.servicemonitor.honorLabels` | Enable label honoring for metrics scraped by Prometheus (see [Prometheus scrape config docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) for details). By setting `honorLabels` to `true`, Prometheus will prefer label contents given by cert-manager on conflicts. Can be used to remove the "exported_namespace" label for example. | `false` |
+| `podAnnotations` | Annotations to add to the cert-manager pod | `{}` |
+| `deploymentAnnotations` | Annotations to add to the cert-manager deployment | `{}` |
+| `podDnsPolicy` | Optional cert-manager pod [DNS policy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pods-dns-policy) | |
+| `podDnsConfig` | Optional cert-manager pod [DNS configurations](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pods-dns-config) | |
+| `podLabels` | Labels to add to the cert-manager pod | `{}` |
+| `serviceLabels` | Labels to add to the cert-manager controller service | `{}` |
+| `serviceAnnotations` | Annotations to add to the cert-manager service | `{}` |
+| `http_proxy` | Value of the `HTTP_PROXY` environment variable in the cert-manager pod | |
+| `https_proxy` | Value of the `HTTPS_PROXY` environment variable in the cert-manager pod | |
+| `no_proxy` | Value of the `NO_PROXY` environment variable in the cert-manager pod | |
+| `webhook.replicaCount` | Number of cert-manager webhook replicas | `1` |
+| `webhook.timeoutSeconds` | Seconds the API server should wait the webhook to respond before treating the call as a failure. | `10` |
+| `webhook.podAnnotations` | Annotations to add to the webhook pods | `{}` |
+| `webhook.podLabels` | Labels to add to the cert-manager webhook pod | `{}` |
+| `webhook.serviceLabels` | Labels to add to the cert-manager webhook service | `{}` |
+| `webhook.deploymentAnnotations` | Annotations to add to the webhook deployment | `{}` |
+| `webhook.mutatingWebhookConfigurationAnnotations` | Annotations to add to the mutating webhook configuration | `{}` |
+| `webhook.validatingWebhookConfigurationAnnotations` | Annotations to add to the validating webhook configuration | `{}` |
+| `webhook.serviceAnnotations` | Annotations to add to the webhook service | `{}` |
+| `webhook.config` | WebhookConfiguration YAML used to configure flags for the webhook. Generates a ConfigMap containing contents of the field. See `values.yaml` for example. | `{}` |
+| `webhook.extraArgs` | Optional flags for cert-manager webhook component | `[]` |
+| `webhook.serviceAccount.create` | If `true`, create a new service account for the webhook component | `true` |
+| `webhook.serviceAccount.name` | Service account for the webhook component to be used. If not set and `webhook.serviceAccount.create` is `true`, a name is generated using the fullname template | |
+| `webhook.serviceAccount.annotations` | Annotations to add to the service account for the webhook component | |
+| `webhook.serviceAccount.automountServiceAccountToken` | Automount API credentials for the webhook Service Account | |
+| `webhook.resources` | CPU/memory resource requests/limits for the webhook pods | `{}` |
+| `webhook.nodeSelector` | Node labels for webhook pod assignment | `{}` |
+| `webhook.affinity` | Node affinity for webhook pod assignment | `{}` |
+| `webhook.tolerations` | Node tolerations for webhook pod assignment | `[]` |
+| `webhook.image.repository` | Webhook image repository | `quay.io/jetstack/cert-manager-webhook` |
+| `webhook.image.tag` | Webhook image tag | `v1.8.0` |
+| `webhook.image.pullPolicy` | Webhook image pull policy | `IfNotPresent` |
+| `webhook.securePort` | The port that the webhook should listen on for requests. | `10250` |
+| `webhook.securityContext` | Security context for webhook pod assignment | `{}` |
+| `webhook.containerSecurityContext` | Security context to be set on the webhook component container | `{}` |
+| `webhook.hostNetwork` | If `true`, run the Webhook on the host network. | `false` |
+| `webhook.serviceType` | The type of the `Service`. | `ClusterIP` |
+| `webhook.loadBalancerIP` | The specific load balancer IP to use (when `serviceType` is `LoadBalancer`). | |
+| `webhook.url.host` | The host to use to reach the webhook, instead of using internal cluster DNS for the service. | |
+| `webhook.livenessProbe.failureThreshold` | The liveness probe failure threshold | `3` |
+| `webhook.livenessProbe.initialDelaySeconds` | The liveness probe initial delay (in seconds) | `60` |
+| `webhook.livenessProbe.periodSeconds` | The liveness probe period (in seconds) | `10` |
+| `webhook.livenessProbe.successThreshold` | The liveness probe success threshold | `1` |
+| `webhook.livenessProbe.timeoutSeconds` | The liveness probe timeout (in seconds) | `1` |
+| `webhook.readinessProbe.failureThreshold` | The readiness probe failure threshold | `3` |
+| `webhook.readinessProbe.initialDelaySeconds` | The readiness probe initial delay (in seconds) | `5` |
+| `webhook.readinessProbe.periodSeconds` | The readiness probe period (in seconds) | `5` |
+| `webhook.readinessProbe.successThreshold` | The readiness probe success threshold | `1` |
+| `webhook.readinessProbe.timeoutSeconds` | The readiness probe timeout (in seconds) | `1` |
+| `cainjector.enabled` | Toggles whether the cainjector component should be installed (required for the webhook component to work) | `true` |
+| `cainjector.replicaCount` | Number of cert-manager cainjector replicas | `1` |
+| `cainjector.podAnnotations` | Annotations to add to the cainjector pods | `{}` |
+| `cainjector.podLabels` | Labels to add to the cert-manager cainjector pod | `{}` |
+| `cainjector.deploymentAnnotations` | Annotations to add to the cainjector deployment | `{}` |
+| `cainjector.extraArgs` | Optional flags for cert-manager cainjector component | `[]` |
+| `cainjector.serviceAccount.create` | If `true`, create a new service account for the cainjector component | `true` |
+| `cainjector.serviceAccount.name` | Service account for the cainjector component to be used. If not set and `cainjector.serviceAccount.create` is `true`, a name is generated using the fullname template | |
+| `cainjector.serviceAccount.annotations` | Annotations to add to the service account for the cainjector component | |
+| `cainjector.serviceAccount.automountServiceAccountToken` | Automount API credentials for the cainjector Service Account | `true` |
+| `cainjector.resources` | CPU/memory resource requests/limits for the cainjector pods | `{}` |
+| `cainjector.nodeSelector` | Node labels for cainjector pod assignment | `{}` |
+| `cainjector.affinity` | Node affinity for cainjector pod assignment | `{}` |
+| `cainjector.tolerations` | Node tolerations for cainjector pod assignment | `[]` |
+| `cainjector.image.repository` | cainjector image repository | `quay.io/jetstack/cert-manager-cainjector` |
+| `cainjector.image.tag` | cainjector image tag | `v1.8.0` |
+| `cainjector.image.pullPolicy` | cainjector image pull policy | `IfNotPresent` |
+| `cainjector.securityContext` | Security context for cainjector pod assignment | `{}` |
+| `cainjector.containerSecurityContext` | Security context to be set on cainjector component container | `{}` |
+| `startupapicheck.enabled` | Toggles whether the startupapicheck Job should be installed | `true` |
+| `startupapicheck.securityContext` | Pod Security Context to be set on the startupapicheck component Pod | `{}` |
+| `startupapicheck.timeout` | Timeout for 'kubectl check api' command | `1m` |
+| `startupapicheck.backoffLimit` | Job backoffLimit | `4` |
+| `startupapicheck.jobAnnotations` | Optional additional annotations to add to the startupapicheck Job | `{}` |
+| `startupapicheck.podAnnotations` | Optional additional annotations to add to the startupapicheck Pods | `{}` |
+| `startupapicheck.extraArgs` | Optional additional arguments for startupapicheck | `[]` |
+| `startupapicheck.resources` | CPU/memory resource requests/limits for the startupapicheck pod | `{}` |
+| `startupapicheck.nodeSelector` | Node labels for startupapicheck pod assignment | `{}` |
+| `startupapicheck.affinity` | Node affinity for startupapicheck pod assignment | `{}` |
+| `startupapicheck.tolerations` | Node tolerations for startupapicheck pod assignment | `[]` |
+| `startupapicheck.podLabels` | Optional additional labels to add to the startupapicheck Pods | `{}` |
+| `startupapicheck.image.repository` | startupapicheck image repository | `quay.io/jetstack/cert-manager-ctl` |
+| `startupapicheck.image.tag` | startupapicheck image tag | `v1.8.0` |
+| `startupapicheck.image.pullPolicy` | startupapicheck image pull policy | `IfNotPresent` |
+| `startupapicheck.serviceAccount.create` | If `true`, create a new service account for the startupapicheck component | `true` |
+| `startupapicheck.serviceAccount.name` | Service account for the startupapicheck component to be used. If not set and `startupapicheck.serviceAccount.create` is `true`, a name is generated using the fullname template | |
+| `startupapicheck.serviceAccount.annotations` | Annotations to add to the service account for the startupapicheck component | |
+| `startupapicheck.serviceAccount.automountServiceAccountToken` | Automount API credentials for the startupapicheck Service Account | `true` |
+
+Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`.
+
+Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example,
+
+```console
+$ helm install my-release -f values.yaml .
+```
+> **Tip**: You can use the default [values.yaml](https://github.com/cert-manager/cert-manager/blob/master/deploy/charts/cert-manager/values.yaml)
+
+## Contributing
+
+This chart is maintained at [github.com/cert-manager/cert-manager](https://github.com/cert-manager/cert-manager/tree/master/deploy/charts/cert-manager).
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/NOTES.txt b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/NOTES.txt
new file mode 100644
index 0000000..1025354
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/NOTES.txt
@@ -0,0 +1,15 @@
+cert-manager {{ .Chart.AppVersion }} has been deployed successfully!
+
+In order to begin issuing certificates, you will need to set up a ClusterIssuer
+or Issuer resource (for example, by creating a 'letsencrypt-staging' issuer).
+
+More information on the different types of issuers and how to configure them
+can be found in our documentation:
+
+https://cert-manager.io/docs/configuration/
+
+For information on how to configure cert-manager to automatically provision
+Certificates for Ingress resources, take a look at the `ingress-shim`
+documentation:
+
+https://cert-manager.io/docs/usage/ingress/
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/_helpers.tpl b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/_helpers.tpl
new file mode 100644
index 0000000..2b6ee7f
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/_helpers.tpl
@@ -0,0 +1,159 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "cert-manager.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "cert-manager.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "cert-manager.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+ {{ default (include "cert-manager.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+ {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Webhook templates
+*/}}
+
+{{/*
+Expand the name of the chart.
+Manually fix the 'app' and 'name' labels to 'webhook' to maintain
+compatibility with the v0.9 deployment selector.
+*/}}
+{{- define "webhook.name" -}}
+{{- printf "webhook" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "webhook.fullname" -}}
+{{- $trimmedName := printf "%s" (include "cert-manager.fullname" .) | trunc 55 | trimSuffix "-" -}}
+{{- printf "%s-webhook" $trimmedName | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "webhook.caRef" -}}
+{{ .Release.Namespace }}/{{ template "webhook.fullname" . }}-ca
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "webhook.serviceAccountName" -}}
+{{- if .Values.webhook.serviceAccount.create -}}
+ {{ default (include "webhook.fullname" .) .Values.webhook.serviceAccount.name }}
+{{- else -}}
+ {{ default "default" .Values.webhook.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+cainjector templates
+*/}}
+
+{{/*
+Expand the name of the chart.
+Manually fix the 'app' and 'name' labels to 'cainjector' to maintain
+compatibility with the v0.9 deployment selector.
+*/}}
+{{- define "cainjector.name" -}}
+{{- printf "cainjector" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "cainjector.fullname" -}}
+{{- $trimmedName := printf "%s" (include "cert-manager.fullname" .) | trunc 52 | trimSuffix "-" -}}
+{{- printf "%s-cainjector" $trimmedName | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "cainjector.serviceAccountName" -}}
+{{- if .Values.cainjector.serviceAccount.create -}}
+ {{ default (include "cainjector.fullname" .) .Values.cainjector.serviceAccount.name }}
+{{- else -}}
+ {{ default "default" .Values.cainjector.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+startupapicheck templates
+*/}}
+
+{{/*
+Expand the name of the chart.
+Manually fix the 'app' and 'name' labels to 'startupapicheck' to maintain
+compatibility with the v0.9 deployment selector.
+*/}}
+{{- define "startupapicheck.name" -}}
+{{- printf "startupapicheck" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "startupapicheck.fullname" -}}
+{{- $trimmedName := printf "%s" (include "cert-manager.fullname" .) | trunc 52 | trimSuffix "-" -}}
+{{- printf "%s-startupapicheck" $trimmedName | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "startupapicheck.serviceAccountName" -}}
+{{- if .Values.startupapicheck.serviceAccount.create -}}
+ {{ default (include "startupapicheck.fullname" .) .Values.startupapicheck.serviceAccount.name }}
+{{- else -}}
+ {{ default "default" .Values.startupapicheck.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "chartName" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Labels that should be added on each resource
+*/}}
+{{- define "labels" -}}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- if eq (default "helm" .Values.creator) "helm" }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+helm.sh/chart: {{ include "chartName" . }}
+{{- end -}}
+{{- end -}}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-deployment.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-deployment.yaml
new file mode 100644
index 0000000..b617527
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-deployment.yaml
@@ -0,0 +1,102 @@
+{{- if .Values.cainjector.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "cainjector.fullname" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ app: {{ include "cainjector.name" . }}
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.cainjector.deploymentAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ replicas: {{ .Values.cainjector.replicaCount }}
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- with .Values.cainjector.strategy }}
+ strategy:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ template:
+ metadata:
+ labels:
+ app: {{ include "cainjector.name" . }}
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- include "labels" . | nindent 8 }}
+ {{- with .Values.cainjector.podLabels }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.cainjector.podAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ spec:
+ serviceAccountName: {{ template "cainjector.serviceAccountName" . }}
+ {{- with .Values.global.priorityClassName }}
+ priorityClassName: {{ . | quote }}
+ {{- end }}
+ {{- with .Values.cainjector.securityContext }}
+ securityContext:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ containers:
+ - name: {{ .Chart.Name }}
+ {{- with .Values.cainjector.image }}
+ image: "{{- if .registry -}}{{ .registry }}/{{- end -}}{{ .repository }}{{- if (.digest) -}} @{{ .digest }}{{- else -}}:{{ default $.Chart.AppVersion .tag }} {{- end -}}"
+ {{- end }}
+ imagePullPolicy: {{ .Values.cainjector.image.pullPolicy }}
+ args:
+ {{- if .Values.global.logLevel }}
+ - --v={{ .Values.global.logLevel }}
+ {{- end }}
+ {{- with .Values.global.leaderElection }}
+ - --leader-election-namespace={{ .namespace }}
+ {{- if .leaseDuration }}
+ - --leader-election-lease-duration={{ .leaseDuration }}
+ {{- end }}
+ {{- if .renewDeadline }}
+ - --leader-election-renew-deadline={{ .renewDeadline }}
+ {{- end }}
+ {{- if .retryPeriod }}
+ - --leader-election-retry-period={{ .retryPeriod }}
+ {{- end }}
+ {{- end }}
+ {{- with .Values.cainjector.extraArgs }}
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ env:
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ {{- with .Values.cainjector.containerSecurityContext }}
+ securityContext:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.cainjector.resources }}
+ resources:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.cainjector.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.cainjector.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.cainjector.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-psp-clusterrole.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-psp-clusterrole.yaml
new file mode 100644
index 0000000..b75b9eb
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-psp-clusterrole.yaml
@@ -0,0 +1,20 @@
+{{- if .Values.cainjector.enabled }}
+{{- if .Values.global.podSecurityPolicy.enabled }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: {{ template "cainjector.fullname" . }}-psp
+ labels:
+ app: {{ include "cainjector.name" . }}
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- include "labels" . | nindent 4 }}
+rules:
+- apiGroups: ['policy']
+ resources: ['podsecuritypolicies']
+ verbs: ['use']
+ resourceNames:
+ - {{ template "cainjector.fullname" . }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-psp-clusterrolebinding.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-psp-clusterrolebinding.yaml
new file mode 100644
index 0000000..b287802
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-psp-clusterrolebinding.yaml
@@ -0,0 +1,22 @@
+{{- if .Values.cainjector.enabled }}
+{{- if .Values.global.podSecurityPolicy.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cainjector.fullname" . }}-psp
+ labels:
+ app: {{ include "cainjector.name" . }}
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cainjector.fullname" . }}-psp
+subjects:
+ - kind: ServiceAccount
+ name: {{ template "cainjector.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-psp.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-psp.yaml
new file mode 100644
index 0000000..24f01da
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-psp.yaml
@@ -0,0 +1,51 @@
+{{- if .Values.cainjector.enabled }}
+{{- if .Values.global.podSecurityPolicy.enabled }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+ name: {{ template "cainjector.fullname" . }}
+ labels:
+ app: {{ include "cainjector.name" . }}
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- include "labels" . | nindent 4 }}
+ annotations:
+ seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+ seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default'
+ {{- if .Values.global.podSecurityPolicy.useAppArmor }}
+ apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+ apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default'
+ {{- end }}
+spec:
+ privileged: false
+ allowPrivilegeEscalation: false
+ allowedCapabilities: [] # default set of capabilities are implicitly allowed
+ volumes:
+ - 'configMap'
+ - 'emptyDir'
+ - 'projected'
+ - 'secret'
+ - 'downwardAPI'
+ hostNetwork: false
+ hostIPC: false
+ hostPID: false
+ runAsUser:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+ seLinux:
+ rule: 'RunAsAny'
+ supplementalGroups:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+ fsGroup:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-rbac.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-rbac.yaml
new file mode 100644
index 0000000..6cb396e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-rbac.yaml
@@ -0,0 +1,103 @@
+{{- if .Values.cainjector.enabled }}
+{{- if .Values.global.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cainjector.fullname" . }}
+ labels:
+ app: {{ include "cainjector.name" . }}
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ - apiGroups: ["cert-manager.io"]
+ resources: ["certificates"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["get", "create", "update", "patch"]
+ - apiGroups: ["admissionregistration.k8s.io"]
+ resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"]
+ verbs: ["get", "list", "watch", "update"]
+ - apiGroups: ["apiregistration.k8s.io"]
+ resources: ["apiservices"]
+ verbs: ["get", "list", "watch", "update"]
+ - apiGroups: ["apiextensions.k8s.io"]
+ resources: ["customresourcedefinitions"]
+ verbs: ["get", "list", "watch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cainjector.fullname" . }}
+ labels:
+ app: {{ include "cainjector.name" . }}
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cainjector.fullname" . }}
+subjects:
+ - name: {{ template "cainjector.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ kind: ServiceAccount
+
+---
+# leader election rules
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ template "cainjector.fullname" . }}:leaderelection
+ namespace: {{ .Values.global.leaderElection.namespace }}
+ labels:
+ app: {{ include "cainjector.name" . }}
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ # Used for leader election by the controller
+ # cert-manager-cainjector-leader-election is used by the CertificateBased injector controller
+ # see cmd/cainjector/start.go#L113
+ # cert-manager-cainjector-leader-election-core is used by the SecretBased injector controller
+ # see cmd/cainjector/start.go#L137
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ resourceNames: ["cert-manager-cainjector-leader-election", "cert-manager-cainjector-leader-election-core"]
+ verbs: ["get", "update", "patch"]
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["create"]
+
+---
+
+# grant cert-manager permission to manage the leaderelection configmap in the
+# leader election namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ include "cainjector.fullname" . }}:leaderelection
+ namespace: {{ .Values.global.leaderElection.namespace }}
+ labels:
+ app: {{ include "cainjector.name" . }}
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ template "cainjector.fullname" . }}:leaderelection
+subjects:
+ - kind: ServiceAccount
+ name: {{ template "cainjector.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-serviceaccount.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-serviceaccount.yaml
new file mode 100644
index 0000000..6457b9e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/cainjector-serviceaccount.yaml
@@ -0,0 +1,27 @@
+{{- if .Values.cainjector.enabled }}
+{{- if .Values.cainjector.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{ .Values.cainjector.serviceAccount.automountServiceAccountToken }}
+metadata:
+ name: {{ template "cainjector.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ {{- with .Values.cainjector.serviceAccount.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ labels:
+ app: {{ include "cainjector.name" . }}
+ app.kubernetes.io/name: {{ include "cainjector.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cainjector"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.cainjector.serviceAccount.labels }}
+ {{ toYaml . | nindent 4 }}
+ {{- end }}
+{{- with .Values.global.imagePullSecrets }}
+imagePullSecrets:
+ {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/crds.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/crds.yaml
new file mode 100644
index 0000000..afea0de
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/crds.yaml
@@ -0,0 +1,4280 @@
+{{- if .Values.installCRDs }}
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ name: certificaterequests.cert-manager.io
+ labels:
+ app: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/name: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/instance: '{{ .Release.Name }}'
+ # Generated labels {{- include "labels" . | nindent 4 }}
+spec:
+ group: cert-manager.io
+ names:
+ kind: CertificateRequest
+ listKind: CertificateRequestList
+ plural: certificaterequests
+ shortNames:
+ - cr
+ - crs
+ singular: certificaterequest
+ categories:
+ - cert-manager
+ scope: Namespaced
+ versions:
+ - name: v1
+ subresources:
+ status: {}
+ additionalPrinterColumns:
+ - jsonPath: .status.conditions[?(@.type=="Approved")].status
+ name: Approved
+ type: string
+ - jsonPath: .status.conditions[?(@.type=="Denied")].status
+ name: Denied
+ type: string
+ - jsonPath: .status.conditions[?(@.type=="Ready")].status
+ name: Ready
+ type: string
+ - jsonPath: .spec.issuerRef.name
+ name: Issuer
+ type: string
+ - jsonPath: .spec.username
+ name: Requestor
+ type: string
+ - jsonPath: .status.conditions[?(@.type=="Ready")].message
+ name: Status
+ priority: 1
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.
+ name: Age
+ type: date
+ schema:
+ openAPIV3Schema:
+ description: "A CertificateRequest is used to request a signed certificate from one of the configured issuers. \n All fields within the CertificateRequest's `spec` are immutable after creation. A CertificateRequest will either succeed or fail, as denoted by its `status.state` field. \n A CertificateRequest is a one-shot resource, meaning it represents a single point in time request for a certificate and cannot be re-used."
+ type: object
+ required:
+ - spec
+ properties:
+ apiVersion:
+ description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+ type: string
+ kind:
+ description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: Desired state of the CertificateRequest resource.
+ type: object
+ required:
+ - issuerRef
+ - request
+ properties:
+ duration:
+ description: The requested 'duration' (i.e. lifetime) of the Certificate. This option may be ignored/overridden by some issuer types.
+ type: string
+ extra:
+ description: Extra contains extra attributes of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable.
+ type: object
+ additionalProperties:
+ type: array
+ items:
+ type: string
+ groups:
+ description: Groups contains group membership of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable.
+ type: array
+ items:
+ type: string
+ x-kubernetes-list-type: atomic
+ isCA:
+ description: IsCA will request to mark the certificate as valid for certificate signing when submitting to the issuer. This will automatically add the `cert sign` usage to the list of `usages`.
+ type: boolean
+ issuerRef:
+ description: IssuerRef is a reference to the issuer for this CertificateRequest. If the `kind` field is not set, or set to `Issuer`, an Issuer resource with the given name in the same namespace as the CertificateRequest will be used. If the `kind` field is set to `ClusterIssuer`, a ClusterIssuer with the provided name will be used. The `name` field in this stanza is required at all times. The group field refers to the API group of the issuer which defaults to `cert-manager.io` if empty.
+ type: object
+ required:
+ - name
+ properties:
+ group:
+ description: Group of the resource being referred to.
+ type: string
+ kind:
+ description: Kind of the resource being referred to.
+ type: string
+ name:
+ description: Name of the resource being referred to.
+ type: string
+ request:
+ description: The PEM-encoded x509 certificate signing request to be submitted to the CA for signing.
+ type: string
+ format: byte
+ uid:
+ description: UID contains the uid of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable.
+ type: string
+ usages:
+ description: Usages is the set of x509 usages that are requested for the certificate. If usages are set they SHOULD be encoded inside the CSR spec Defaults to `digital signature` and `key encipherment` if not specified.
+ type: array
+ items:
+ description: 'KeyUsage specifies valid usage contexts for keys. See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3 https://tools.ietf.org/html/rfc5280#section-4.2.1.12 Valid KeyUsage values are as follows: "signing", "digital signature", "content commitment", "key encipherment", "key agreement", "data encipherment", "cert sign", "crl sign", "encipher only", "decipher only", "any", "server auth", "client auth", "code signing", "email protection", "s/mime", "ipsec end system", "ipsec tunnel", "ipsec user", "timestamping", "ocsp signing", "microsoft sgc", "netscape sgc"'
+ type: string
+ enum:
+ - signing
+ - digital signature
+ - content commitment
+ - key encipherment
+ - key agreement
+ - data encipherment
+ - cert sign
+ - crl sign
+ - encipher only
+ - decipher only
+ - any
+ - server auth
+ - client auth
+ - code signing
+ - email protection
+ - s/mime
+ - ipsec end system
+ - ipsec tunnel
+ - ipsec user
+ - timestamping
+ - ocsp signing
+ - microsoft sgc
+ - netscape sgc
+ username:
+ description: Username contains the name of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable.
+ type: string
+ status:
+ description: Status of the CertificateRequest. This is set and managed automatically.
+ type: object
+ properties:
+ ca:
+ description: The PEM encoded x509 certificate of the signer, also known as the CA (Certificate Authority). This is set on a best-effort basis by different issuers. If not set, the CA is assumed to be unknown/not available.
+ type: string
+ format: byte
+ certificate:
+ description: The PEM encoded x509 certificate resulting from the certificate signing request. If not set, the CertificateRequest has either not been completed or has failed. More information on failure can be found by checking the `conditions` field.
+ type: string
+ format: byte
+ conditions:
+ description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready` and `InvalidRequest`.
+ type: array
+ items:
+ description: CertificateRequestCondition contains condition information for a CertificateRequest.
+ type: object
+ required:
+ - status
+ - type
+ properties:
+ lastTransitionTime:
+ description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+ type: string
+ format: date-time
+ message:
+ description: Message is a human readable description of the details of the last transition, complementing reason.
+ type: string
+ reason:
+ description: Reason is a brief machine readable explanation for the condition's last transition.
+ type: string
+ status:
+ description: Status of the condition, one of (`True`, `False`, `Unknown`).
+ type: string
+ enum:
+ - "True"
+ - "False"
+ - Unknown
+ type:
+ description: Type of the condition, known values are (`Ready`, `InvalidRequest`, `Approved`, `Denied`).
+ type: string
+ x-kubernetes-list-map-keys:
+ - type
+ x-kubernetes-list-type: map
+ failureTime:
+ description: FailureTime stores the time that this CertificateRequest failed. This is used to influence garbage collection and back-off.
+ type: string
+ format: date-time
+ served: true
+ storage: true
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ name: certificates.cert-manager.io
+ labels:
+ app: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/name: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/instance: '{{ .Release.Name }}'
+ # Generated labels {{- include "labels" . | nindent 4 }}
+spec:
+ group: cert-manager.io
+ names:
+ kind: Certificate
+ listKind: CertificateList
+ plural: certificates
+ shortNames:
+ - cert
+ - certs
+ singular: certificate
+ categories:
+ - cert-manager
+ scope: Namespaced
+ versions:
+ - name: v1
+ subresources:
+ status: {}
+ additionalPrinterColumns:
+ - jsonPath: .status.conditions[?(@.type=="Ready")].status
+ name: Ready
+ type: string
+ - jsonPath: .spec.secretName
+ name: Secret
+ type: string
+ - jsonPath: .spec.issuerRef.name
+ name: Issuer
+ priority: 1
+ type: string
+ - jsonPath: .status.conditions[?(@.type=="Ready")].message
+ name: Status
+ priority: 1
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.
+ name: Age
+ type: date
+ schema:
+ openAPIV3Schema:
+ description: "A Certificate resource should be created to ensure an up to date and signed x509 certificate is stored in the Kubernetes Secret resource named in `spec.secretName`. \n The stored certificate will be renewed before it expires (as configured by `spec.renewBefore`)."
+ type: object
+ required:
+ - spec
+ properties:
+ apiVersion:
+ description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+ type: string
+ kind:
+ description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: Desired state of the Certificate resource.
+ type: object
+ required:
+ - issuerRef
+ - secretName
+ properties:
+ additionalOutputFormats:
+ description: AdditionalOutputFormats defines extra output formats of the private key and signed certificate chain to be written to this Certificate's target Secret. This is an Alpha Feature and is only enabled with the `--feature-gates=AdditionalCertificateOutputFormats=true` option on both the controller and webhook components.
+ type: array
+ items:
+ description: CertificateAdditionalOutputFormat defines an additional output format of a Certificate resource. These contain supplementary data formats of the signed certificate chain and paired private key.
+ type: object
+ required:
+ - type
+ properties:
+ type:
+ description: Type is the name of the format type that should be written to the Certificate's target Secret.
+ type: string
+ enum:
+ - DER
+ - CombinedPEM
+ commonName:
+ description: 'CommonName is a common name to be used on the Certificate. The CommonName should have a length of 64 characters or fewer to avoid generating invalid CSRs. This value is ignored by TLS clients when any subject alt name is set. This is x509 behaviour: https://tools.ietf.org/html/rfc6125#section-6.4.4'
+ type: string
+ dnsNames:
+ description: DNSNames is a list of DNS subjectAltNames to be set on the Certificate.
+ type: array
+ items:
+ type: string
+ duration:
+ description: The requested 'duration' (i.e. lifetime) of the Certificate. This option may be ignored/overridden by some issuer types. If unset this defaults to 90 days. Certificate will be renewed either 2/3 through its duration or `renewBefore` period before its expiry, whichever is later. Minimum accepted duration is 1 hour. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration
+ type: string
+ emailAddresses:
+ description: EmailAddresses is a list of email subjectAltNames to be set on the Certificate.
+ type: array
+ items:
+ type: string
+ encodeUsagesInRequest:
+ description: EncodeUsagesInRequest controls whether key usages should be present in the CertificateRequest
+ type: boolean
+ ipAddresses:
+ description: IPAddresses is a list of IP address subjectAltNames to be set on the Certificate.
+ type: array
+ items:
+ type: string
+ isCA:
+ description: IsCA will mark this Certificate as valid for certificate signing. This will automatically add the `cert sign` usage to the list of `usages`.
+ type: boolean
+ issuerRef:
+ description: IssuerRef is a reference to the issuer for this certificate. If the `kind` field is not set, or set to `Issuer`, an Issuer resource with the given name in the same namespace as the Certificate will be used. If the `kind` field is set to `ClusterIssuer`, a ClusterIssuer with the provided name will be used. The `name` field in this stanza is required at all times.
+ type: object
+ required:
+ - name
+ properties:
+ group:
+ description: Group of the resource being referred to.
+ type: string
+ kind:
+ description: Kind of the resource being referred to.
+ type: string
+ name:
+ description: Name of the resource being referred to.
+ type: string
+ keystores:
+ description: Keystores configures additional keystore output formats stored in the `secretName` Secret resource.
+ type: object
+ properties:
+ jks:
+ description: JKS configures options for storing a JKS keystore in the `spec.secretName` Secret resource.
+ type: object
+ required:
+ - create
+ - passwordSecretRef
+ properties:
+ create:
+ description: Create enables JKS keystore creation for the Certificate. If true, a file named `keystore.jks` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will only be updated upon re-issuance. A file named `truststore.jks` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority
+ type: boolean
+ passwordSecretRef:
+ description: PasswordSecretRef is a reference to a key in a Secret resource containing the password used to encrypt the JKS keystore.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ pkcs12:
+ description: PKCS12 configures options for storing a PKCS12 keystore in the `spec.secretName` Secret resource.
+ type: object
+ required:
+ - create
+ - passwordSecretRef
+ properties:
+ create:
+ description: Create enables PKCS12 keystore creation for the Certificate. If true, a file named `keystore.p12` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will only be updated upon re-issuance. A file named `truststore.p12` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority
+ type: boolean
+ passwordSecretRef:
+ description: PasswordSecretRef is a reference to a key in a Secret resource containing the password used to encrypt the PKCS12 keystore.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ privateKey:
+ description: Options to control private keys used for the Certificate.
+ type: object
+ properties:
+ algorithm:
+ description: Algorithm is the private key algorithm of the corresponding private key for this certificate. If provided, allowed values are either `RSA`,`Ed25519` or `ECDSA` If `algorithm` is specified and `size` is not provided, key size of 256 will be used for `ECDSA` key algorithm and key size of 2048 will be used for `RSA` key algorithm. key size is ignored when using the `Ed25519` key algorithm.
+ type: string
+ enum:
+ - RSA
+ - ECDSA
+ - Ed25519
+ encoding:
+ description: The private key cryptography standards (PKCS) encoding for this certificate's private key to be encoded in. If provided, allowed values are `PKCS1` and `PKCS8` standing for PKCS#1 and PKCS#8, respectively. Defaults to `PKCS1` if not specified.
+ type: string
+ enum:
+ - PKCS1
+ - PKCS8
+ rotationPolicy:
+ description: RotationPolicy controls how private keys should be regenerated when a re-issuance is being processed. If set to Never, a private key will only be generated if one does not already exist in the target `spec.secretName`. If one does exists but it does not have the correct algorithm or size, a warning will be raised to await user intervention. If set to Always, a private key matching the specified requirements will be generated whenever a re-issuance occurs. Default is 'Never' for backward compatibility.
+ type: string
+ enum:
+ - Never
+ - Always
+ size:
+ description: Size is the key bit size of the corresponding private key for this certificate. If `algorithm` is set to `RSA`, valid values are `2048`, `4096` or `8192`, and will default to `2048` if not specified. If `algorithm` is set to `ECDSA`, valid values are `256`, `384` or `521`, and will default to `256` if not specified. If `algorithm` is set to `Ed25519`, Size is ignored. No other values are allowed.
+ type: integer
+ renewBefore:
+ description: How long before the currently issued certificate's expiry cert-manager should renew the certificate. The default is 2/3 of the issued certificate's duration. Minimum accepted value is 5 minutes. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration
+ type: string
+ revisionHistoryLimit:
+ description: revisionHistoryLimit is the maximum number of CertificateRequest revisions that are maintained in the Certificate's history. Each revision represents a single `CertificateRequest` created by this Certificate, either when it was created, renewed, or Spec was changed. Revisions will be removed by oldest first if the number of revisions exceeds this number. If set, revisionHistoryLimit must be a value of `1` or greater. If unset (`nil`), revisions will not be garbage collected. Default value is `nil`.
+ type: integer
+ format: int32
+ secretName:
+ description: SecretName is the name of the secret resource that will be automatically created and managed by this Certificate resource. It will be populated with a private key and certificate, signed by the denoted issuer.
+ type: string
+ secretTemplate:
+ description: SecretTemplate defines annotations and labels to be copied to the Certificate's Secret. Labels and annotations on the Secret will be changed as they appear on the SecretTemplate when added or removed. SecretTemplate annotations are added in conjunction with, and cannot overwrite, the base set of annotations cert-manager sets on the Certificate's Secret.
+ type: object
+ properties:
+ annotations:
+ description: Annotations is a key value map to be copied to the target Kubernetes Secret.
+ type: object
+ additionalProperties:
+ type: string
+ labels:
+ description: Labels is a key value map to be copied to the target Kubernetes Secret.
+ type: object
+ additionalProperties:
+ type: string
+ subject:
+ description: Full X509 name specification (https://golang.org/pkg/crypto/x509/pkix/#Name).
+ type: object
+ properties:
+ countries:
+ description: Countries to be used on the Certificate.
+ type: array
+ items:
+ type: string
+ localities:
+ description: Cities to be used on the Certificate.
+ type: array
+ items:
+ type: string
+ organizationalUnits:
+ description: Organizational Units to be used on the Certificate.
+ type: array
+ items:
+ type: string
+ organizations:
+ description: Organizations to be used on the Certificate.
+ type: array
+ items:
+ type: string
+ postalCodes:
+ description: Postal codes to be used on the Certificate.
+ type: array
+ items:
+ type: string
+ provinces:
+ description: State/Provinces to be used on the Certificate.
+ type: array
+ items:
+ type: string
+ serialNumber:
+ description: Serial number to be used on the Certificate.
+ type: string
+ streetAddresses:
+ description: Street addresses to be used on the Certificate.
+ type: array
+ items:
+ type: string
+ uris:
+ description: URIs is a list of URI subjectAltNames to be set on the Certificate.
+ type: array
+ items:
+ type: string
+ usages:
+ description: Usages is the set of x509 usages that are requested for the certificate. Defaults to `digital signature` and `key encipherment` if not specified.
+ type: array
+ items:
+ description: 'KeyUsage specifies valid usage contexts for keys. See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3 https://tools.ietf.org/html/rfc5280#section-4.2.1.12 Valid KeyUsage values are as follows: "signing", "digital signature", "content commitment", "key encipherment", "key agreement", "data encipherment", "cert sign", "crl sign", "encipher only", "decipher only", "any", "server auth", "client auth", "code signing", "email protection", "s/mime", "ipsec end system", "ipsec tunnel", "ipsec user", "timestamping", "ocsp signing", "microsoft sgc", "netscape sgc"'
+ type: string
+ enum:
+ - signing
+ - digital signature
+ - content commitment
+ - key encipherment
+ - key agreement
+ - data encipherment
+ - cert sign
+ - crl sign
+ - encipher only
+ - decipher only
+ - any
+ - server auth
+ - client auth
+ - code signing
+ - email protection
+ - s/mime
+ - ipsec end system
+ - ipsec tunnel
+ - ipsec user
+ - timestamping
+ - ocsp signing
+ - microsoft sgc
+ - netscape sgc
+ status:
+ description: Status of the Certificate. This is set and managed automatically.
+ type: object
+ properties:
+ conditions:
+ description: List of status conditions to indicate the status of certificates. Known condition types are `Ready` and `Issuing`.
+ type: array
+ items:
+ description: CertificateCondition contains condition information for an Certificate.
+ type: object
+ required:
+ - status
+ - type
+ properties:
+ lastTransitionTime:
+ description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+ type: string
+ format: date-time
+ message:
+ description: Message is a human readable description of the details of the last transition, complementing reason.
+ type: string
+ observedGeneration:
+ description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Certificate.
+ type: integer
+ format: int64
+ reason:
+ description: Reason is a brief machine readable explanation for the condition's last transition.
+ type: string
+ status:
+ description: Status of the condition, one of (`True`, `False`, `Unknown`).
+ type: string
+ enum:
+ - "True"
+ - "False"
+ - Unknown
+ type:
+ description: Type of the condition, known values are (`Ready`, `Issuing`).
+ type: string
+ x-kubernetes-list-map-keys:
+ - type
+ x-kubernetes-list-type: map
+ failedIssuanceAttempts:
+ description: The number of continuous failed issuance attempts up till now. This field gets removed (if set) on a successful issuance and gets set to 1 if unset and an issuance has failed. If an issuance has failed, the delay till the next issuance will be calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts - 1).
+ type: integer
+ lastFailureTime:
+ description: LastFailureTime is the time as recorded by the Certificate controller of the most recent failure to complete a CertificateRequest for this Certificate resource. If set, cert-manager will not re-request another Certificate until 1 hour has elapsed from this time.
+ type: string
+ format: date-time
+ nextPrivateKeySecretName:
+ description: The name of the Secret resource containing the private key to be used for the next certificate iteration. The keymanager controller will automatically set this field if the `Issuing` condition is set to `True`. It will automatically unset this field when the Issuing condition is not set or False.
+ type: string
+ notAfter:
+ description: The expiration time of the certificate stored in the secret named by this resource in `spec.secretName`.
+ type: string
+ format: date-time
+ notBefore:
+ description: The time after which the certificate stored in the secret named by this resource in spec.secretName is valid.
+ type: string
+ format: date-time
+ renewalTime:
+ description: RenewalTime is the time at which the certificate will be next renewed. If not set, no upcoming renewal is scheduled.
+ type: string
+ format: date-time
+ revision:
+ description: "The current 'revision' of the certificate as issued. \n When a CertificateRequest resource is created, it will have the `cert-manager.io/certificate-revision` set to one greater than the current value of this field. \n Upon issuance, this field will be set to the value of the annotation on the CertificateRequest resource used to issue the certificate. \n Persisting the value on the CertificateRequest resource allows the certificates controller to know whether a request is part of an old issuance or if it is part of the ongoing revision's issuance by checking if the revision value in the annotation is greater than this field."
+ type: integer
+ served: true
+ storage: true
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ name: challenges.acme.cert-manager.io
+ labels:
+ app: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/name: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/instance: '{{ .Release.Name }}'
+ # Generated labels {{- include "labels" . | nindent 4 }}
+spec:
+ group: acme.cert-manager.io
+ names:
+ kind: Challenge
+ listKind: ChallengeList
+ plural: challenges
+ singular: challenge
+ categories:
+ - cert-manager
+ - cert-manager-acme
+ scope: Namespaced
+ versions:
+ - additionalPrinterColumns:
+ - jsonPath: .status.state
+ name: State
+ type: string
+ - jsonPath: .spec.dnsName
+ name: Domain
+ type: string
+ - jsonPath: .status.reason
+ name: Reason
+ priority: 1
+ type: string
+ - description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.
+ jsonPath: .metadata.creationTimestamp
+ name: Age
+ type: date
+ name: v1
+ schema:
+ openAPIV3Schema:
+ description: Challenge is a type to represent a Challenge request with an ACME server
+ type: object
+ required:
+ - metadata
+ - spec
+ properties:
+ apiVersion:
+ description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+ type: string
+ kind:
+ description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+ type: string
+ metadata:
+ type: object
+ spec:
+ type: object
+ required:
+ - authorizationURL
+ - dnsName
+ - issuerRef
+ - key
+ - solver
+ - token
+ - type
+ - url
+ properties:
+ authorizationURL:
+ description: The URL to the ACME Authorization resource that this challenge is a part of.
+ type: string
+ dnsName:
+ description: dnsName is the identifier that this challenge is for, e.g. example.com. If the requested DNSName is a 'wildcard', this field MUST be set to the non-wildcard domain, e.g. for `*.example.com`, it must be `example.com`.
+ type: string
+ issuerRef:
+ description: References a properly configured ACME-type Issuer which should be used to create this Challenge. If the Issuer does not exist, processing will be retried. If the Issuer is not an 'ACME' Issuer, an error will be returned and the Challenge will be marked as failed.
+ type: object
+ required:
+ - name
+ properties:
+ group:
+ description: Group of the resource being referred to.
+ type: string
+ kind:
+ description: Kind of the resource being referred to.
+ type: string
+ name:
+ description: Name of the resource being referred to.
+ type: string
+ key:
+ description: 'The ACME challenge key for this challenge For HTTP01 challenges, this is the value that must be responded with to complete the HTTP01 challenge in the format: `.`. For DNS01 challenges, this is the base64 encoded SHA256 sum of the `.` text that must be set as the TXT record content.'
+ type: string
+ solver:
+ description: Contains the domain solving configuration that should be used to solve this challenge resource.
+ type: object
+ properties:
+ dns01:
+ description: Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow.
+ type: object
+ properties:
+ acmeDNS:
+ description: Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records.
+ type: object
+ required:
+ - accountSecretRef
+ - host
+ properties:
+ accountSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ host:
+ type: string
+ akamai:
+ description: Use the Akamai DNS zone management API to manage DNS01 challenge records.
+ type: object
+ required:
+ - accessTokenSecretRef
+ - clientSecretSecretRef
+ - clientTokenSecretRef
+ - serviceConsumerDomain
+ properties:
+ accessTokenSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ clientSecretSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ clientTokenSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ serviceConsumerDomain:
+ type: string
+ azureDNS:
+ description: Use the Microsoft Azure DNS API to manage DNS01 challenge records.
+ type: object
+ required:
+ - resourceGroupName
+ - subscriptionID
+ properties:
+ clientID:
+ description: if both this and ClientSecret are left unset MSI will be used
+ type: string
+ clientSecretSecretRef:
+ description: if both this and ClientID are left unset MSI will be used
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ environment:
+ description: name of the Azure environment (default AzurePublicCloud)
+ type: string
+ enum:
+ - AzurePublicCloud
+ - AzureChinaCloud
+ - AzureGermanCloud
+ - AzureUSGovernmentCloud
+ hostedZoneName:
+ description: name of the DNS zone that should be used
+ type: string
+ managedIdentity:
+ description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID
+ type: object
+ properties:
+ clientID:
+ description: client ID of the managed identity, can not be used at the same time as resourceID
+ type: string
+ resourceID:
+ description: resource ID of the managed identity, can not be used at the same time as clientID
+ type: string
+ resourceGroupName:
+ description: resource group the DNS zone is located in
+ type: string
+ subscriptionID:
+ description: ID of the Azure subscription
+ type: string
+ tenantID:
+ description: when specifying ClientID and ClientSecret then this field is also needed
+ type: string
+ cloudDNS:
+ description: Use the Google Cloud DNS API to manage DNS01 challenge records.
+ type: object
+ required:
+ - project
+ properties:
+ hostedZoneName:
+ description: HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone.
+ type: string
+ project:
+ type: string
+ serviceAccountSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ cloudflare:
+ description: Use the Cloudflare API to manage DNS01 challenge records.
+ type: object
+ properties:
+ apiKeySecretRef:
+ description: 'API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions.'
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ apiTokenSecretRef:
+ description: API token used to authenticate with Cloudflare.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ email:
+ description: Email of the account, only required when using API key based authentication.
+ type: string
+ cnameStrategy:
+ description: CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones.
+ type: string
+ enum:
+ - None
+ - Follow
+ digitalocean:
+ description: Use the DigitalOcean DNS API to manage DNS01 challenge records.
+ type: object
+ required:
+ - tokenSecretRef
+ properties:
+ tokenSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ rfc2136:
+ description: Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records.
+ type: object
+ required:
+ - nameserver
+ properties:
+ nameserver:
+ description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1])Â ; port is optional. This field is required.
+ type: string
+ tsigAlgorithm:
+ description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.'
+ type: string
+ tsigKeyName:
+ description: The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required.
+ type: string
+ tsigSecretSecretRef:
+ description: The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ route53:
+ description: Use the AWS Route53 API to manage DNS01 challenge records.
+ type: object
+ required:
+ - region
+ properties:
+ accessKeyID:
+ description: 'The AccessKeyID is used for authentication. If not set we fall-back to using env vars, shared credentials file or AWS Instance metadata see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials'
+ type: string
+ hostedZoneID:
+ description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call.
+ type: string
+ region:
+ description: Always set the region when using AccessKeyID and SecretAccessKey
+ type: string
+ role:
+ description: Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata
+ type: string
+ secretAccessKeySecretRef:
+ description: The SecretAccessKey is used for authentication. If not set we fall-back to using env vars, shared credentials file or AWS Instance metadata https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ webhook:
+ description: Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records.
+ type: object
+ required:
+ - groupName
+ - solverName
+ properties:
+ config:
+ description: Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation.
+ x-kubernetes-preserve-unknown-fields: true
+ groupName:
+ description: The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation.
+ type: string
+ solverName:
+ description: The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'.
+ type: string
+ http01:
+ description: Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism.
+ type: object
+ properties:
+ gatewayHTTPRoute:
+ description: The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future.
+ type: object
+ properties:
+ labels:
+ description: Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges.
+ type: object
+ additionalProperties:
+ type: string
+ parentRefs:
+ description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/v1alpha2/api-types/httproute/#attaching-to-gateways'
+ type: array
+ items:
+ description: "ParentRef identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). The only kind of parent resource with \"Core\" support is Gateway. This API may be extended in the future to support additional kinds of parent resources, such as HTTPRoute. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid. \n References to objects with invalid Group and Kind are not valid, and must be rejected by the implementation, with appropriate Conditions set on the containing object."
+ type: object
+ required:
+ - name
+ properties:
+ group:
+ description: "Group is the group of the referent. \n Support: Core"
+ type: string
+ default: gateway.networking.k8s.io
+ maxLength: 253
+ pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+ kind:
+ description: "Kind is kind of the referent. \n Support: Core (Gateway) Support: Custom (Other Resources)"
+ type: string
+ default: Gateway
+ maxLength: 63
+ minLength: 1
+ pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$
+ name:
+ description: "Name is the name of the referent. \n Support: Core"
+ type: string
+ maxLength: 253
+ minLength: 1
+ namespace:
+ description: "Namespace is the namespace of the referent. When unspecified (or empty string), this refers to the local namespace of the Route. \n Support: Core"
+ type: string
+ maxLength: 63
+ minLength: 1
+ pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+ sectionName:
+ description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core"
+ type: string
+ maxLength: 253
+ minLength: 1
+ pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+ serviceType:
+ description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort.
+ type: string
+ ingress:
+ description: The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed.
+ type: object
+ properties:
+ class:
+ description: The ingress class to use when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of 'class' or 'name' may be specified.
+ type: string
+ ingressTemplate:
+ description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges.
+ type: object
+ properties:
+ metadata:
+ description: ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values.
+ type: object
+ properties:
+ annotations:
+ description: Annotations that should be added to the created ACME HTTP01 solver ingress.
+ type: object
+ additionalProperties:
+ type: string
+ labels:
+ description: Labels that should be added to the created ACME HTTP01 solver ingress.
+ type: object
+ additionalProperties:
+ type: string
+ name:
+ description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources.
+ type: string
+ podTemplate:
+ description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges.
+ type: object
+ properties:
+ metadata:
+ description: ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values.
+ type: object
+ properties:
+ annotations:
+ description: Annotations that should be added to the create ACME HTTP01 solver pods.
+ type: object
+ additionalProperties:
+ type: string
+ labels:
+ description: Labels that should be added to the created ACME HTTP01 solver pods.
+ type: object
+ additionalProperties:
+ type: string
+ spec:
+ description: PodSpec defines overrides for the HTTP01 challenge solver pod. Only the 'priorityClassName', 'nodeSelector', 'affinity', 'serviceAccountName' and 'tolerations' fields are supported currently. All other fields will be ignored.
+ type: object
+ properties:
+ affinity:
+ description: If specified, the pod's scheduling constraints
+ type: object
+ properties:
+ nodeAffinity:
+ description: Describes node affinity scheduling rules for the pod.
+ type: object
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.
+ type: array
+ items:
+ description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).
+ type: object
+ required:
+ - preference
+ - weight
+ properties:
+ preference:
+ description: A node selector term, associated with the corresponding weight.
+ type: object
+ properties:
+ matchExpressions:
+ description: A list of node selector requirements by node's labels.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchFields:
+ description: A list of node selector requirements by node's fields.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ weight:
+ description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.
+ type: integer
+ format: int32
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node.
+ type: object
+ required:
+ - nodeSelectorTerms
+ properties:
+ nodeSelectorTerms:
+ description: Required. A list of node selector terms. The terms are ORed.
+ type: array
+ items:
+ description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
+ type: object
+ properties:
+ matchExpressions:
+ description: A list of node selector requirements by node's labels.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchFields:
+ description: A list of node selector requirements by node's fields.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ podAffinity:
+ description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).
+ type: object
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.
+ type: array
+ items:
+ description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
+ type: object
+ required:
+ - podAffinityTerm
+ - weight
+ properties:
+ podAffinityTerm:
+ description: Required. A pod affinity term, associated with the corresponding weight.
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ weight:
+ description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.
+ type: integer
+ format: int32
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ type: array
+ items:
+ description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ podAntiAffinity:
+ description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).
+ type: object
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.
+ type: array
+ items:
+ description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
+ type: object
+ required:
+ - podAffinityTerm
+ - weight
+ properties:
+ podAffinityTerm:
+ description: Required. A pod affinity term, associated with the corresponding weight.
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ weight:
+ description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.
+ type: integer
+ format: int32
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ type: array
+ items:
+ description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ nodeSelector:
+ description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/'
+ type: object
+ additionalProperties:
+ type: string
+ priorityClassName:
+ description: If specified, the pod's priorityClassName.
+ type: string
+ serviceAccountName:
+ description: If specified, the pod's service account
+ type: string
+ tolerations:
+ description: If specified, the pod's tolerations.
+ type: array
+ items:
+ description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator .
+ type: object
+ properties:
+ effect:
+ description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+ type: string
+ key:
+ description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+ type: string
+ operator:
+ description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.
+ type: string
+ tolerationSeconds:
+ description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.
+ type: integer
+ format: int64
+ value:
+ description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.
+ type: string
+ serviceType:
+ description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort.
+ type: string
+ selector:
+ description: Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead.
+ type: object
+ properties:
+ dnsNames:
+ description: List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected.
+ type: array
+ items:
+ type: string
+ dnsZones:
+ description: List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: A label selector that is used to refine the set of certificate's that this challenge solver will apply to.
+ type: object
+ additionalProperties:
+ type: string
+ token:
+ description: The ACME challenge token for this challenge. This is the raw value returned from the ACME server.
+ type: string
+ type:
+ description: The type of ACME challenge this resource represents. One of "HTTP-01" or "DNS-01".
+ type: string
+ enum:
+ - HTTP-01
+ - DNS-01
+ url:
+ description: The URL of the ACME Challenge resource for this challenge. This can be used to lookup details about the status of this challenge.
+ type: string
+ wildcard:
+ description: wildcard will be true if this challenge is for a wildcard identifier, for example '*.example.com'.
+ type: boolean
+ status:
+ type: object
+ properties:
+ presented:
+ description: presented will be set to true if the challenge values for this challenge are currently 'presented'. This *does not* imply the self check is passing. Only that the values have been 'submitted' for the appropriate challenge mechanism (i.e. the DNS01 TXT record has been presented, or the HTTP01 configuration has been configured).
+ type: boolean
+ processing:
+ description: Used to denote whether this challenge should be processed or not. This field will only be set to true by the 'scheduling' component. It will only be set to false by the 'challenges' controller, after the challenge has reached a final state or timed out. If this field is set to false, the challenge controller will not take any more action.
+ type: boolean
+ reason:
+ description: Contains human readable information on why the Challenge is in the current state.
+ type: string
+ state:
+ description: Contains the current 'state' of the challenge. If not set, the state of the challenge is unknown.
+ type: string
+ enum:
+ - valid
+ - ready
+ - pending
+ - processing
+ - invalid
+ - expired
+ - errored
+ served: true
+ storage: true
+ subresources:
+ status: {}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ name: clusterissuers.cert-manager.io
+ labels:
+ app: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/name: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/instance: '{{ .Release.Name }}'
+ # Generated labels {{- include "labels" . | nindent 4 }}
+spec:
+ group: cert-manager.io
+ names:
+ kind: ClusterIssuer
+ listKind: ClusterIssuerList
+ plural: clusterissuers
+ singular: clusterissuer
+ categories:
+ - cert-manager
+ scope: Cluster
+ versions:
+ - name: v1
+ subresources:
+ status: {}
+ additionalPrinterColumns:
+ - jsonPath: .status.conditions[?(@.type=="Ready")].status
+ name: Ready
+ type: string
+ - jsonPath: .status.conditions[?(@.type=="Ready")].message
+ name: Status
+ priority: 1
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.
+ name: Age
+ type: date
+ schema:
+ openAPIV3Schema:
+ description: A ClusterIssuer represents a certificate issuing authority which can be referenced as part of `issuerRef` fields. It is similar to an Issuer, however it is cluster-scoped and therefore can be referenced by resources that exist in *any* namespace, not just the same namespace as the referent.
+ type: object
+ required:
+ - spec
+ properties:
+ apiVersion:
+ description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+ type: string
+ kind:
+ description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: Desired state of the ClusterIssuer resource.
+ type: object
+ properties:
+ acme:
+ description: ACME configures this issuer to communicate with a RFC8555 (ACME) server to obtain signed x509 certificates.
+ type: object
+ required:
+ - privateKeySecretRef
+ - server
+ properties:
+ disableAccountKeyGeneration:
+ description: Enables or disables generating a new ACME account key. If true, the Issuer resource will *not* request a new account but will expect the account key to be supplied via an existing secret. If false, the cert-manager system will generate a new ACME account key for the Issuer. Defaults to false.
+ type: boolean
+ email:
+ description: Email is the email address to be associated with the ACME account. This field is optional, but it is strongly recommended to be set. It will be used to contact you in case of issues with your account or certificates, including expiry notification emails. This field may be updated after the account is initially registered.
+ type: string
+ enableDurationFeature:
+ description: Enables requesting a Not After date on certificates that matches the duration of the certificate. This is not supported by all ACME servers like Let's Encrypt. If set to true when the ACME server does not support it it will create an error on the Order. Defaults to false.
+ type: boolean
+ externalAccountBinding:
+ description: ExternalAccountBinding is a reference to a CA external account of the ACME server. If set, upon registration cert-manager will attempt to associate the given external account credentials with the registered ACME account.
+ type: object
+ required:
+ - keyID
+ - keySecretRef
+ properties:
+ keyAlgorithm:
+ description: 'Deprecated: keyAlgorithm field exists for historical compatibility reasons and should not be used. The algorithm is now hardcoded to HS256 in golang/x/crypto/acme.'
+ type: string
+ enum:
+ - HS256
+ - HS384
+ - HS512
+ keyID:
+ description: keyID is the ID of the CA key that the External Account is bound to.
+ type: string
+ keySecretRef:
+ description: keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes Secret which holds the symmetric MAC key of the External Account Binding. The `key` is the index string that is paired with the key data in the Secret and should not be confused with the key data itself, or indeed with the External Account Binding keyID above. The secret key stored in the Secret **must** be un-padded, base64 URL encoded data.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ preferredChain:
+ description: 'PreferredChain is the chain to use if the ACME server outputs multiple. PreferredChain is no guarantee that this one gets delivered by the ACME endpoint. For example, for Let''s Encrypt''s DST crosssign you would use: "DST Root CA X3" or "ISRG Root X1" for the newer Let''s Encrypt root CA. This value picks the first certificate bundle in the ACME alternative chains that has a certificate with this value as its issuer''s CN'
+ type: string
+ maxLength: 64
+ privateKeySecretRef:
+ description: PrivateKey is the name of a Kubernetes Secret resource that will be used to store the automatically generated ACME account private key. Optionally, a `key` may be specified to select a specific entry within the named Secret resource. If `key` is not specified, a default of `tls.key` will be used.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ server:
+ description: 'Server is the URL used to access the ACME server''s ''directory'' endpoint. For example, for Let''s Encrypt''s staging endpoint, you would use: "https://acme-staging-v02.api.letsencrypt.org/directory". Only ACME v2 endpoints (i.e. RFC 8555) are supported.'
+ type: string
+ skipTLSVerify:
+ description: Enables or disables validation of the ACME server TLS certificate. If true, requests to the ACME server will not have their TLS certificate validated (i.e. insecure connections will be allowed). Only enable this option in development environments. The cert-manager system installed roots will be used to verify connections to the ACME server if this is false. Defaults to false.
+ type: boolean
+ solvers:
+ description: 'Solvers is a list of challenge solvers that will be used to solve ACME challenges for the matching domains. Solver configurations must be provided in order to obtain certificates from an ACME server. For more information, see: https://cert-manager.io/docs/configuration/acme/'
+ type: array
+ items:
+ description: An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of. A selector may be provided to use different solving strategies for different DNS names. Only one of HTTP01 or DNS01 must be provided.
+ type: object
+ properties:
+ dns01:
+ description: Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow.
+ type: object
+ properties:
+ acmeDNS:
+ description: Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records.
+ type: object
+ required:
+ - accountSecretRef
+ - host
+ properties:
+ accountSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ host:
+ type: string
+ akamai:
+ description: Use the Akamai DNS zone management API to manage DNS01 challenge records.
+ type: object
+ required:
+ - accessTokenSecretRef
+ - clientSecretSecretRef
+ - clientTokenSecretRef
+ - serviceConsumerDomain
+ properties:
+ accessTokenSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ clientSecretSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ clientTokenSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ serviceConsumerDomain:
+ type: string
+ azureDNS:
+ description: Use the Microsoft Azure DNS API to manage DNS01 challenge records.
+ type: object
+ required:
+ - resourceGroupName
+ - subscriptionID
+ properties:
+ clientID:
+ description: if both this and ClientSecret are left unset MSI will be used
+ type: string
+ clientSecretSecretRef:
+ description: if both this and ClientID are left unset MSI will be used
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ environment:
+ description: name of the Azure environment (default AzurePublicCloud)
+ type: string
+ enum:
+ - AzurePublicCloud
+ - AzureChinaCloud
+ - AzureGermanCloud
+ - AzureUSGovernmentCloud
+ hostedZoneName:
+ description: name of the DNS zone that should be used
+ type: string
+ managedIdentity:
+ description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID
+ type: object
+ properties:
+ clientID:
+ description: client ID of the managed identity, can not be used at the same time as resourceID
+ type: string
+ resourceID:
+ description: resource ID of the managed identity, can not be used at the same time as clientID
+ type: string
+ resourceGroupName:
+ description: resource group the DNS zone is located in
+ type: string
+ subscriptionID:
+ description: ID of the Azure subscription
+ type: string
+ tenantID:
+ description: when specifying ClientID and ClientSecret then this field is also needed
+ type: string
+ cloudDNS:
+ description: Use the Google Cloud DNS API to manage DNS01 challenge records.
+ type: object
+ required:
+ - project
+ properties:
+ hostedZoneName:
+ description: HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone.
+ type: string
+ project:
+ type: string
+ serviceAccountSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ cloudflare:
+ description: Use the Cloudflare API to manage DNS01 challenge records.
+ type: object
+ properties:
+ apiKeySecretRef:
+ description: 'API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions.'
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ apiTokenSecretRef:
+ description: API token used to authenticate with Cloudflare.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ email:
+ description: Email of the account, only required when using API key based authentication.
+ type: string
+ cnameStrategy:
+ description: CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones.
+ type: string
+ enum:
+ - None
+ - Follow
+ digitalocean:
+ description: Use the DigitalOcean DNS API to manage DNS01 challenge records.
+ type: object
+ required:
+ - tokenSecretRef
+ properties:
+ tokenSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ rfc2136:
+ description: Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records.
+ type: object
+ required:
+ - nameserver
+ properties:
+ nameserver:
+ description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1])Â ; port is optional. This field is required.
+ type: string
+ tsigAlgorithm:
+ description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.'
+ type: string
+ tsigKeyName:
+ description: The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required.
+ type: string
+ tsigSecretSecretRef:
+ description: The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ route53:
+ description: Use the AWS Route53 API to manage DNS01 challenge records.
+ type: object
+ required:
+ - region
+ properties:
+ accessKeyID:
+ description: 'The AccessKeyID is used for authentication. If not set we fall-back to using env vars, shared credentials file or AWS Instance metadata see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials'
+ type: string
+ hostedZoneID:
+ description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call.
+ type: string
+ region:
+ description: Always set the region when using AccessKeyID and SecretAccessKey
+ type: string
+ role:
+ description: Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata
+ type: string
+ secretAccessKeySecretRef:
+ description: The SecretAccessKey is used for authentication. If not set we fall-back to using env vars, shared credentials file or AWS Instance metadata https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ webhook:
+ description: Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records.
+ type: object
+ required:
+ - groupName
+ - solverName
+ properties:
+ config:
+ description: Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation.
+ x-kubernetes-preserve-unknown-fields: true
+ groupName:
+ description: The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation.
+ type: string
+ solverName:
+ description: The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'.
+ type: string
+ http01:
+ description: Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism.
+ type: object
+ properties:
+ gatewayHTTPRoute:
+ description: The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future.
+ type: object
+ properties:
+ labels:
+ description: Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges.
+ type: object
+ additionalProperties:
+ type: string
+ parentRefs:
+ description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/v1alpha2/api-types/httproute/#attaching-to-gateways'
+ type: array
+ items:
+ description: "ParentRef identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). The only kind of parent resource with \"Core\" support is Gateway. This API may be extended in the future to support additional kinds of parent resources, such as HTTPRoute. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid. \n References to objects with invalid Group and Kind are not valid, and must be rejected by the implementation, with appropriate Conditions set on the containing object."
+ type: object
+ required:
+ - name
+ properties:
+ group:
+ description: "Group is the group of the referent. \n Support: Core"
+ type: string
+ default: gateway.networking.k8s.io
+ maxLength: 253
+ pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+ kind:
+ description: "Kind is kind of the referent. \n Support: Core (Gateway) Support: Custom (Other Resources)"
+ type: string
+ default: Gateway
+ maxLength: 63
+ minLength: 1
+ pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$
+ name:
+ description: "Name is the name of the referent. \n Support: Core"
+ type: string
+ maxLength: 253
+ minLength: 1
+ namespace:
+ description: "Namespace is the namespace of the referent. When unspecified (or empty string), this refers to the local namespace of the Route. \n Support: Core"
+ type: string
+ maxLength: 63
+ minLength: 1
+ pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+ sectionName:
+ description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core"
+ type: string
+ maxLength: 253
+ minLength: 1
+ pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+ serviceType:
+ description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort.
+ type: string
+ ingress:
+ description: The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed.
+ type: object
+ properties:
+ class:
+ description: The ingress class to use when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of 'class' or 'name' may be specified.
+ type: string
+ ingressTemplate:
+ description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges.
+ type: object
+ properties:
+ metadata:
+ description: ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values.
+ type: object
+ properties:
+ annotations:
+ description: Annotations that should be added to the created ACME HTTP01 solver ingress.
+ type: object
+ additionalProperties:
+ type: string
+ labels:
+ description: Labels that should be added to the created ACME HTTP01 solver ingress.
+ type: object
+ additionalProperties:
+ type: string
+ name:
+ description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources.
+ type: string
+ podTemplate:
+ description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges.
+ type: object
+ properties:
+ metadata:
+ description: ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values.
+ type: object
+ properties:
+ annotations:
+ description: Annotations that should be added to the create ACME HTTP01 solver pods.
+ type: object
+ additionalProperties:
+ type: string
+ labels:
+ description: Labels that should be added to the created ACME HTTP01 solver pods.
+ type: object
+ additionalProperties:
+ type: string
+ spec:
+ description: PodSpec defines overrides for the HTTP01 challenge solver pod. Only the 'priorityClassName', 'nodeSelector', 'affinity', 'serviceAccountName' and 'tolerations' fields are supported currently. All other fields will be ignored.
+ type: object
+ properties:
+ affinity:
+ description: If specified, the pod's scheduling constraints
+ type: object
+ properties:
+ nodeAffinity:
+ description: Describes node affinity scheduling rules for the pod.
+ type: object
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.
+ type: array
+ items:
+ description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).
+ type: object
+ required:
+ - preference
+ - weight
+ properties:
+ preference:
+ description: A node selector term, associated with the corresponding weight.
+ type: object
+ properties:
+ matchExpressions:
+ description: A list of node selector requirements by node's labels.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchFields:
+ description: A list of node selector requirements by node's fields.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ weight:
+ description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.
+ type: integer
+ format: int32
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node.
+ type: object
+ required:
+ - nodeSelectorTerms
+ properties:
+ nodeSelectorTerms:
+ description: Required. A list of node selector terms. The terms are ORed.
+ type: array
+ items:
+ description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
+ type: object
+ properties:
+ matchExpressions:
+ description: A list of node selector requirements by node's labels.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchFields:
+ description: A list of node selector requirements by node's fields.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ podAffinity:
+ description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).
+ type: object
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.
+ type: array
+ items:
+ description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
+ type: object
+ required:
+ - podAffinityTerm
+ - weight
+ properties:
+ podAffinityTerm:
+ description: Required. A pod affinity term, associated with the corresponding weight.
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ weight:
+ description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.
+ type: integer
+ format: int32
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ type: array
+ items:
+ description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ podAntiAffinity:
+ description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).
+ type: object
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.
+ type: array
+ items:
+ description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
+ type: object
+ required:
+ - podAffinityTerm
+ - weight
+ properties:
+ podAffinityTerm:
+ description: Required. A pod affinity term, associated with the corresponding weight.
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ weight:
+ description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.
+ type: integer
+ format: int32
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ type: array
+ items:
+ description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ nodeSelector:
+ description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/'
+ type: object
+ additionalProperties:
+ type: string
+ priorityClassName:
+ description: If specified, the pod's priorityClassName.
+ type: string
+ serviceAccountName:
+ description: If specified, the pod's service account
+ type: string
+ tolerations:
+ description: If specified, the pod's tolerations.
+ type: array
+ items:
+ description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator .
+ type: object
+ properties:
+ effect:
+ description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+ type: string
+ key:
+ description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+ type: string
+ operator:
+ description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.
+ type: string
+ tolerationSeconds:
+ description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.
+ type: integer
+ format: int64
+ value:
+ description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.
+ type: string
+ serviceType:
+ description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort.
+ type: string
+ selector:
+ description: Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead.
+ type: object
+ properties:
+ dnsNames:
+ description: List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected.
+ type: array
+ items:
+ type: string
+ dnsZones:
+ description: List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: A label selector that is used to refine the set of certificate's that this challenge solver will apply to.
+ type: object
+ additionalProperties:
+ type: string
+ ca:
+ description: CA configures this issuer to sign certificates using a signing CA keypair stored in a Secret resource. This is used to build internal PKIs that are managed by cert-manager.
+ type: object
+ required:
+ - secretName
+ properties:
+ crlDistributionPoints:
+ description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set, certificates will be issued without distribution points set.
+ type: array
+ items:
+ type: string
+ ocspServers:
+ description: The OCSP server list is an X.509 v3 extension that defines a list of URLs of OCSP responders. The OCSP responders can be queried for the revocation status of an issued certificate. If not set, the certificate will be issued with no OCSP servers set. For example, an OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org".
+ type: array
+ items:
+ type: string
+ secretName:
+ description: SecretName is the name of the secret used to sign Certificates issued by this Issuer.
+ type: string
+ selfSigned:
+ description: SelfSigned configures this issuer to 'self sign' certificates using the private key used to create the CertificateRequest object.
+ type: object
+ properties:
+ crlDistributionPoints:
+ description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set certificate will be issued without CDP. Values are strings.
+ type: array
+ items:
+ type: string
+ vault:
+ description: Vault configures this issuer to sign certificates using a HashiCorp Vault PKI backend.
+ type: object
+ required:
+ - auth
+ - path
+ - server
+ properties:
+ auth:
+ description: Auth configures how cert-manager authenticates with the Vault server.
+ type: object
+ properties:
+ appRole:
+ description: AppRole authenticates with Vault using the App Role auth mechanism, with the role and secret stored in a Kubernetes Secret resource.
+ type: object
+ required:
+ - path
+ - roleId
+ - secretRef
+ properties:
+ path:
+ description: 'Path where the App Role authentication backend is mounted in Vault, e.g: "approle"'
+ type: string
+ roleId:
+ description: RoleID configured in the App Role authentication backend when setting up the authentication backend in Vault.
+ type: string
+ secretRef:
+ description: Reference to a key in a Secret that contains the App Role secret used to authenticate with Vault. The `key` field must be specified and denotes which entry within the Secret resource is used as the app role secret.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ kubernetes:
+ description: Kubernetes authenticates with Vault by passing the ServiceAccount token stored in the named Secret resource to the Vault server.
+ type: object
+ required:
+ - role
+ - secretRef
+ properties:
+ mountPath:
+ description: The Vault mountPath here is the mount path to use when authenticating with Vault. For example, setting a value to `/v1/auth/foo`, will use the path `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the default value "/v1/auth/kubernetes" will be used.
+ type: string
+ role:
+ description: A required field containing the Vault Role to assume. A Role binds a Kubernetes ServiceAccount with a set of Vault policies.
+ type: string
+ secretRef:
+ description: The required Secret field containing a Kubernetes ServiceAccount JWT used for authenticating with Vault. Use of 'ambient credentials' is not supported.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ tokenSecretRef:
+ description: TokenSecretRef authenticates with Vault by presenting a token.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ caBundle:
+ description: PEM-encoded CA bundle (base64-encoded) used to validate Vault server certificate. Only used if the Server URL is using HTTPS protocol. This parameter is ignored for plain HTTP protocol connection. If not set the system root certificates are used to validate the TLS connection.
+ type: string
+ format: byte
+ namespace:
+ description: 'Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: "ns1" More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces'
+ type: string
+ path:
+ description: 'Path is the mount path of the Vault PKI backend''s `sign` endpoint, e.g: "my_pki_mount/sign/my-role-name".'
+ type: string
+ server:
+ description: 'Server is the connection address for the Vault server, e.g: "https://vault.example.com:8200".'
+ type: string
+ venafi:
+ description: Venafi configures this issuer to sign certificates using a Venafi TPP or Venafi Cloud policy zone.
+ type: object
+ required:
+ - zone
+ properties:
+ cloud:
+ description: Cloud specifies the Venafi cloud configuration settings. Only one of TPP or Cloud may be specified.
+ type: object
+ required:
+ - apiTokenSecretRef
+ properties:
+ apiTokenSecretRef:
+ description: APITokenSecretRef is a secret key selector for the Venafi Cloud API token.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ url:
+ description: URL is the base URL for Venafi Cloud. Defaults to "https://api.venafi.cloud/v1".
+ type: string
+ tpp:
+ description: TPP specifies Trust Protection Platform configuration settings. Only one of TPP or Cloud may be specified.
+ type: object
+ required:
+ - credentialsRef
+ - url
+ properties:
+ caBundle:
+ description: CABundle is a PEM encoded TLS certificate to use to verify connections to the TPP instance. If specified, system roots will not be used and the issuing CA for the TPP instance must be verifiable using the provided root. If not specified, the connection will be verified using the cert-manager system root certificates.
+ type: string
+ format: byte
+ credentialsRef:
+ description: CredentialsRef is a reference to a Secret containing the username and password for the TPP server. The secret must contain two keys, 'username' and 'password'.
+ type: object
+ required:
+ - name
+ properties:
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ url:
+ description: 'URL is the base URL for the vedsdk endpoint of the Venafi TPP instance, for example: "https://tpp.example.com/vedsdk".'
+ type: string
+ zone:
+ description: Zone is the Venafi Policy Zone to use for this issuer. All requests made to the Venafi platform will be restricted by the named zone policy. This field is required.
+ type: string
+ status:
+ description: Status of the ClusterIssuer. This is set and managed automatically.
+ type: object
+ properties:
+ acme:
+ description: ACME specific status options. This field should only be set if the Issuer is configured to use an ACME server to issue certificates.
+ type: object
+ properties:
+ lastRegisteredEmail:
+ description: LastRegisteredEmail is the email associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer
+ type: string
+ uri:
+ description: URI is the unique account identifier, which can also be used to retrieve account details from the CA
+ type: string
+ conditions:
+ description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`.
+ type: array
+ items:
+ description: IssuerCondition contains condition information for an Issuer.
+ type: object
+ required:
+ - status
+ - type
+ properties:
+ lastTransitionTime:
+ description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+ type: string
+ format: date-time
+ message:
+ description: Message is a human readable description of the details of the last transition, complementing reason.
+ type: string
+ observedGeneration:
+ description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Issuer.
+ type: integer
+ format: int64
+ reason:
+ description: Reason is a brief machine readable explanation for the condition's last transition.
+ type: string
+ status:
+ description: Status of the condition, one of (`True`, `False`, `Unknown`).
+ type: string
+ enum:
+ - "True"
+ - "False"
+ - Unknown
+ type:
+ description: Type of the condition, known values are (`Ready`).
+ type: string
+ x-kubernetes-list-map-keys:
+ - type
+ x-kubernetes-list-type: map
+ served: true
+ storage: true
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ name: issuers.cert-manager.io
+ annotations:
+ cert-manager.io/inject-ca-from-secret: '{{ template "webhook.caRef" . }}'
+ labels:
+ app: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/name: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/instance: '{{ .Release.Name }}'
+ # Generated labels {{- include "labels" . | nindent 4 }}
+spec:
+ group: cert-manager.io
+ names:
+ kind: Issuer
+ listKind: IssuerList
+ plural: issuers
+ singular: issuer
+ categories:
+ - cert-manager
+ scope: Namespaced
+ versions:
+ - name: v1
+ subresources:
+ status: {}
+ additionalPrinterColumns:
+ - jsonPath: .status.conditions[?(@.type=="Ready")].status
+ name: Ready
+ type: string
+ - jsonPath: .status.conditions[?(@.type=="Ready")].message
+ name: Status
+ priority: 1
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.
+ name: Age
+ type: date
+ schema:
+ openAPIV3Schema:
+ description: An Issuer represents a certificate issuing authority which can be referenced as part of `issuerRef` fields. It is scoped to a single namespace and can therefore only be referenced by resources within the same namespace.
+ type: object
+ required:
+ - spec
+ properties:
+ apiVersion:
+ description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+ type: string
+ kind:
+ description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: Desired state of the Issuer resource.
+ type: object
+ properties:
+ acme:
+ description: ACME configures this issuer to communicate with a RFC8555 (ACME) server to obtain signed x509 certificates.
+ type: object
+ required:
+ - privateKeySecretRef
+ - server
+ properties:
+ disableAccountKeyGeneration:
+ description: Enables or disables generating a new ACME account key. If true, the Issuer resource will *not* request a new account but will expect the account key to be supplied via an existing secret. If false, the cert-manager system will generate a new ACME account key for the Issuer. Defaults to false.
+ type: boolean
+ email:
+ description: Email is the email address to be associated with the ACME account. This field is optional, but it is strongly recommended to be set. It will be used to contact you in case of issues with your account or certificates, including expiry notification emails. This field may be updated after the account is initially registered.
+ type: string
+ enableDurationFeature:
+ description: Enables requesting a Not After date on certificates that matches the duration of the certificate. This is not supported by all ACME servers like Let's Encrypt. If set to true when the ACME server does not support it it will create an error on the Order. Defaults to false.
+ type: boolean
+ externalAccountBinding:
+ description: ExternalAccountBinding is a reference to a CA external account of the ACME server. If set, upon registration cert-manager will attempt to associate the given external account credentials with the registered ACME account.
+ type: object
+ required:
+ - keyID
+ - keySecretRef
+ properties:
+ keyAlgorithm:
+ description: 'Deprecated: keyAlgorithm field exists for historical compatibility reasons and should not be used. The algorithm is now hardcoded to HS256 in golang/x/crypto/acme.'
+ type: string
+ enum:
+ - HS256
+ - HS384
+ - HS512
+ keyID:
+ description: keyID is the ID of the CA key that the External Account is bound to.
+ type: string
+ keySecretRef:
+ description: keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes Secret which holds the symmetric MAC key of the External Account Binding. The `key` is the index string that is paired with the key data in the Secret and should not be confused with the key data itself, or indeed with the External Account Binding keyID above. The secret key stored in the Secret **must** be un-padded, base64 URL encoded data.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ preferredChain:
+ description: 'PreferredChain is the chain to use if the ACME server outputs multiple. PreferredChain is no guarantee that this one gets delivered by the ACME endpoint. For example, for Let''s Encrypt''s DST crosssign you would use: "DST Root CA X3" or "ISRG Root X1" for the newer Let''s Encrypt root CA. This value picks the first certificate bundle in the ACME alternative chains that has a certificate with this value as its issuer''s CN'
+ type: string
+ maxLength: 64
+ privateKeySecretRef:
+ description: PrivateKey is the name of a Kubernetes Secret resource that will be used to store the automatically generated ACME account private key. Optionally, a `key` may be specified to select a specific entry within the named Secret resource. If `key` is not specified, a default of `tls.key` will be used.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ server:
+ description: 'Server is the URL used to access the ACME server''s ''directory'' endpoint. For example, for Let''s Encrypt''s staging endpoint, you would use: "https://acme-staging-v02.api.letsencrypt.org/directory". Only ACME v2 endpoints (i.e. RFC 8555) are supported.'
+ type: string
+ skipTLSVerify:
+ description: Enables or disables validation of the ACME server TLS certificate. If true, requests to the ACME server will not have their TLS certificate validated (i.e. insecure connections will be allowed). Only enable this option in development environments. The cert-manager system installed roots will be used to verify connections to the ACME server if this is false. Defaults to false.
+ type: boolean
+ solvers:
+ description: 'Solvers is a list of challenge solvers that will be used to solve ACME challenges for the matching domains. Solver configurations must be provided in order to obtain certificates from an ACME server. For more information, see: https://cert-manager.io/docs/configuration/acme/'
+ type: array
+ items:
+ description: An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of. A selector may be provided to use different solving strategies for different DNS names. Only one of HTTP01 or DNS01 must be provided.
+ type: object
+ properties:
+ dns01:
+ description: Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow.
+ type: object
+ properties:
+ acmeDNS:
+ description: Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records.
+ type: object
+ required:
+ - accountSecretRef
+ - host
+ properties:
+ accountSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ host:
+ type: string
+ akamai:
+ description: Use the Akamai DNS zone management API to manage DNS01 challenge records.
+ type: object
+ required:
+ - accessTokenSecretRef
+ - clientSecretSecretRef
+ - clientTokenSecretRef
+ - serviceConsumerDomain
+ properties:
+ accessTokenSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ clientSecretSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ clientTokenSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ serviceConsumerDomain:
+ type: string
+ azureDNS:
+ description: Use the Microsoft Azure DNS API to manage DNS01 challenge records.
+ type: object
+ required:
+ - resourceGroupName
+ - subscriptionID
+ properties:
+ clientID:
+ description: if both this and ClientSecret are left unset MSI will be used
+ type: string
+ clientSecretSecretRef:
+ description: if both this and ClientID are left unset MSI will be used
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ environment:
+ description: name of the Azure environment (default AzurePublicCloud)
+ type: string
+ enum:
+ - AzurePublicCloud
+ - AzureChinaCloud
+ - AzureGermanCloud
+ - AzureUSGovernmentCloud
+ hostedZoneName:
+ description: name of the DNS zone that should be used
+ type: string
+ managedIdentity:
+ description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID
+ type: object
+ properties:
+ clientID:
+ description: client ID of the managed identity, can not be used at the same time as resourceID
+ type: string
+ resourceID:
+ description: resource ID of the managed identity, can not be used at the same time as clientID
+ type: string
+ resourceGroupName:
+ description: resource group the DNS zone is located in
+ type: string
+ subscriptionID:
+ description: ID of the Azure subscription
+ type: string
+ tenantID:
+ description: when specifying ClientID and ClientSecret then this field is also needed
+ type: string
+ cloudDNS:
+ description: Use the Google Cloud DNS API to manage DNS01 challenge records.
+ type: object
+ required:
+ - project
+ properties:
+ hostedZoneName:
+ description: HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone.
+ type: string
+ project:
+ type: string
+ serviceAccountSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ cloudflare:
+ description: Use the Cloudflare API to manage DNS01 challenge records.
+ type: object
+ properties:
+ apiKeySecretRef:
+ description: 'API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions.'
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ apiTokenSecretRef:
+ description: API token used to authenticate with Cloudflare.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ email:
+ description: Email of the account, only required when using API key based authentication.
+ type: string
+ cnameStrategy:
+ description: CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones.
+ type: string
+ enum:
+ - None
+ - Follow
+ digitalocean:
+ description: Use the DigitalOcean DNS API to manage DNS01 challenge records.
+ type: object
+ required:
+ - tokenSecretRef
+ properties:
+ tokenSecretRef:
+ description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ rfc2136:
+ description: Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records.
+ type: object
+ required:
+ - nameserver
+ properties:
+ nameserver:
+ description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1])Â ; port is optional. This field is required.
+ type: string
+ tsigAlgorithm:
+ description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.'
+ type: string
+ tsigKeyName:
+ description: The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required.
+ type: string
+ tsigSecretSecretRef:
+ description: The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ route53:
+ description: Use the AWS Route53 API to manage DNS01 challenge records.
+ type: object
+ required:
+ - region
+ properties:
+ accessKeyID:
+ description: 'The AccessKeyID is used for authentication. If not set we fall-back to using env vars, shared credentials file or AWS Instance metadata see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials'
+ type: string
+ hostedZoneID:
+ description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call.
+ type: string
+ region:
+ description: Always set the region when using AccessKeyID and SecretAccessKey
+ type: string
+ role:
+ description: Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata
+ type: string
+ secretAccessKeySecretRef:
+ description: The SecretAccessKey is used for authentication. If not set we fall-back to using env vars, shared credentials file or AWS Instance metadata https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ webhook:
+ description: Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records.
+ type: object
+ required:
+ - groupName
+ - solverName
+ properties:
+ config:
+ description: Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation.
+ x-kubernetes-preserve-unknown-fields: true
+ groupName:
+ description: The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation.
+ type: string
+ solverName:
+ description: The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'.
+ type: string
+ http01:
+ description: Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism.
+ type: object
+ properties:
+ gatewayHTTPRoute:
+ description: The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future.
+ type: object
+ properties:
+ labels:
+ description: Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges.
+ type: object
+ additionalProperties:
+ type: string
+ parentRefs:
+ description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/v1alpha2/api-types/httproute/#attaching-to-gateways'
+ type: array
+ items:
+ description: "ParentRef identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). The only kind of parent resource with \"Core\" support is Gateway. This API may be extended in the future to support additional kinds of parent resources, such as HTTPRoute. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid. \n References to objects with invalid Group and Kind are not valid, and must be rejected by the implementation, with appropriate Conditions set on the containing object."
+ type: object
+ required:
+ - name
+ properties:
+ group:
+ description: "Group is the group of the referent. \n Support: Core"
+ type: string
+ default: gateway.networking.k8s.io
+ maxLength: 253
+ pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+ kind:
+ description: "Kind is kind of the referent. \n Support: Core (Gateway) Support: Custom (Other Resources)"
+ type: string
+ default: Gateway
+ maxLength: 63
+ minLength: 1
+ pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$
+ name:
+ description: "Name is the name of the referent. \n Support: Core"
+ type: string
+ maxLength: 253
+ minLength: 1
+ namespace:
+ description: "Namespace is the namespace of the referent. When unspecified (or empty string), this refers to the local namespace of the Route. \n Support: Core"
+ type: string
+ maxLength: 63
+ minLength: 1
+ pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+ sectionName:
+ description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core"
+ type: string
+ maxLength: 253
+ minLength: 1
+ pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+ serviceType:
+ description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort.
+ type: string
+ ingress:
+ description: The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed.
+ type: object
+ properties:
+ class:
+ description: The ingress class to use when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of 'class' or 'name' may be specified.
+ type: string
+ ingressTemplate:
+ description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges.
+ type: object
+ properties:
+ metadata:
+ description: ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values.
+ type: object
+ properties:
+ annotations:
+ description: Annotations that should be added to the created ACME HTTP01 solver ingress.
+ type: object
+ additionalProperties:
+ type: string
+ labels:
+ description: Labels that should be added to the created ACME HTTP01 solver ingress.
+ type: object
+ additionalProperties:
+ type: string
+ name:
+ description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources.
+ type: string
+ podTemplate:
+ description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges.
+ type: object
+ properties:
+ metadata:
+ description: ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values.
+ type: object
+ properties:
+ annotations:
+ description: Annotations that should be added to the create ACME HTTP01 solver pods.
+ type: object
+ additionalProperties:
+ type: string
+ labels:
+ description: Labels that should be added to the created ACME HTTP01 solver pods.
+ type: object
+ additionalProperties:
+ type: string
+ spec:
+ description: PodSpec defines overrides for the HTTP01 challenge solver pod. Only the 'priorityClassName', 'nodeSelector', 'affinity', 'serviceAccountName' and 'tolerations' fields are supported currently. All other fields will be ignored.
+ type: object
+ properties:
+ affinity:
+ description: If specified, the pod's scheduling constraints
+ type: object
+ properties:
+ nodeAffinity:
+ description: Describes node affinity scheduling rules for the pod.
+ type: object
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.
+ type: array
+ items:
+ description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).
+ type: object
+ required:
+ - preference
+ - weight
+ properties:
+ preference:
+ description: A node selector term, associated with the corresponding weight.
+ type: object
+ properties:
+ matchExpressions:
+ description: A list of node selector requirements by node's labels.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchFields:
+ description: A list of node selector requirements by node's fields.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ weight:
+ description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.
+ type: integer
+ format: int32
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node.
+ type: object
+ required:
+ - nodeSelectorTerms
+ properties:
+ nodeSelectorTerms:
+ description: Required. A list of node selector terms. The terms are ORed.
+ type: array
+ items:
+ description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
+ type: object
+ properties:
+ matchExpressions:
+ description: A list of node selector requirements by node's labels.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchFields:
+ description: A list of node selector requirements by node's fields.
+ type: array
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ podAffinity:
+ description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).
+ type: object
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.
+ type: array
+ items:
+ description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
+ type: object
+ required:
+ - podAffinityTerm
+ - weight
+ properties:
+ podAffinityTerm:
+ description: Required. A pod affinity term, associated with the corresponding weight.
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ weight:
+ description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.
+ type: integer
+ format: int32
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ type: array
+ items:
+ description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ podAntiAffinity:
+ description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).
+ type: object
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.
+ type: array
+ items:
+ description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
+ type: object
+ required:
+ - podAffinityTerm
+ - weight
+ properties:
+ podAffinityTerm:
+ description: Required. A pod affinity term, associated with the corresponding weight.
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ weight:
+ description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.
+ type: integer
+ format: int32
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ type: array
+ items:
+ description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running
+ type: object
+ required:
+ - topologyKey
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
+ type: object
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ type: array
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ type: object
+ required:
+ - key
+ - operator
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ additionalProperties:
+ type: string
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace"
+ type: array
+ items:
+ type: string
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ nodeSelector:
+ description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/'
+ type: object
+ additionalProperties:
+ type: string
+ priorityClassName:
+ description: If specified, the pod's priorityClassName.
+ type: string
+ serviceAccountName:
+ description: If specified, the pod's service account
+ type: string
+ tolerations:
+ description: If specified, the pod's tolerations.
+ type: array
+ items:
+ description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator .
+ type: object
+ properties:
+ effect:
+ description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+ type: string
+ key:
+ description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+ type: string
+ operator:
+ description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.
+ type: string
+ tolerationSeconds:
+ description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.
+ type: integer
+ format: int64
+ value:
+ description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.
+ type: string
+ serviceType:
+ description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort.
+ type: string
+ selector:
+ description: Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead.
+ type: object
+ properties:
+ dnsNames:
+ description: List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected.
+ type: array
+ items:
+ type: string
+ dnsZones:
+ description: List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected.
+ type: array
+ items:
+ type: string
+ matchLabels:
+ description: A label selector that is used to refine the set of certificate's that this challenge solver will apply to.
+ type: object
+ additionalProperties:
+ type: string
+ ca:
+ description: CA configures this issuer to sign certificates using a signing CA keypair stored in a Secret resource. This is used to build internal PKIs that are managed by cert-manager.
+ type: object
+ required:
+ - secretName
+ properties:
+ crlDistributionPoints:
+ description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set, certificates will be issued without distribution points set.
+ type: array
+ items:
+ type: string
+ ocspServers:
+ description: The OCSP server list is an X.509 v3 extension that defines a list of URLs of OCSP responders. The OCSP responders can be queried for the revocation status of an issued certificate. If not set, the certificate will be issued with no OCSP servers set. For example, an OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org".
+ type: array
+ items:
+ type: string
+ secretName:
+ description: SecretName is the name of the secret used to sign Certificates issued by this Issuer.
+ type: string
+ selfSigned:
+ description: SelfSigned configures this issuer to 'self sign' certificates using the private key used to create the CertificateRequest object.
+ type: object
+ properties:
+ crlDistributionPoints:
+ description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set certificate will be issued without CDP. Values are strings.
+ type: array
+ items:
+ type: string
+ vault:
+ description: Vault configures this issuer to sign certificates using a HashiCorp Vault PKI backend.
+ type: object
+ required:
+ - auth
+ - path
+ - server
+ properties:
+ auth:
+ description: Auth configures how cert-manager authenticates with the Vault server.
+ type: object
+ properties:
+ appRole:
+ description: AppRole authenticates with Vault using the App Role auth mechanism, with the role and secret stored in a Kubernetes Secret resource.
+ type: object
+ required:
+ - path
+ - roleId
+ - secretRef
+ properties:
+ path:
+ description: 'Path where the App Role authentication backend is mounted in Vault, e.g: "approle"'
+ type: string
+ roleId:
+ description: RoleID configured in the App Role authentication backend when setting up the authentication backend in Vault.
+ type: string
+ secretRef:
+ description: Reference to a key in a Secret that contains the App Role secret used to authenticate with Vault. The `key` field must be specified and denotes which entry within the Secret resource is used as the app role secret.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ kubernetes:
+ description: Kubernetes authenticates with Vault by passing the ServiceAccount token stored in the named Secret resource to the Vault server.
+ type: object
+ required:
+ - role
+ - secretRef
+ properties:
+ mountPath:
+ description: The Vault mountPath here is the mount path to use when authenticating with Vault. For example, setting a value to `/v1/auth/foo`, will use the path `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the default value "/v1/auth/kubernetes" will be used.
+ type: string
+ role:
+ description: A required field containing the Vault Role to assume. A Role binds a Kubernetes ServiceAccount with a set of Vault policies.
+ type: string
+ secretRef:
+ description: The required Secret field containing a Kubernetes ServiceAccount JWT used for authenticating with Vault. Use of 'ambient credentials' is not supported.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ tokenSecretRef:
+ description: TokenSecretRef authenticates with Vault by presenting a token.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ caBundle:
+ description: PEM-encoded CA bundle (base64-encoded) used to validate Vault server certificate. Only used if the Server URL is using HTTPS protocol. This parameter is ignored for plain HTTP protocol connection. If not set the system root certificates are used to validate the TLS connection.
+ type: string
+ format: byte
+ namespace:
+ description: 'Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: "ns1" More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces'
+ type: string
+ path:
+ description: 'Path is the mount path of the Vault PKI backend''s `sign` endpoint, e.g: "my_pki_mount/sign/my-role-name".'
+ type: string
+ server:
+ description: 'Server is the connection address for the Vault server, e.g: "https://vault.example.com:8200".'
+ type: string
+ venafi:
+ description: Venafi configures this issuer to sign certificates using a Venafi TPP or Venafi Cloud policy zone.
+ type: object
+ required:
+ - zone
+ properties:
+ cloud:
+ description: Cloud specifies the Venafi cloud configuration settings. Only one of TPP or Cloud may be specified.
+ type: object
+ required:
+ - apiTokenSecretRef
+ properties:
+ apiTokenSecretRef:
+ description: APITokenSecretRef is a secret key selector for the Venafi Cloud API token.
+ type: object
+ required:
+ - name
+ properties:
+ key:
+ description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required.
+ type: string
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ url:
+ description: URL is the base URL for Venafi Cloud. Defaults to "https://api.venafi.cloud/v1".
+ type: string
+ tpp:
+ description: TPP specifies Trust Protection Platform configuration settings. Only one of TPP or Cloud may be specified.
+ type: object
+ required:
+ - credentialsRef
+ - url
+ properties:
+ caBundle:
+ description: CABundle is a PEM encoded TLS certificate to use to verify connections to the TPP instance. If specified, system roots will not be used and the issuing CA for the TPP instance must be verifiable using the provided root. If not specified, the connection will be verified using the cert-manager system root certificates.
+ type: string
+ format: byte
+ credentialsRef:
+ description: CredentialsRef is a reference to a Secret containing the username and password for the TPP server. The secret must contain two keys, 'username' and 'password'.
+ type: object
+ required:
+ - name
+ properties:
+ name:
+ description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+ type: string
+ url:
+ description: 'URL is the base URL for the vedsdk endpoint of the Venafi TPP instance, for example: "https://tpp.example.com/vedsdk".'
+ type: string
+ zone:
+ description: Zone is the Venafi Policy Zone to use for this issuer. All requests made to the Venafi platform will be restricted by the named zone policy. This field is required.
+ type: string
+ status:
+ description: Status of the Issuer. This is set and managed automatically.
+ type: object
+ properties:
+ acme:
+ description: ACME specific status options. This field should only be set if the Issuer is configured to use an ACME server to issue certificates.
+ type: object
+ properties:
+ lastRegisteredEmail:
+ description: LastRegisteredEmail is the email associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer
+ type: string
+ uri:
+ description: URI is the unique account identifier, which can also be used to retrieve account details from the CA
+ type: string
+ conditions:
+ description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`.
+ type: array
+ items:
+ description: IssuerCondition contains condition information for an Issuer.
+ type: object
+ required:
+ - status
+ - type
+ properties:
+ lastTransitionTime:
+ description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+ type: string
+ format: date-time
+ message:
+ description: Message is a human readable description of the details of the last transition, complementing reason.
+ type: string
+ observedGeneration:
+ description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Issuer.
+ type: integer
+ format: int64
+ reason:
+ description: Reason is a brief machine readable explanation for the condition's last transition.
+ type: string
+ status:
+ description: Status of the condition, one of (`True`, `False`, `Unknown`).
+ type: string
+ enum:
+ - "True"
+ - "False"
+ - Unknown
+ type:
+ description: Type of the condition, known values are (`Ready`).
+ type: string
+ x-kubernetes-list-map-keys:
+ - type
+ x-kubernetes-list-type: map
+ served: true
+ storage: true
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ name: orders.acme.cert-manager.io
+ annotations:
+ cert-manager.io/inject-ca-from-secret: '{{ template "webhook.caRef" . }}'
+ labels:
+ app: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/name: '{{ template "cert-manager.name" . }}'
+ app.kubernetes.io/instance: '{{ .Release.Name }}'
+ # Generated labels {{- include "labels" . | nindent 4 }}
+spec:
+ group: acme.cert-manager.io
+ names:
+ kind: Order
+ listKind: OrderList
+ plural: orders
+ singular: order
+ categories:
+ - cert-manager
+ - cert-manager-acme
+ scope: Namespaced
+ versions:
+ - name: v1
+ subresources:
+ status: {}
+ additionalPrinterColumns:
+ - jsonPath: .status.state
+ name: State
+ type: string
+ - jsonPath: .spec.issuerRef.name
+ name: Issuer
+ priority: 1
+ type: string
+ - jsonPath: .status.reason
+ name: Reason
+ priority: 1
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.
+ name: Age
+ type: date
+ schema:
+ openAPIV3Schema:
+ description: Order is a type to represent an Order with an ACME server
+ type: object
+ required:
+ - metadata
+ - spec
+ properties:
+ apiVersion:
+ description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+ type: string
+ kind:
+ description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+ type: string
+ metadata:
+ type: object
+ spec:
+ type: object
+ required:
+ - issuerRef
+ - request
+ properties:
+ commonName:
+ description: CommonName is the common name as specified on the DER encoded CSR. If specified, this value must also be present in `dnsNames` or `ipAddresses`. This field must match the corresponding field on the DER encoded CSR.
+ type: string
+ dnsNames:
+ description: DNSNames is a list of DNS names that should be included as part of the Order validation process. This field must match the corresponding field on the DER encoded CSR.
+ type: array
+ items:
+ type: string
+ duration:
+ description: Duration is the duration for the not after date for the requested certificate. this is set on order creation as pe the ACME spec.
+ type: string
+ ipAddresses:
+ description: IPAddresses is a list of IP addresses that should be included as part of the Order validation process. This field must match the corresponding field on the DER encoded CSR.
+ type: array
+ items:
+ type: string
+ issuerRef:
+ description: IssuerRef references a properly configured ACME-type Issuer which should be used to create this Order. If the Issuer does not exist, processing will be retried. If the Issuer is not an 'ACME' Issuer, an error will be returned and the Order will be marked as failed.
+ type: object
+ required:
+ - name
+ properties:
+ group:
+ description: Group of the resource being referred to.
+ type: string
+ kind:
+ description: Kind of the resource being referred to.
+ type: string
+ name:
+ description: Name of the resource being referred to.
+ type: string
+ request:
+ description: Certificate signing request bytes in DER encoding. This will be used when finalizing the order. This field must be set on the order.
+ type: string
+ format: byte
+ status:
+ type: object
+ properties:
+ authorizations:
+ description: Authorizations contains data returned from the ACME server on what authorizations must be completed in order to validate the DNS names specified on the Order.
+ type: array
+ items:
+ description: ACMEAuthorization contains data returned from the ACME server on an authorization that must be completed in order validate a DNS name on an ACME Order resource.
+ type: object
+ required:
+ - url
+ properties:
+ challenges:
+ description: Challenges specifies the challenge types offered by the ACME server. One of these challenge types will be selected when validating the DNS name and an appropriate Challenge resource will be created to perform the ACME challenge process.
+ type: array
+ items:
+ description: Challenge specifies a challenge offered by the ACME server for an Order. An appropriate Challenge resource can be created to perform the ACME challenge process.
+ type: object
+ required:
+ - token
+ - type
+ - url
+ properties:
+ token:
+ description: Token is the token that must be presented for this challenge. This is used to compute the 'key' that must also be presented.
+ type: string
+ type:
+ description: Type is the type of challenge being offered, e.g. 'http-01', 'dns-01', 'tls-sni-01', etc. This is the raw value retrieved from the ACME server. Only 'http-01' and 'dns-01' are supported by cert-manager, other values will be ignored.
+ type: string
+ url:
+ description: URL is the URL of this challenge. It can be used to retrieve additional metadata about the Challenge from the ACME server.
+ type: string
+ identifier:
+ description: Identifier is the DNS name to be validated as part of this authorization
+ type: string
+ initialState:
+ description: InitialState is the initial state of the ACME authorization when first fetched from the ACME server. If an Authorization is already 'valid', the Order controller will not create a Challenge resource for the authorization. This will occur when working with an ACME server that enables 'authz reuse' (such as Let's Encrypt's production endpoint). If not set and 'identifier' is set, the state is assumed to be pending and a Challenge will be created.
+ type: string
+ enum:
+ - valid
+ - ready
+ - pending
+ - processing
+ - invalid
+ - expired
+ - errored
+ url:
+ description: URL is the URL of the Authorization that must be completed
+ type: string
+ wildcard:
+ description: Wildcard will be true if this authorization is for a wildcard DNS name. If this is true, the identifier will be the *non-wildcard* version of the DNS name. For example, if '*.example.com' is the DNS name being validated, this field will be 'true' and the 'identifier' field will be 'example.com'.
+ type: boolean
+ certificate:
+ description: Certificate is a copy of the PEM encoded certificate for this Order. This field will be populated after the order has been successfully finalized with the ACME server, and the order has transitioned to the 'valid' state.
+ type: string
+ format: byte
+ failureTime:
+ description: FailureTime stores the time that this order failed. This is used to influence garbage collection and back-off.
+ type: string
+ format: date-time
+ finalizeURL:
+ description: FinalizeURL of the Order. This is used to obtain certificates for this order once it has been completed.
+ type: string
+ reason:
+ description: Reason optionally provides more information about a why the order is in the current state.
+ type: string
+ state:
+ description: State contains the current state of this Order resource. States 'success' and 'expired' are 'final'
+ type: string
+ enum:
+ - valid
+ - ready
+ - pending
+ - processing
+ - invalid
+ - expired
+ - errored
+ url:
+ description: URL of the Order. This will initially be empty when the resource is first created. The Order controller will populate this field when the Order is first processed. This field will be immutable after it is initially set.
+ type: string
+ served: true
+ storage: true
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/deployment.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/deployment.yaml
new file mode 100644
index 0000000..b7f549e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/deployment.yaml
@@ -0,0 +1,170 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ template "cert-manager.fullname" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ app: {{ template "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ template "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.deploymentAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ replicas: {{ .Values.replicaCount }}
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: {{ template "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- with .Values.strategy }}
+ strategy:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ template:
+ metadata:
+ labels:
+ app: {{ template "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ template "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 8 }}
+ {{- with .Values.podLabels }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.podAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if and .Values.prometheus.enabled (not .Values.prometheus.servicemonitor.enabled) }}
+ {{- if not .Values.podAnnotations }}
+ annotations:
+ {{- end }}
+ prometheus.io/path: "/metrics"
+ prometheus.io/scrape: 'true'
+ prometheus.io/port: '9402'
+ {{- end }}
+ spec:
+ serviceAccountName: {{ template "cert-manager.serviceAccountName" . }}
+ {{- with .Values.global.priorityClassName }}
+ priorityClassName: {{ . | quote }}
+ {{- end }}
+ {{- $enabledDefined := gt (len (keys (pick .Values.securityContext "enabled"))) 0 }}
+ {{- $legacyEnabledExplicitlyOff := and $enabledDefined (not .Values.securityContext.enabled) }}
+ {{- if and .Values.securityContext (not $legacyEnabledExplicitlyOff) }}
+ securityContext:
+ {{- if .Values.securityContext.enabled }}
+ {{/* support legacy securityContext.enabled and its two parameters */}}
+ fsGroup: {{ default 1001 .Values.securityContext.fsGroup }}
+ runAsUser: {{ default 1001 .Values.securityContext.runAsUser }}
+ {{- else }}
+ {{/* this is the way forward: support an arbitrary yaml block */}}
+ {{- toYaml .Values.securityContext | nindent 8 }}
+ {{- end }}
+ {{- end }}
+ {{- with .Values.volumes }}
+ volumes:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ containers:
+ - name: {{ .Chart.Name }}
+ {{- with .Values.image }}
+ image: "{{- if .registry -}}{{ .registry }}/{{- end -}}{{ .repository }}{{- if (.digest) -}} @{{ .digest }}{{- else -}}:{{ default $.Chart.AppVersion .tag }} {{- end -}}"
+ {{- end }}
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ {{- if .Values.global.logLevel }}
+ - --v={{ .Values.global.logLevel }}
+ {{- end }}
+ {{- if .Values.clusterResourceNamespace }}
+ - --cluster-resource-namespace={{ .Values.clusterResourceNamespace }}
+ {{- else }}
+ - --cluster-resource-namespace=$(POD_NAMESPACE)
+ {{- end }}
+ {{- with .Values.global.leaderElection }}
+ - --leader-election-namespace={{ .namespace }}
+ {{- if .leaseDuration }}
+ - --leader-election-lease-duration={{ .leaseDuration }}
+ {{- end }}
+ {{- if .renewDeadline }}
+ - --leader-election-renew-deadline={{ .renewDeadline }}
+ {{- end }}
+ {{- if .retryPeriod }}
+ - --leader-election-retry-period={{ .retryPeriod }}
+ {{- end }}
+ {{- end }}
+ {{- with .Values.extraArgs }}
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ {{- with .Values.ingressShim }}
+ {{- if .defaultIssuerName }}
+ - --default-issuer-name={{ .defaultIssuerName }}
+ {{- end }}
+ {{- if .defaultIssuerKind }}
+ - --default-issuer-kind={{ .defaultIssuerKind }}
+ {{- end }}
+ {{- if .defaultIssuerGroup }}
+ - --default-issuer-group={{ .defaultIssuerGroup }}
+ {{- end }}
+ {{- end }}
+ {{- if .Values.featureGates }}
+ - --feature-gates={{ .Values.featureGates }}
+ {{- end }}
+ ports:
+ - containerPort: 9402
+ name: http-metrics
+ protocol: TCP
+ {{- with .Values.containerSecurityContext }}
+ securityContext:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.volumeMounts }}
+ volumeMounts:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ env:
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ {{- with .Values.extraEnv }}
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ {{- with .Values.http_proxy }}
+ - name: HTTP_PROXY
+ value: {{ . }}
+ {{- end }}
+ {{- with .Values.https_proxy }}
+ - name: HTTPS_PROXY
+ value: {{ . }}
+ {{- end }}
+ {{- with .Values.no_proxy }}
+ - name: NO_PROXY
+ value: {{ . }}
+ {{- end }}
+ {{- with .Values.resources }}
+ resources:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.podDnsPolicy }}
+ dnsPolicy: {{ . }}
+ {{- end }}
+ {{- with .Values.podDnsConfig }}
+ dnsConfig:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/psp-clusterrole.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/psp-clusterrole.yaml
new file mode 100644
index 0000000..1d40a02
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/psp-clusterrole.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.global.podSecurityPolicy.enabled }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-psp
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+rules:
+- apiGroups: ['policy']
+ resources: ['podsecuritypolicies']
+ verbs: ['use']
+ resourceNames:
+ - {{ template "cert-manager.fullname" . }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/psp-clusterrolebinding.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/psp-clusterrolebinding.yaml
new file mode 100644
index 0000000..1da89c8
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/psp-clusterrolebinding.yaml
@@ -0,0 +1,20 @@
+{{- if .Values.global.podSecurityPolicy.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-psp
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cert-manager.fullname" . }}-psp
+subjects:
+ - kind: ServiceAccount
+ name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/psp.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/psp.yaml
new file mode 100644
index 0000000..9e99f5c
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/psp.yaml
@@ -0,0 +1,49 @@
+{{- if .Values.global.podSecurityPolicy.enabled }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+ name: {{ template "cert-manager.fullname" . }}
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+ annotations:
+ seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+ seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default'
+ {{- if .Values.global.podSecurityPolicy.useAppArmor }}
+ apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+ apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default'
+ {{- end }}
+spec:
+ privileged: false
+ allowPrivilegeEscalation: false
+ allowedCapabilities: [] # default set of capabilities are implicitly allowed
+ volumes:
+ - 'configMap'
+ - 'emptyDir'
+ - 'projected'
+ - 'secret'
+ - 'downwardAPI'
+ hostNetwork: false
+ hostIPC: false
+ hostPID: false
+ runAsUser:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+ seLinux:
+ rule: 'RunAsAny'
+ supplementalGroups:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+ fsGroup:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/rbac.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/rbac.yaml
new file mode 100644
index 0000000..f8ead0c
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/rbac.yaml
@@ -0,0 +1,545 @@
+{{- if .Values.global.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ template "cert-manager.fullname" . }}:leaderelection
+ namespace: {{ .Values.global.leaderElection.namespace }}
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ resourceNames: ["cert-manager-controller"]
+ verbs: ["get", "update", "patch"]
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["create"]
+
+---
+
+# grant cert-manager permission to manage the leaderelection configmap in the
+# leader election namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ include "cert-manager.fullname" . }}:leaderelection
+ namespace: {{ .Values.global.leaderElection.namespace }}
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ template "cert-manager.fullname" . }}:leaderelection
+subjects:
+ - apiGroup: ""
+ kind: ServiceAccount
+ name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+
+---
+
+# Issuer controller role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-issuers
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ - apiGroups: ["cert-manager.io"]
+ resources: ["issuers", "issuers/status"]
+ verbs: ["update", "patch"]
+ - apiGroups: ["cert-manager.io"]
+ resources: ["issuers"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get", "list", "watch", "create", "update", "delete"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["create", "patch"]
+
+---
+
+# ClusterIssuer controller role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-clusterissuers
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ - apiGroups: ["cert-manager.io"]
+ resources: ["clusterissuers", "clusterissuers/status"]
+ verbs: ["update", "patch"]
+ - apiGroups: ["cert-manager.io"]
+ resources: ["clusterissuers"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get", "list", "watch", "create", "update", "delete"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["create", "patch"]
+
+---
+
+# Certificates controller role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-certificates
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ - apiGroups: ["cert-manager.io"]
+ resources: ["certificates", "certificates/status", "certificaterequests", "certificaterequests/status"]
+ verbs: ["update", "patch"]
+ - apiGroups: ["cert-manager.io"]
+ resources: ["certificates", "certificaterequests", "clusterissuers", "issuers"]
+ verbs: ["get", "list", "watch"]
+ # We require these rules to support users with the OwnerReferencesPermissionEnforcement
+ # admission controller enabled:
+ # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement
+ - apiGroups: ["cert-manager.io"]
+ resources: ["certificates/finalizers", "certificaterequests/finalizers"]
+ verbs: ["update"]
+ - apiGroups: ["acme.cert-manager.io"]
+ resources: ["orders"]
+ verbs: ["create", "delete", "get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["create", "patch"]
+
+---
+
+# Orders controller role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-orders
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ - apiGroups: ["acme.cert-manager.io"]
+ resources: ["orders", "orders/status"]
+ verbs: ["update", "patch"]
+ - apiGroups: ["acme.cert-manager.io"]
+ resources: ["orders", "challenges"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["cert-manager.io"]
+ resources: ["clusterissuers", "issuers"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["acme.cert-manager.io"]
+ resources: ["challenges"]
+ verbs: ["create", "delete"]
+ # We require these rules to support users with the OwnerReferencesPermissionEnforcement
+ # admission controller enabled:
+ # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement
+ - apiGroups: ["acme.cert-manager.io"]
+ resources: ["orders/finalizers"]
+ verbs: ["update"]
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["create", "patch"]
+
+---
+
+# Challenges controller role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-challenges
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ # Use to update challenge resource status
+ - apiGroups: ["acme.cert-manager.io"]
+ resources: ["challenges", "challenges/status"]
+ verbs: ["update", "patch"]
+ # Used to watch challenge resources
+ - apiGroups: ["acme.cert-manager.io"]
+ resources: ["challenges"]
+ verbs: ["get", "list", "watch"]
+ # Used to watch challenges, issuer and clusterissuer resources
+ - apiGroups: ["cert-manager.io"]
+ resources: ["issuers", "clusterissuers"]
+ verbs: ["get", "list", "watch"]
+ # Need to be able to retrieve ACME account private key to complete challenges
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get", "list", "watch"]
+ # Used to create events
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["create", "patch"]
+ # HTTP01 rules
+ - apiGroups: [""]
+ resources: ["pods", "services"]
+ verbs: ["get", "list", "watch", "create", "delete"]
+ - apiGroups: ["networking.k8s.io"]
+ resources: ["ingresses"]
+ verbs: ["get", "list", "watch", "create", "delete", "update"]
+ - apiGroups: [ "gateway.networking.k8s.io" ]
+ resources: [ "httproutes" ]
+ verbs: ["get", "list", "watch", "create", "delete", "update"]
+ # We require the ability to specify a custom hostname when we are creating
+ # new ingress resources.
+ # See: https://github.com/openshift/origin/blob/21f191775636f9acadb44fa42beeb4f75b255532/pkg/route/apiserver/admission/ingress_admission.go#L84-L148
+ - apiGroups: ["route.openshift.io"]
+ resources: ["routes/custom-host"]
+ verbs: ["create"]
+ # We require these rules to support users with the OwnerReferencesPermissionEnforcement
+ # admission controller enabled:
+ # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement
+ - apiGroups: ["acme.cert-manager.io"]
+ resources: ["challenges/finalizers"]
+ verbs: ["update"]
+ # DNS01 rules (duplicated above)
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get", "list", "watch"]
+
+---
+
+# ingress-shim controller role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-ingress-shim
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ - apiGroups: ["cert-manager.io"]
+ resources: ["certificates", "certificaterequests"]
+ verbs: ["create", "update", "delete"]
+ - apiGroups: ["cert-manager.io"]
+ resources: ["certificates", "certificaterequests", "issuers", "clusterissuers"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["networking.k8s.io"]
+ resources: ["ingresses"]
+ verbs: ["get", "list", "watch"]
+ # We require these rules to support users with the OwnerReferencesPermissionEnforcement
+ # admission controller enabled:
+ # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement
+ - apiGroups: ["networking.k8s.io"]
+ resources: ["ingresses/finalizers"]
+ verbs: ["update"]
+ - apiGroups: ["gateway.networking.k8s.io"]
+ resources: ["gateways", "httproutes"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["gateway.networking.k8s.io"]
+ resources: ["gateways/finalizers", "httproutes/finalizers"]
+ verbs: ["update"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["create", "patch"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-issuers
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cert-manager.fullname" . }}-controller-issuers
+subjects:
+ - name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ kind: ServiceAccount
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-clusterissuers
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cert-manager.fullname" . }}-controller-clusterissuers
+subjects:
+ - name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ kind: ServiceAccount
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-certificates
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cert-manager.fullname" . }}-controller-certificates
+subjects:
+ - name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ kind: ServiceAccount
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-orders
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cert-manager.fullname" . }}-controller-orders
+subjects:
+ - name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ kind: ServiceAccount
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-challenges
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cert-manager.fullname" . }}-controller-challenges
+subjects:
+ - name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ kind: ServiceAccount
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-ingress-shim
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cert-manager.fullname" . }}-controller-ingress-shim
+subjects:
+ - name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ kind: ServiceAccount
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-view
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+ {{- if .Values.global.rbac.aggregateClusterRoles }}
+ rbac.authorization.k8s.io/aggregate-to-view: "true"
+ rbac.authorization.k8s.io/aggregate-to-edit: "true"
+ rbac.authorization.k8s.io/aggregate-to-admin: "true"
+ {{- end }}
+rules:
+ - apiGroups: ["cert-manager.io"]
+ resources: ["certificates", "certificaterequests", "issuers"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["acme.cert-manager.io"]
+ resources: ["challenges", "orders"]
+ verbs: ["get", "list", "watch"]
+
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-edit
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+ {{- if .Values.global.rbac.aggregateClusterRoles }}
+ rbac.authorization.k8s.io/aggregate-to-edit: "true"
+ rbac.authorization.k8s.io/aggregate-to-admin: "true"
+ {{- end }}
+rules:
+ - apiGroups: ["cert-manager.io"]
+ resources: ["certificates", "certificaterequests", "issuers"]
+ verbs: ["create", "delete", "deletecollection", "patch", "update"]
+ - apiGroups: ["cert-manager.io"]
+ resources: ["certificates/status"]
+ verbs: ["update"]
+ - apiGroups: ["acme.cert-manager.io"]
+ resources: ["challenges", "orders"]
+ verbs: ["create", "delete", "deletecollection", "patch", "update"]
+
+---
+
+# Permission to approve CertificateRequests referencing cert-manager.io Issuers and ClusterIssuers
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-approve:cert-manager-io
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cert-manager"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ - apiGroups: ["cert-manager.io"]
+ resources: ["signers"]
+ verbs: ["approve"]
+ resourceNames: ["issuers.cert-manager.io/*", "clusterissuers.cert-manager.io/*"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-approve:cert-manager-io
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cert-manager"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cert-manager.fullname" . }}-controller-approve:cert-manager-io
+subjects:
+ - name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ kind: ServiceAccount
+
+---
+
+# Permission to:
+# - Update and sign CertificatSigningeRequests referencing cert-manager.io Issuers and ClusterIssuers
+# - Perform SubjectAccessReviews to test whether users are able to reference Namespaced Issuers
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-certificatesigningrequests
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cert-manager"
+ {{- include "labels" . | nindent 4 }}
+rules:
+ - apiGroups: ["certificates.k8s.io"]
+ resources: ["certificatesigningrequests"]
+ verbs: ["get", "list", "watch", "update"]
+ - apiGroups: ["certificates.k8s.io"]
+ resources: ["certificatesigningrequests/status"]
+ verbs: ["update", "patch"]
+ - apiGroups: ["certificates.k8s.io"]
+ resources: ["signers"]
+ resourceNames: ["issuers.cert-manager.io/*", "clusterissuers.cert-manager.io/*"]
+ verbs: ["sign"]
+ - apiGroups: ["authorization.k8s.io"]
+ resources: ["subjectaccessreviews"]
+ verbs: ["create"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "cert-manager.fullname" . }}-controller-certificatesigningrequests
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "cert-manager"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "cert-manager.fullname" . }}-controller-certificatesigningrequests
+subjects:
+ - name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ kind: ServiceAccount
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/service.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/service.yaml
new file mode 100644
index 0000000..8ad24ca
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/service.yaml
@@ -0,0 +1,31 @@
+{{- if .Values.prometheus.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ template "cert-manager.fullname" . }}
+ namespace: {{ .Release.Namespace | quote }}
+{{- with .Values.serviceAnnotations }}
+ annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.serviceLabels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ type: ClusterIP
+ ports:
+ - protocol: TCP
+ port: 9402
+ name: tcp-prometheus-servicemonitor
+ targetPort: {{ .Values.prometheus.servicemonitor.targetPort }}
+ selector:
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/serviceaccount.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/serviceaccount.yaml
new file mode 100644
index 0000000..6e53f78
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/serviceaccount.yaml
@@ -0,0 +1,25 @@
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+{{- with .Values.global.imagePullSecrets }}
+imagePullSecrets:
+ {{- toYaml . | nindent 2 }}
+{{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
+metadata:
+ name: {{ template "cert-manager.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ {{- with .Values.serviceAccount.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.serviceAccount.labels }}
+ {{ toYaml . | nindent 4 }}
+ {{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/servicemonitor.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/servicemonitor.yaml
new file mode 100644
index 0000000..f962c6b
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/servicemonitor.yaml
@@ -0,0 +1,39 @@
+{{- if and .Values.prometheus.enabled .Values.prometheus.servicemonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ name: {{ template "cert-manager.fullname" . }}
+{{- if .Values.prometheus.servicemonitor.namespace }}
+ namespace: {{ .Values.prometheus.servicemonitor.namespace }}
+{{- else }}
+ namespace: {{ .Release.Namespace | quote }}
+{{- end }}
+ labels:
+ app: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/name: {{ include "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+ {{- include "labels" . | nindent 4 }}
+ prometheus: {{ .Values.prometheus.servicemonitor.prometheusInstance }}
+ {{- with .Values.prometheus.servicemonitor.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ jobLabel: {{ template "cert-manager.fullname" . }}
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: {{ template "cert-manager.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "controller"
+{{- if .Values.prometheus.servicemonitor.namespace }}
+ namespaceSelector:
+ matchNames:
+ - {{ .Release.Namespace }}
+{{- end }}
+ endpoints:
+ - targetPort: {{ .Values.prometheus.servicemonitor.targetPort }}
+ path: {{ .Values.prometheus.servicemonitor.path }}
+ interval: {{ .Values.prometheus.servicemonitor.interval }}
+ scrapeTimeout: {{ .Values.prometheus.servicemonitor.scrapeTimeout }}
+ honorLabels: {{ .Values.prometheus.servicemonitor.honorLabels }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-job.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-job.yaml
new file mode 100644
index 0000000..66db004
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-job.yaml
@@ -0,0 +1,77 @@
+{{- if .Values.startupapicheck.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ include "startupapicheck.fullname" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ app: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/name: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "startupapicheck"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.startupapicheck.jobAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ backoffLimit: {{ .Values.startupapicheck.backoffLimit }}
+ template:
+ metadata:
+ labels:
+ app: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/name: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "startupapicheck"
+ {{- include "labels" . | nindent 8 }}
+ {{- with .Values.startupapicheck.podLabels }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.startupapicheck.podAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ spec:
+ restartPolicy: OnFailure
+ serviceAccountName: {{ template "startupapicheck.serviceAccountName" . }}
+ {{- with .Values.global.priorityClassName }}
+ priorityClassName: {{ . | quote }}
+ {{- end }}
+ {{- with .Values.startupapicheck.securityContext }}
+ securityContext:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ containers:
+ - name: {{ .Chart.Name }}
+ {{- with .Values.startupapicheck.image }}
+ image: "{{- if .registry -}}{{ .registry }}/{{- end -}}{{ .repository }}{{- if (.digest) -}} @{{ .digest }}{{- else -}}:{{ default $.Chart.AppVersion .tag }} {{- end -}}"
+ {{- end }}
+ imagePullPolicy: {{ .Values.startupapicheck.image.pullPolicy }}
+ args:
+ - check
+ - api
+ - --wait={{ .Values.startupapicheck.timeout }}
+ {{- with .Values.startupapicheck.extraArgs }}
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ {{- with .Values.startupapicheck.containerSecurityContext }}
+ securityContext:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.startupapicheck.resources }}
+ resources:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.startupapicheck.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.startupapicheck.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.startupapicheck.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-psp-clusterrole.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-psp-clusterrole.yaml
new file mode 100644
index 0000000..dacd4be
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-psp-clusterrole.yaml
@@ -0,0 +1,24 @@
+{{- if .Values.startupapicheck.enabled }}
+{{- if .Values.global.podSecurityPolicy.enabled }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: {{ template "startupapicheck.fullname" . }}-psp
+ labels:
+ app: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/name: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "startupapicheck"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.startupapicheck.rbac.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+rules:
+- apiGroups: ['policy']
+ resources: ['podsecuritypolicies']
+ verbs: ['use']
+ resourceNames:
+ - {{ template "startupapicheck.fullname" . }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-psp-clusterrolebinding.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-psp-clusterrolebinding.yaml
new file mode 100644
index 0000000..d19fa84
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-psp-clusterrolebinding.yaml
@@ -0,0 +1,26 @@
+{{- if .Values.startupapicheck.enabled }}
+{{- if .Values.global.podSecurityPolicy.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "startupapicheck.fullname" . }}-psp
+ labels:
+ app: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/name: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "startupapicheck"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.startupapicheck.rbac.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "startupapicheck.fullname" . }}-psp
+subjects:
+ - kind: ServiceAccount
+ name: {{ template "startupapicheck.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-psp.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-psp.yaml
new file mode 100644
index 0000000..f09d60d
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-psp.yaml
@@ -0,0 +1,51 @@
+{{- if .Values.startupapicheck.enabled }}
+{{- if .Values.global.podSecurityPolicy.enabled }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+ name: {{ template "startupapicheck.fullname" . }}
+ labels:
+ app: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/name: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "startupapicheck"
+ {{- include "labels" . | nindent 4 }}
+ annotations:
+ seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+ seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default'
+ {{- if .Values.global.podSecurityPolicy.useAppArmor }}
+ apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+ apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default'
+ {{- end }}
+ {{- with .Values.startupapicheck.rbac.annotations }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ privileged: false
+ allowPrivilegeEscalation: false
+ allowedCapabilities: [] # default set of capabilities are implicitly allowed
+ volumes:
+ - 'projected'
+ - 'secret'
+ hostNetwork: false
+ hostIPC: false
+ hostPID: false
+ runAsUser:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+ seLinux:
+ rule: 'RunAsAny'
+ supplementalGroups:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+ fsGroup:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-rbac.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-rbac.yaml
new file mode 100644
index 0000000..a6be7b9
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-rbac.yaml
@@ -0,0 +1,48 @@
+{{- if .Values.startupapicheck.enabled }}
+{{- if .Values.global.rbac.create }}
+# create certificate role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ template "startupapicheck.fullname" . }}:create-cert
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ app: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/name: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "startupapicheck"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.startupapicheck.rbac.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+rules:
+ - apiGroups: ["cert-manager.io"]
+ resources: ["certificates"]
+ verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ include "startupapicheck.fullname" . }}:create-cert
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ app: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/name: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "startupapicheck"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.startupapicheck.rbac.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ template "startupapicheck.fullname" . }}:create-cert
+subjects:
+ - kind: ServiceAccount
+ name: {{ template "startupapicheck.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-serviceaccount.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-serviceaccount.yaml
new file mode 100644
index 0000000..46262d5
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/startupapicheck-serviceaccount.yaml
@@ -0,0 +1,27 @@
+{{- if .Values.startupapicheck.enabled }}
+{{- if .Values.startupapicheck.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{ .Values.startupapicheck.serviceAccount.automountServiceAccountToken }}
+metadata:
+ name: {{ template "startupapicheck.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ {{- with .Values.startupapicheck.serviceAccount.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ labels:
+ app: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/name: {{ include "startupapicheck.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "startupapicheck"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.startupapicheck.serviceAccount.labels }}
+ {{ toYaml . | nindent 4 }}
+ {{- end }}
+{{- with .Values.global.imagePullSecrets }}
+imagePullSecrets:
+ {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-config.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-config.yaml
new file mode 100644
index 0000000..29558e9
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-config.yaml
@@ -0,0 +1,24 @@
+{{- if .Values.webhook.config -}}
+ {{- if not .Values.webhook.config.apiVersion -}}
+ {{- fail "webhook.config.apiVersion must be set" -}}
+ {{- end -}}
+
+ {{- if not .Values.webhook.config.kind -}}
+ {{- fail "webhook.config.kind must be set" -}}
+ {{- end -}}
+{{- end -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "webhook.fullname" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+data:
+ {{- if .Values.webhook.config }}
+ config.yaml: |
+ {{ .Values.webhook.config | toYaml | nindent 4 }}
+ {{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-deployment.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-deployment.yaml
new file mode 100644
index 0000000..0de1df2
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-deployment.yaml
@@ -0,0 +1,153 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "webhook.fullname" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.webhook.deploymentAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ replicas: {{ .Values.webhook.replicaCount }}
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- with .Values.webhook.strategy }}
+ strategy:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ template:
+ metadata:
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 8 }}
+ {{- with .Values.webhook.podLabels }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.webhook.podAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ spec:
+ serviceAccountName: {{ template "webhook.serviceAccountName" . }}
+ {{- with .Values.global.priorityClassName }}
+ priorityClassName: {{ . | quote }}
+ {{- end }}
+ {{- with .Values.webhook.securityContext }}
+ securityContext:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.webhook.hostNetwork }}
+ hostNetwork: true
+ {{- end }}
+ containers:
+ - name: {{ .Chart.Name }}
+ {{- with .Values.webhook.image }}
+ image: "{{- if .registry -}}{{ .registry }}/{{- end -}}{{ .repository }}{{- if (.digest) -}} @{{ .digest }}{{- else -}}:{{ default $.Chart.AppVersion .tag }} {{- end -}}"
+ {{- end }}
+ imagePullPolicy: {{ .Values.webhook.image.pullPolicy }}
+ args:
+ {{- if .Values.global.logLevel }}
+ - --v={{ .Values.global.logLevel }}
+ {{- end }}
+ {{- if .Values.webhook.config }}
+ - --config=/var/cert-manager/config/config.yaml
+ {{- end }}
+ {{- $config := default .Values.webhook.config "" }}
+ {{ if not $config.securePort -}}
+ - --secure-port={{ .Values.webhook.securePort }}
+ {{- end }}
+ {{- $tlsConfig := default $config.tlsConfig "" }}
+ {{ if or (not $config.tlsConfig) (and (not $tlsConfig.dynamic) (not $tlsConfig.filesystem) ) -}}
+ - --dynamic-serving-ca-secret-namespace=$(POD_NAMESPACE)
+ - --dynamic-serving-ca-secret-name={{ template "webhook.fullname" . }}-ca
+ - --dynamic-serving-dns-names={{ template "webhook.fullname" . }},{{ template "webhook.fullname" . }}.{{ .Release.Namespace }},{{ template "webhook.fullname" . }}.{{ .Release.Namespace }}.svc{{ if .Values.webhook.url.host }},{{ .Values.webhook.url.host }}{{ end }}
+ {{- end }}
+ {{- with .Values.webhook.extraArgs }}
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ ports:
+ - name: https
+ protocol: TCP
+ {{- if $config.securePort }}
+ containerPort: {{ $config.securePort }}
+ {{- else if .Values.webhook.securePort }}
+ containerPort: {{ .Values.webhook.securePort }}
+ {{- else }}
+ containerPort: 6443
+ {{- end }}
+ livenessProbe:
+ httpGet:
+ path: /livez
+ {{- if $config.healthzPort }}
+ port: {{ $config.healthzPort }}
+ {{- else }}
+ port: 6080
+ {{- end }}
+ scheme: HTTP
+ initialDelaySeconds: {{ .Values.webhook.livenessProbe.initialDelaySeconds }}
+ periodSeconds: {{ .Values.webhook.livenessProbe.periodSeconds }}
+ timeoutSeconds: {{ .Values.webhook.livenessProbe.timeoutSeconds }}
+ successThreshold: {{ .Values.webhook.livenessProbe.successThreshold }}
+ failureThreshold: {{ .Values.webhook.livenessProbe.failureThreshold }}
+ readinessProbe:
+ httpGet:
+ path: /healthz
+ {{- if $config.healthzPort }}
+ port: {{ $config.healthzPort }}
+ {{- else }}
+ port: 6080
+ {{- end }}
+ scheme: HTTP
+ initialDelaySeconds: {{ .Values.webhook.readinessProbe.initialDelaySeconds }}
+ periodSeconds: {{ .Values.webhook.readinessProbe.periodSeconds }}
+ timeoutSeconds: {{ .Values.webhook.readinessProbe.timeoutSeconds }}
+ successThreshold: {{ .Values.webhook.readinessProbe.successThreshold }}
+ failureThreshold: {{ .Values.webhook.readinessProbe.failureThreshold }}
+ {{- with .Values.webhook.containerSecurityContext }}
+ securityContext:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ env:
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ {{- with .Values.webhook.resources }}
+ resources:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- if .Values.webhook.config }}
+ volumeMounts:
+ - name: config
+ mountPath: /var/cert-manager/config
+ {{- end }}
+ {{- with .Values.webhook.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.webhook.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.webhook.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.webhook.config }}
+ volumes:
+ - name: config
+ configMap:
+ name: {{ include "webhook.fullname" . }}
+ {{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-mutating-webhook.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-mutating-webhook.yaml
new file mode 100644
index 0000000..fb8b795
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-mutating-webhook.yaml
@@ -0,0 +1,46 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+ name: {{ include "webhook.fullname" . }}
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+ annotations:
+ cert-manager.io/inject-ca-from-secret: "{{ .Release.Namespace }}/{{ template "webhook.fullname" . }}-ca"
+ {{- with .Values.webhook.mutatingWebhookConfigurationAnnotations }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+webhooks:
+ - name: webhook.cert-manager.io
+ rules:
+ - apiGroups:
+ - "cert-manager.io"
+ - "acme.cert-manager.io"
+ apiVersions:
+ - "v1"
+ operations:
+ - CREATE
+ - UPDATE
+ resources:
+ - "*/*"
+ admissionReviewVersions: ["v1"]
+ # This webhook only accepts v1 cert-manager resources.
+ # Equivalent matchPolicy ensures that non-v1 resource requests are sent to
+ # this webhook (after the resources have been converted to v1).
+ matchPolicy: Equivalent
+ timeoutSeconds: {{ .Values.webhook.timeoutSeconds }}
+ failurePolicy: Fail
+ # Only include 'sideEffects' field in Kubernetes 1.12+
+ sideEffects: None
+ clientConfig:
+ {{- if .Values.webhook.url.host }}
+ url: https://{{ .Values.webhook.url.host }}/mutate
+ {{- else }}
+ service:
+ name: {{ template "webhook.fullname" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ path: /mutate
+ {{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-psp-clusterrole.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-psp-clusterrole.yaml
new file mode 100644
index 0000000..2a8808e
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-psp-clusterrole.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.global.podSecurityPolicy.enabled }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: {{ template "webhook.fullname" . }}-psp
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+rules:
+- apiGroups: ['policy']
+ resources: ['podsecuritypolicies']
+ verbs: ['use']
+ resourceNames:
+ - {{ template "webhook.fullname" . }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-psp-clusterrolebinding.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-psp-clusterrolebinding.yaml
new file mode 100644
index 0000000..e8e1bb2
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-psp-clusterrolebinding.yaml
@@ -0,0 +1,20 @@
+{{- if .Values.global.podSecurityPolicy.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "webhook.fullname" . }}-psp
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "webhook.fullname" . }}-psp
+subjects:
+ - kind: ServiceAccount
+ name: {{ template "webhook.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-psp.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-psp.yaml
new file mode 100644
index 0000000..4d5d959
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-psp.yaml
@@ -0,0 +1,54 @@
+{{- if .Values.global.podSecurityPolicy.enabled }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+ name: {{ template "webhook.fullname" . }}
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+ annotations:
+ seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+ seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default'
+ {{- if .Values.global.podSecurityPolicy.useAppArmor }}
+ apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+ apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default'
+ {{- end }}
+spec:
+ privileged: false
+ allowPrivilegeEscalation: false
+ allowedCapabilities: [] # default set of capabilities are implicitly allowed
+ volumes:
+ - 'configMap'
+ - 'emptyDir'
+ - 'projected'
+ - 'secret'
+ - 'downwardAPI'
+ hostNetwork: {{ .Values.webhook.hostNetwork }}
+ {{- if .Values.webhook.hostNetwork }}
+ hostPorts:
+ - max: {{ .Values.webhook.securePort }}
+ min: {{ .Values.webhook.securePort }}
+ {{- end }}
+ hostIPC: false
+ hostPID: false
+ runAsUser:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+ seLinux:
+ rule: 'RunAsAny'
+ supplementalGroups:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+ fsGroup:
+ rule: 'MustRunAs'
+ ranges:
+ - min: 1000
+ max: 1000
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-rbac.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-rbac.yaml
new file mode 100644
index 0000000..1070333
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-rbac.yaml
@@ -0,0 +1,83 @@
+{{- if .Values.global.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ template "webhook.fullname" . }}:dynamic-serving
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+rules:
+- apiGroups: [""]
+ resources: ["secrets"]
+ resourceNames:
+ - '{{ template "webhook.fullname" . }}-ca'
+ verbs: ["get", "list", "watch", "update"]
+# It's not possible to grant CREATE permission on a single resourceName.
+- apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["create"]
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ template "webhook.fullname" . }}:dynamic-serving
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ template "webhook.fullname" . }}:dynamic-serving
+subjects:
+- apiGroup: ""
+ kind: ServiceAccount
+ name: {{ template "webhook.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ template "webhook.fullname" . }}:subjectaccessreviews
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+rules:
+- apiGroups: ["authorization.k8s.io"]
+ resources: ["subjectaccessreviews"]
+ verbs: ["create"]
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ template "webhook.fullname" . }}:subjectaccessreviews
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ template "webhook.fullname" . }}:subjectaccessreviews
+subjects:
+- apiGroup: ""
+ kind: ServiceAccount
+ name: {{ template "webhook.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-service.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-service.yaml
new file mode 100644
index 0000000..ed0278a
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-service.yaml
@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ template "webhook.fullname" . }}
+ namespace: {{ .Release.Namespace | quote }}
+{{- with .Values.webhook.serviceAnnotations }}
+ annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.webhook.serviceLabels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ type: {{ .Values.webhook.serviceType }}
+ {{- with .Values.webhook.loadBalancerIP }}
+ loadBalancerIP: {{ . }}
+ {{- end }}
+ ports:
+ - name: https
+ port: 443
+ protocol: TCP
+ targetPort: "https"
+ selector:
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-serviceaccount.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-serviceaccount.yaml
new file mode 100644
index 0000000..2792a43
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-serviceaccount.yaml
@@ -0,0 +1,25 @@
+{{- if .Values.webhook.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{ .Values.webhook.serviceAccount.automountServiceAccountToken }}
+metadata:
+ name: {{ template "webhook.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ {{- with .Values.webhook.serviceAccount.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+ {{- with .Values.webhook.serviceAccount.labels }}
+ {{ toYaml . | nindent 4 }}
+ {{- end }}
+{{- with .Values.global.imagePullSecrets }}
+imagePullSecrets:
+ {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-validating-webhook.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-validating-webhook.yaml
new file mode 100644
index 0000000..2ca6403
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/templates/webhook-validating-webhook.yaml
@@ -0,0 +1,55 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+ name: {{ include "webhook.fullname" . }}
+ labels:
+ app: {{ include "webhook.name" . }}
+ app.kubernetes.io/name: {{ include "webhook.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/component: "webhook"
+ {{- include "labels" . | nindent 4 }}
+ annotations:
+ cert-manager.io/inject-ca-from-secret: "{{ .Release.Namespace }}/{{ template "webhook.fullname" . }}-ca"
+ {{- with .Values.webhook.validatingWebhookConfigurationAnnotations }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+webhooks:
+ - name: webhook.cert-manager.io
+ namespaceSelector:
+ matchExpressions:
+ - key: "cert-manager.io/disable-validation"
+ operator: "NotIn"
+ values:
+ - "true"
+ - key: "name"
+ operator: "NotIn"
+ values:
+ - {{ .Release.Namespace }}
+ rules:
+ - apiGroups:
+ - "cert-manager.io"
+ - "acme.cert-manager.io"
+ apiVersions:
+ - "v1"
+ operations:
+ - CREATE
+ - UPDATE
+ resources:
+ - "*/*"
+ admissionReviewVersions: ["v1"]
+ # This webhook only accepts v1 cert-manager resources.
+ # Equivalent matchPolicy ensures that non-v1 resource requests are sent to
+ # this webhook (after the resources have been converted to v1).
+ matchPolicy: Equivalent
+ timeoutSeconds: {{ .Values.webhook.timeoutSeconds }}
+ failurePolicy: Fail
+ sideEffects: None
+ clientConfig:
+ {{- if .Values.webhook.url.host }}
+ url: https://{{ .Values.webhook.url.host }}/validate
+ {{- else }}
+ service:
+ name: {{ template "webhook.fullname" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ path: /validate
+ {{- end }}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/cert-manager/values.yaml b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/values.yaml
new file mode 100644
index 0000000..3a21b15
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/cert-manager/values.yaml
@@ -0,0 +1,543 @@
+# Default values for cert-manager.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+global:
+ ## Reference to one or more secrets to be used when pulling images
+ ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+ ##
+ imagePullSecrets: []
+ # - name: "image-pull-secret"
+
+ # Optional priority class to be used for the cert-manager pods
+ priorityClassName: ""
+ rbac:
+ create: true
+ # Aggregate ClusterRoles to Kubernetes default user-facing roles. Ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles
+ aggregateClusterRoles: true
+
+ podSecurityPolicy:
+ enabled: false
+ useAppArmor: true
+
+ # Set the verbosity of cert-manager. Range of 0 - 6 with 6 being the most verbose.
+ logLevel: 2
+
+ leaderElection:
+ # Override the namespace used to store the ConfigMap for leader election
+ namespace: "kube-system"
+
+ # The duration that non-leader candidates will wait after observing a
+ # leadership renewal until attempting to acquire leadership of a led but
+ # unrenewed leader slot. This is effectively the maximum duration that a
+ # leader can be stopped before it is replaced by another candidate.
+ # leaseDuration: 60s
+
+ # The interval between attempts by the acting master to renew a leadership
+ # slot before it stops leading. This must be less than or equal to the
+ # lease duration.
+ # renewDeadline: 40s
+
+ # The duration the clients should wait between attempting acquisition and
+ # renewal of a leadership.
+ # retryPeriod: 15s
+
+installCRDs: false
+
+replicaCount: 1
+
+strategy: {}
+ # type: RollingUpdate
+ # rollingUpdate:
+ # maxSurge: 0
+ # maxUnavailable: 1
+
+# Comma separated list of feature gates that should be enabled on the
+# controller pod.
+featureGates: ""
+
+image:
+ repository: quay.io/jetstack/cert-manager-controller
+ # You can manage a registry with
+ # registry: quay.io
+ # repository: jetstack/cert-manager-controller
+
+ # Override the image tag to deploy by setting this variable.
+ # If no value is set, the chart's appVersion will be used.
+ # tag: canary
+
+ # Setting a digest will override any tag
+ # digest: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20
+ pullPolicy: IfNotPresent
+
+# Override the namespace used to store DNS provider credentials etc. for ClusterIssuer
+# resources. By default, the same namespace as cert-manager is deployed within is
+# used. This namespace will not be automatically created by the Helm chart.
+clusterResourceNamespace: ""
+
+serviceAccount:
+ # Specifies whether a service account should be created
+ create: true
+ # The name of the service account to use.
+ # If not set and create is true, a name is generated using the fullname template
+ # name: ""
+ # Optional additional annotations to add to the controller's ServiceAccount
+ # annotations: {}
+ # Automount API credentials for a Service Account.
+ # Optional additional labels to add to the controller's ServiceAccount
+ # labels: {}
+ automountServiceAccountToken: true
+
+# Additional command line flags to pass to cert-manager controller binary.
+# To see all available flags run docker run quay.io/jetstack/cert-manager-controller: --help
+extraArgs: []
+ # When this flag is enabled, secrets will be automatically removed when the certificate resource is deleted
+ # - --enable-certificate-owner-ref=true
+ # Use this flag to enabled or disable arbitrary controllers, for example, disable the CertificiateRequests approver
+ # - --controllers=*,-certificaterequests-approver
+
+extraEnv: []
+# - name: SOME_VAR
+# value: 'some value'
+
+resources: {}
+ # requests:
+ # cpu: 10m
+ # memory: 32Mi
+
+# Pod Security Context
+# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+securityContext:
+ runAsNonRoot: true
+# legacy securityContext parameter format: if enabled is set to true, only fsGroup and runAsUser are supported
+# securityContext:
+# enabled: false
+# fsGroup: 1001
+# runAsUser: 1001
+# to support additional securityContext parameters, omit the `enabled` parameter and simply specify the parameters
+# you want to set, e.g.
+# securityContext:
+# fsGroup: 1000
+# runAsUser: 1000
+# runAsNonRoot: true
+
+# Container Security Context to be set on the controller component container
+# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+containerSecurityContext:
+ allowPrivilegeEscalation: false
+ # capabilities:
+ # drop:
+ # - ALL
+ # readOnlyRootFilesystem: true
+ # runAsNonRoot: true
+
+
+volumes: []
+
+volumeMounts: []
+
+# Optional additional annotations to add to the controller Deployment
+# deploymentAnnotations: {}
+
+# Optional additional annotations to add to the controller Pods
+# podAnnotations: {}
+
+podLabels: {}
+
+# Optional annotations to add to the controller Service
+# serviceAnnotations: {}
+
+# Optional additional labels to add to the controller Service
+# serviceLabels: {}
+
+# Optional DNS settings, useful if you have a public and private DNS zone for
+# the same domain on Route 53. What follows is an example of ensuring
+# cert-manager can access an ingress or DNS TXT records at all times.
+# NOTE: This requires Kubernetes 1.10 or `CustomPodDNS` feature gate enabled for
+# the cluster to work.
+# podDnsPolicy: "None"
+# podDnsConfig:
+# nameservers:
+# - "1.1.1.1"
+# - "8.8.8.8"
+
+nodeSelector:
+ kubernetes.io/os: linux
+
+ingressShim: {}
+ # defaultIssuerName: ""
+ # defaultIssuerKind: ""
+ # defaultIssuerGroup: ""
+
+prometheus:
+ enabled: true
+ servicemonitor:
+ enabled: false
+ prometheusInstance: default
+ targetPort: 9402
+ path: /metrics
+ interval: 60s
+ scrapeTimeout: 30s
+ labels: {}
+ honorLabels: false
+
+# Use these variables to configure the HTTP_PROXY environment variables
+# http_proxy: "http://proxy:8080"
+# https_proxy: "https://proxy:8080"
+# no_proxy: 127.0.0.1,localhost
+
+# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core
+# for example:
+# affinity:
+# nodeAffinity:
+# requiredDuringSchedulingIgnoredDuringExecution:
+# nodeSelectorTerms:
+# - matchExpressions:
+# - key: foo.bar.com/role
+# operator: In
+# values:
+# - master
+affinity: {}
+
+# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core
+# for example:
+# tolerations:
+# - key: foo.bar.com/role
+# operator: Equal
+# value: master
+# effect: NoSchedule
+tolerations: []
+
+webhook:
+ replicaCount: 1
+ timeoutSeconds: 10
+
+ # Used to configure options for the webhook pod.
+ # This allows setting options that'd usually be provided via flags.
+ # An APIVersion and Kind must be specified in your values.yaml file.
+ # Flags will override options that are set here.
+ config:
+ # apiVersion: webhook.config.cert-manager.io/v1alpha1
+ # kind: WebhookConfiguration
+
+ # The port that the webhook should listen on for requests.
+ # In GKE private clusters, by default kubernetes apiservers are allowed to
+ # talk to the cluster nodes only on 443 and 10250. so configuring
+ # securePort: 10250, will work out of the box without needing to add firewall
+ # rules or requiring NET_BIND_SERVICE capabilities to bind port numbers <1000.
+ # This should be uncommented and set as a default by the chart once we graduate
+ # the apiVersion of WebhookConfiguration past v1alpha1.
+ # securePort: 10250
+
+ strategy: {}
+ # type: RollingUpdate
+ # rollingUpdate:
+ # maxSurge: 0
+ # maxUnavailable: 1
+
+ # Pod Security Context to be set on the webhook component Pod
+ # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+ securityContext:
+ runAsNonRoot: true
+
+ # Container Security Context to be set on the webhook component container
+ # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ # capabilities:
+ # drop:
+ # - ALL
+ # readOnlyRootFilesystem: true
+ # runAsNonRoot: true
+
+ # Optional additional annotations to add to the webhook Deployment
+ # deploymentAnnotations: {}
+
+ # Optional additional annotations to add to the webhook Pods
+ # podAnnotations: {}
+
+ # Optional additional annotations to add to the webhook Service
+ # serviceAnnotations: {}
+
+ # Optional additional annotations to add to the webhook MutatingWebhookConfiguration
+ # mutatingWebhookConfigurationAnnotations: {}
+
+ # Optional additional annotations to add to the webhook ValidatingWebhookConfiguration
+ # validatingWebhookConfigurationAnnotations: {}
+
+ # Additional command line flags to pass to cert-manager webhook binary.
+ # To see all available flags run docker run quay.io/jetstack/cert-manager-webhook: --help
+ extraArgs: []
+ # Path to a file containing a WebhookConfiguration object used to configure the webhook
+ # - --config=
+
+ resources: {}
+ # requests:
+ # cpu: 10m
+ # memory: 32Mi
+
+ ## Liveness and readiness probe values
+ ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes
+ ##
+ livenessProbe:
+ failureThreshold: 3
+ initialDelaySeconds: 60
+ periodSeconds: 10
+ successThreshold: 1
+ timeoutSeconds: 1
+ readinessProbe:
+ failureThreshold: 3
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ successThreshold: 1
+ timeoutSeconds: 1
+
+ nodeSelector:
+ kubernetes.io/os: linux
+
+ affinity: {}
+
+ tolerations: []
+
+ # Optional additional labels to add to the Webhook Pods
+ podLabels: {}
+
+ # Optional additional labels to add to the Webhook Service
+ serviceLabels: {}
+
+ image:
+ repository: quay.io/jetstack/cert-manager-webhook
+ # You can manage a registry with
+ # registry: quay.io
+ # repository: jetstack/cert-manager-webhook
+
+ # Override the image tag to deploy by setting this variable.
+ # If no value is set, the chart's appVersion will be used.
+ # tag: canary
+
+ # Setting a digest will override any tag
+ # digest: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20
+
+ pullPolicy: IfNotPresent
+
+ serviceAccount:
+ # Specifies whether a service account should be created
+ create: true
+ # The name of the service account to use.
+ # If not set and create is true, a name is generated using the fullname template
+ # name: ""
+ # Optional additional annotations to add to the controller's ServiceAccount
+ # annotations: {}
+ # Optional additional labels to add to the webhook's ServiceAccount
+ # labels: {}
+ # Automount API credentials for a Service Account.
+ automountServiceAccountToken: true
+
+ # The port that the webhook should listen on for requests.
+ # In GKE private clusters, by default kubernetes apiservers are allowed to
+ # talk to the cluster nodes only on 443 and 10250. so configuring
+ # securePort: 10250, will work out of the box without needing to add firewall
+ # rules or requiring NET_BIND_SERVICE capabilities to bind port numbers <1000
+ securePort: 10250
+
+ # Specifies if the webhook should be started in hostNetwork mode.
+ #
+ # Required for use in some managed kubernetes clusters (such as AWS EKS) with custom
+ # CNI (such as calico), because control-plane managed by AWS cannot communicate
+ # with pods' IP CIDR and admission webhooks are not working
+ #
+ # Since the default port for the webhook conflicts with kubelet on the host
+ # network, `webhook.securePort` should be changed to an available port if
+ # running in hostNetwork mode.
+ hostNetwork: false
+
+ # Specifies how the service should be handled. Useful if you want to expose the
+ # webhook to outside of the cluster. In some cases, the control plane cannot
+ # reach internal services.
+ serviceType: ClusterIP
+ # loadBalancerIP:
+
+ # Overrides the mutating webhook and validating webhook so they reach the webhook
+ # service using the `url` field instead of a service.
+ url: {}
+ # host:
+
+cainjector:
+ enabled: true
+ replicaCount: 1
+
+ strategy: {}
+ # type: RollingUpdate
+ # rollingUpdate:
+ # maxSurge: 0
+ # maxUnavailable: 1
+
+ # Pod Security Context to be set on the cainjector component Pod
+ # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+ securityContext:
+ runAsNonRoot: true
+
+ # Container Security Context to be set on the cainjector component container
+ # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ # capabilities:
+ # drop:
+ # - ALL
+ # readOnlyRootFilesystem: true
+ # runAsNonRoot: true
+
+
+ # Optional additional annotations to add to the cainjector Deployment
+ # deploymentAnnotations: {}
+
+ # Optional additional annotations to add to the cainjector Pods
+ # podAnnotations: {}
+
+ # Additional command line flags to pass to cert-manager cainjector binary.
+ # To see all available flags run docker run quay.io/jetstack/cert-manager-cainjector: --help
+ extraArgs: []
+ # Enable profiling for cainjector
+ # - --enable-profiling=true
+
+ resources: {}
+ # requests:
+ # cpu: 10m
+ # memory: 32Mi
+
+ nodeSelector:
+ kubernetes.io/os: linux
+
+ affinity: {}
+
+ tolerations: []
+
+ # Optional additional labels to add to the CA Injector Pods
+ podLabels: {}
+
+ image:
+ repository: quay.io/jetstack/cert-manager-cainjector
+ # You can manage a registry with
+ # registry: quay.io
+ # repository: jetstack/cert-manager-cainjector
+
+ # Override the image tag to deploy by setting this variable.
+ # If no value is set, the chart's appVersion will be used.
+ # tag: canary
+
+ # Setting a digest will override any tag
+ # digest: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20
+
+ pullPolicy: IfNotPresent
+
+ serviceAccount:
+ # Specifies whether a service account should be created
+ create: true
+ # The name of the service account to use.
+ # If not set and create is true, a name is generated using the fullname template
+ # name: ""
+ # Optional additional annotations to add to the controller's ServiceAccount
+ # annotations: {}
+ # Automount API credentials for a Service Account.
+ # Optional additional labels to add to the cainjector's ServiceAccount
+ # labels: {}
+ automountServiceAccountToken: true
+
+# This startupapicheck is a Helm post-install hook that waits for the webhook
+# endpoints to become available.
+# The check is implemented using a Kubernetes Job- if you are injecting mesh
+# sidecar proxies into cert-manager pods, you probably want to ensure that they
+# are not injected into this Job's pod. Otherwise the installation may time out
+# due to the Job never being completed because the sidecar proxy does not exit.
+# See https://github.com/cert-manager/cert-manager/pull/4414 for context.
+startupapicheck:
+ enabled: true
+
+ # Pod Security Context to be set on the startupapicheck component Pod
+ # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+ securityContext:
+ runAsNonRoot: true
+
+ # Container Security Context to be set on the controller component container
+ # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ # capabilities:
+ # drop:
+ # - ALL
+ # readOnlyRootFilesystem: true
+ # runAsNonRoot: true
+
+ # Timeout for 'kubectl check api' command
+ timeout: 1m
+
+ # Job backoffLimit
+ backoffLimit: 4
+
+ # Optional additional annotations to add to the startupapicheck Job
+ jobAnnotations:
+ helm.sh/hook: post-install
+ helm.sh/hook-weight: "1"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+
+ # Optional additional annotations to add to the startupapicheck Pods
+ # podAnnotations: {}
+
+ # Additional command line flags to pass to startupapicheck binary.
+ # To see all available flags run docker run quay.io/jetstack/cert-manager-ctl: --help
+ extraArgs: []
+
+ resources: {}
+ # requests:
+ # cpu: 10m
+ # memory: 32Mi
+
+ nodeSelector: {}
+
+ affinity: {}
+
+ tolerations: []
+
+ # Optional additional labels to add to the startupapicheck Pods
+ podLabels: {}
+
+ image:
+ repository: quay.io/jetstack/cert-manager-ctl
+ # You can manage a registry with
+ # registry: quay.io
+ # repository: jetstack/cert-manager-ctl
+
+ # Override the image tag to deploy by setting this variable.
+ # If no value is set, the chart's appVersion will be used.
+ # tag: canary
+
+ # Setting a digest will override any tag
+ # digest: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20
+
+ pullPolicy: IfNotPresent
+
+ rbac:
+ # annotations for the startup API Check job RBAC and PSP resources
+ annotations:
+ helm.sh/hook: post-install
+ helm.sh/hook-weight: "-5"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+
+ serviceAccount:
+ # Specifies whether a service account should be created
+ create: true
+
+ # The name of the service account to use.
+ # If not set and create is true, a name is generated using the fullname template
+ # name: ""
+
+ # Optional additional annotations to add to the Job's ServiceAccount
+ annotations:
+ helm.sh/hook: post-install
+ helm.sh/hook-weight: "-5"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+
+ # Automount API credentials for a Service Account.
+ automountServiceAccountToken: true
+
+ # Optional additional labels to add to the startupapicheck's ServiceAccount
+ # labels: {}
diff --git a/observability/lgtm-distributed/charts/oncall/charts/grafana/.helmignore b/observability/lgtm-distributed/charts/oncall/charts/grafana/.helmignore
new file mode 100644
index 0000000..8cade13
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/grafana/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.vscode
+.project
+.idea/
+*.tmproj
+OWNERS
diff --git a/observability/lgtm-distributed/charts/oncall/charts/grafana/Chart.yaml b/observability/lgtm-distributed/charts/oncall/charts/grafana/Chart.yaml
new file mode 100644
index 0000000..e022cdf
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/grafana/Chart.yaml
@@ -0,0 +1,29 @@
+annotations:
+ artifacthub.io/links: |
+ - name: Chart Source
+ url: https://github.com/grafana/helm-charts
+ - name: Upstream Project
+ url: https://github.com/grafana/grafana
+apiVersion: v2
+appVersion: 9.5.3
+description: The leading tool for querying and visualizing time series and metrics.
+home: https://grafana.net
+icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png
+kubeVersion: ^1.8.0-0
+maintainers:
+- email: zanhsieh@gmail.com
+ name: zanhsieh
+- email: rluckie@cisco.com
+ name: rtluckie
+- email: maor.friedman@redhat.com
+ name: maorfr
+- email: miroslav.hadzhiev@gmail.com
+ name: Xtigyro
+- email: mail@torstenwalter.de
+ name: torstenwalter
+name: grafana
+sources:
+- https://github.com/grafana/grafana
+- https://github.com/grafana/helm-charts
+type: application
+version: 6.57.1
diff --git a/observability/lgtm-distributed/charts/oncall/charts/grafana/README.md b/observability/lgtm-distributed/charts/oncall/charts/grafana/README.md
new file mode 100644
index 0000000..0da2569
--- /dev/null
+++ b/observability/lgtm-distributed/charts/oncall/charts/grafana/README.md
@@ -0,0 +1,685 @@
+# Grafana Helm Chart
+
+* Installs the web dashboarding system [Grafana](http://grafana.org/)
+
+## Get Repo Info
+
+```console
+helm repo add grafana https://grafana.github.io/helm-charts
+helm repo update
+```
+
+_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._
+
+## Installing the Chart
+
+To install the chart with the release name `my-release`:
+
+```console
+helm install my-release grafana/grafana
+```
+
+## Uninstalling the Chart
+
+To uninstall/delete the my-release deployment:
+
+```console
+helm delete my-release
+```
+
+The command removes all the Kubernetes components associated with the chart and deletes the release.
+
+## Upgrading an existing Release to a new major version
+
+A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an
+incompatible breaking change needing manual actions.
+
+### To 4.0.0 (And 3.12.1)
+
+This version requires Helm >= 2.12.0.
+
+### To 5.0.0
+
+You have to add --force to your helm upgrade command as the labels of the chart have changed.
+
+### To 6.0.0
+
+This version requires Helm >= 3.1.0.
+
+## Configuration
+
+| Parameter | Description | Default |
+|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------|
+| `replicas` | Number of nodes | `1` |
+| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` |
+| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` |
+| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` |
+| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` |
+| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`|
+| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` |
+| `priorityClassName` | Name of Priority Class to assign pods | `nil` |
+| `image.repository` | Image repository | `grafana/grafana` |
+| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` |
+| `image.sha` | Image sha (optional) | `` |
+| `image.pullPolicy` | Image pull policy | `IfNotPresent` |
+| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` |
+| `service.enabled` | Enable grafana service | `true` |
+| `service.type` | Kubernetes service type | `ClusterIP` |
+| `service.port` | Kubernetes port where service is exposed | `80` |
+| `service.portName` | Name of the port on the service | `service` |
+| `service.appProtocol` | Adds the appProtocol field to the service | `` |
+| `service.targetPort` | Internal service is port | `3000` |
+| `service.nodePort` | Kubernetes service nodePort | `nil` |
+| `service.annotations` | Service annotations (can be templated) | `{}` |
+| `service.labels` | Custom labels | `{}` |
+| `service.clusterIP` | internal cluster service IP | `nil` |
+| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` |
+| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` |
+| `service.externalIPs` | service external IP addresses | `[]` |
+| `headlessService` | Create a headless service | `false` |
+| `extraExposePorts` | Additional service ports for sidecar containers| `[]` |
+| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` |
+| `ingress.enabled` | Enables Ingress | `false` |
+| `ingress.annotations` | Ingress annotations (values are templated) | `{}` |
+| `ingress.labels` | Custom labels | `{}` |
+| `ingress.path` | Ingress accepted path | `/` |
+| `ingress.pathType` | Ingress type of path | `Prefix` |
+| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` |
+| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-alb-ingress-controller/guide/ingress/annotation/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` |
+| `ingress.tls` | Ingress TLS configuration | `[]` |
+| `ingress.ingressClassName` | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""` |
+| `resources` | CPU/Memory resource requests/limits | `{}` |
+| `nodeSelector` | Node labels for pod assignment | `{}` |
+| `tolerations` | Toleration labels for pod assignment | `[]` |
+| `affinity` | Affinity settings for pod assignment | `{}` |
+| `extraInitContainers` | Init containers to add to the grafana pod | `{}` |
+| `extraContainers` | Sidecar containers to add to the grafana pod | `""` |
+| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` |
+| `extraLabels` | Custom labels for all manifests | `{}` |
+| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` |
+| `persistence.enabled` | Use persistent volume to store data | `false` |
+| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` |
+| `persistence.size` | Size of persistent volume claim | `10Gi` |
+| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` |
+| `persistence.storageClassName` | Type of persistent volume claim | `nil` |
+| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` |
+| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` |
+| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` |
+| `persistence.extraPvcLabels` | Extra labels to apply to a PVC. | `{}` |
+| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` |
+| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` |
+| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` |
+| `initChownData.enabled` | If false, don't reset data ownership at startup | true |
+| `initChownData.image.repository` | init-chown-data container image repository | `busybox` |
+| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` |
+| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` |
+| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` |
+| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` |
+| `schedulerName` | Alternate scheduler name | `nil` |
+| `env` | Extra environment variables passed to pods | `{}` |
+| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` |
+| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` |
+| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` |
+| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` |
+| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret. (passed through [tpl](https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function)) | `{}` |
+| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` |
+| `extraSecretMounts` | Additional grafana server secret mounts | `[]` |
+| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` |
+| `createConfigmap` | Enable creating the grafana configmap | `true` |
+| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` |
+| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` |
+| `plugins` | Plugins to be loaded along with Grafana | `[]` |
+| `datasources` | Configure grafana datasources (passed through tpl) | `{}` |
+| `alerting` | Configure grafana alerting (passed through tpl) | `{}` |
+| `notifiers` | Configure grafana notifiers | `{}` |
+| `dashboardProviders` | Configure grafana dashboard providers | `{}` |
+| `dashboards` | Dashboards to import | `{}` |
+| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` |
+| `grafana.ini` | Grafana's primary configuration | `{}` |
+| `global.imagePullSecrets` | Global image pull secrets (can be templated). Allows either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). | `[]` |
+| `ldap.enabled` | Enable LDAP authentication | `false` |
+| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` |
+| `ldap.config` | Grafana's LDAP configuration | `""` |
+| `annotations` | Deployment annotations | `{}` |
+| `labels` | Deployment labels | `{}` |
+| `podAnnotations` | Pod annotations | `{}` |
+| `podLabels` | Pod labels | `{}` |
+| `podPortName` | Name of the grafana port on the pod | `grafana` |
+| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` |
+| `sidecar.image.repository` | Sidecar image repository | `quay.io/kiwigrid/k8s-sidecar` |
+| `sidecar.image.tag` | Sidecar image tag | `1.24.3` |
+| `sidecar.image.sha` | Sidecar image sha (optional) | `""` |
+| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` |
+| `sidecar.resources` | Sidecar resources | `{}` |
+| `sidecar.securityContext` | Sidecar securityContext | `{}` |
+| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` |
+| `sidecar.alerts.enabled` | Enables the cluster wide search for alerts and adds/updates/deletes them in grafana |`false` |
+| `sidecar.alerts.label` | Label that config maps with alerts should have to be added | `grafana_alert` |
+| `sidecar.alerts.labelValue` | Label value that config maps with alerts should have to be added | `""` |
+| `sidecar.alerts.searchNamespace` | Namespaces list. If specified, the sidecar will search for alerts config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` |
+| `sidecar.alerts.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` |
+| `sidecar.alerts.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` |
+| `sidecar.alerts.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/alerting/reload"` |
+| `sidecar.alerts.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` |
+| `sidecar.alerts.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any alerts defined at startup time. | `false` |
+| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` |
+| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` |
+| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` |
+| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` |
+| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` |
+| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` |
+| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` |
+| `sidecar.dashboards.provider.type` | Provider type | `file` |
+| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` |
+| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` |
+| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` |
+| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` |
+| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` |
+| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` |
+| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` |
+| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` |
+| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` |
+| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` |
+| `sidecar.dashboards.reloadURL` | Full url of dashboards configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/dashboards/reload"` |
+| `sidecar.dashboards.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` |
+| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` |
+| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` |
+| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` |
+| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` |
+| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` |
+| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` |
+| `sidecar.datasources.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` |
+| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` |
+| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` |
+| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` |
+| `sidecar.datasources.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any datasources defined at startup time. | `false` |
+| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` |
+| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` |
+| `sidecar.notifiers.labelValue` | Label value that config maps with notifiers should have to be added | `""` |
+| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` |
+| `sidecar.notifiers.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` |
+| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` |
+| `sidecar.notifiers.reloadURL` | Full url of notifier configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/notifications/reload"` |
+| `sidecar.notifiers.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` |
+| `sidecar.notifiers.initNotifiers` | Set to true to deploy the notifier sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any notifiers defined at startup time. | `false` |
+| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` |
+| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` |
+| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` |
+| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` |
+| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` |
+| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` |
+| `serviceAccount.autoMount` | Automount the service account token in the pod| `true` |
+| `serviceAccount.annotations` | ServiceAccount annotations | |
+| `serviceAccount.create` | Create service account | `true` |
+| `serviceAccount.labels` | ServiceAccount labels | `{}` |
+| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` |
+| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` |
+| `rbac.create` | Create and use RBAC resources | `true` |
+| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` |
+| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` |
+| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false` |
+| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `false` |
+| `rbac.extraRoleRules` | Additional rules to add to the Role | [] |
+| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] |
+| `command` | Define command to be executed by grafana container at startup | `nil` |
+| `args` | Define additional args if command is used | `nil` |
+| `testFramework.enabled` | Whether to create test-related resources | `true` |
+| `testFramework.image` | `test-framework` image repository. | `bats/bats` |
+| `testFramework.tag` | `test-framework` image tag. | `v1.4.1` |
+| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` |
+| `testFramework.securityContext` | `test-framework` securityContext | `{}` |
+| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` |
+| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` |
+| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` |
+| `downloadDashboardsImage.repository` | Curl docker image repo | `curlimages/curl` |
+| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` |
+| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` |
+| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` |
+| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) |
+| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` |
+| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | |
+| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` |
+| `serviceMonitor.path` | Path to scrape | `/metrics` |
+| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` |
+| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` |
+| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` |
+| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` |
+| `serviceMonitor.relabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` |
+| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` |
+| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` |
+| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` |
+| `imageRenderer.image.tag` | image-renderer Image tag | `latest` |
+| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` |
+| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` |
+| `imageRenderer.env` | extra env-vars for image-renderer | `{}` |
+| `imageRenderer.envValueFrom` | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` |
+| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` |
+| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` |
+| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` |
+| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` |
+| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` |
+| `imageRenderer.service.portName` | image-renderer service port name | `http` |
+| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` |
+| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` |
+| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` |
+| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` |
+| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` |
+| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` |
+| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` |
+| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` |
+| `imageRenderer.resources` | Set resource limits for image-renderer pdos | `{}` |
+| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` |
+| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` |
+| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` |
+| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` |
+| `networkPolicy.allowExternal` | Don't require client label for connections | `true` |
+| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` |
+| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` |
+| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` |
+| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` |
+| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` |
+
+### Example ingress with path
+
+With grafana 6.3 and above
+
+```yaml
+grafana.ini:
+ server:
+ domain: monitoring.example.com
+ root_url: "%(protocol)s://%(domain)s/grafana"
+ serve_from_sub_path: true
+ingress:
+ enabled: true
+ hosts:
+ - "monitoring.example.com"
+ path: "/grafana"
+```
+
+### Example of extraVolumeMounts
+
+Volume can be type persistentVolumeClaim or hostPath but not both at same time.
+If neither existingClaim or hostPath argument is given then type is emptyDir.
+
+```yaml
+- extraVolumeMounts:
+ - name: plugins
+ mountPath: /var/lib/grafana/plugins
+ subPath: configs/grafana/plugins
+ existingClaim: existing-grafana-claim
+ readOnly: false
+ - name: dashboards
+ mountPath: /var/lib/grafana/dashboards
+ hostPath: /usr/shared/grafana/dashboards
+ readOnly: false
+```
+
+## Import dashboards
+
+There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method:
+
+```yaml
+dashboards:
+ default:
+ some-dashboard:
+ json: |
+ {
+ "annotations":
+
+ ...
+ # Complete json file here
+ ...
+
+ "title": "Some Dashboard",
+ "uid": "abcd1234",
+ "version": 1
+ }
+ custom-dashboard:
+ # This is a path to a file inside the dashboards directory inside the chart directory
+ file: dashboards/custom-dashboard.json
+ prometheus-stats:
+ # Ref: https://grafana.com/dashboards/2
+ gnetId: 2
+ revision: 2
+ datasource: Prometheus
+ loki-dashboard-quick-search:
+ gnetId: 12019
+ revision: 2
+ datasource:
+ - name: DS_PROMETHEUS
+ value: Prometheus
+ - name: DS_LOKI
+ value: Loki
+ local-dashboard:
+ url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json
+```
+
+## BASE64 dashboards
+
+Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit)
+A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk.
+If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk.
+
+### Gerrit use case
+
+Gerrit API for download files has the following schema: where {project-name} and
+{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard
+the url value is
+
+## Sidecar for dashboards
+
+If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana
+pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with
+a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written
+to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported
+dashboards are deleted/updated.
+
+A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside
+one configmap is currently not properly mirrored in grafana.
+
+Example dashboard config:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: sample-grafana-dashboard
+ labels:
+ grafana_dashboard: "1"
+data:
+ k8s-dashboard.json: |-
+ [...]
+```
+
+## Sidecar for datasources
+
+If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana
+pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and
+filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in
+those secrets are written to a folder and accessed by grafana on startup. Using these yaml files,
+the data sources in grafana can be imported.
+
+Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://..svc.cluster.local/api/admin/provisioning/datasources/reload`.
+
+Secrets are recommended over configmaps for this usecase because datasources usually contain private
+data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those.
+
+Example values to add a postgres datasource as a kubernetes secret:
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+ name: grafana-datasources
+ labels:
+ grafana_datasource: 'true' # default value for: sidecar.datasources.label
+stringData:
+ pg-db.yaml: |-
+ apiVersion: 1
+ datasources:
+ - name: My pg db datasource
+ type: postgres
+ url: my-postgresql-db:5432
+ user: db-readonly-user
+ secureJsonData:
+ password: 'SUperSEcretPa$$word'
+ jsonData:
+ database: my_datase
+ sslmode: 'disable' # disable/require/verify-ca/verify-full
+ maxOpenConns: 0 # Grafana v5.4+
+ maxIdleConns: 2 # Grafana v5.4+
+ connMaxLifetime: 14400 # Grafana v5.4+
+ postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10
+ timescaledb: false
+ # allow users to edit datasources from the UI.
+ editable: false
+```
+
+Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file):
+
+```yaml
+datasources:
+ datasources.yaml:
+ apiVersion: 1
+ datasources:
+ # name of the datasource. Required
+ - name: Graphite
+ # datasource type. Required
+ type: graphite
+ # access mode. proxy or direct (Server or Browser in the UI). Required
+ access: proxy
+ # org id. will default to orgId 1 if not specified
+ orgId: 1
+ # url
+ url: http://localhost:8080
+ # database password, if used
+ password:
+ # database user, if used
+ user:
+ # database name, if used
+ database:
+ # enable/disable basic auth
+ basicAuth:
+ # basic auth username
+ basicAuthUser:
+ # basic auth password
+ basicAuthPassword:
+ # enable/disable with credentials headers
+ withCredentials:
+ # mark as default datasource. Max one per org
+ isDefault:
+ #