From fe013a6d24ef21b6812cd2f55f28246f444ef563 Mon Sep 17 00:00:00 2001 From: Michael Welles Date: Mon, 20 Jul 2026 23:25:33 -0400 Subject: [PATCH] feat(dgraph): add optional PodDisruptionBudget, ServiceMonitor, PrometheusRule, and NetworkPolicy All four resources default off. - pdb.yaml: alpha/zero PodDisruptionBudgets, default-off to avoid the single-replica drain deadlock (minAvailable >= replicaCount blocks kubectl drain / cluster-autoscaler eviction indefinitely). - servicemonitor.yaml: companion to commit 2's headless-Service ports; scoped to the ClusterIP Services via the monitor label. - prometheusrule.yaml: conservative default alerts on stable metrics (up, kube_job_status_failed); extraRules for image-specific alerts. - networkpolicy.yaml: ingress-only, intra-cluster allow plus clientPodLabels-gated alpha client access. ServiceMonitor/PrometheusRule require the Prometheus Operator CRDs; default-off means a stock install imposes no CRD dependency. --- charts/dgraph/templates/networkpolicy.yaml | 49 ++++++++++++++++++ charts/dgraph/templates/pdb.yaml | 53 +++++++++++++++++++ charts/dgraph/templates/prometheusrule.yaml | 57 +++++++++++++++++++++ charts/dgraph/templates/servicemonitor.yaml | 44 ++++++++++++++++ charts/dgraph/values.yaml | 47 +++++++++++++++++ 5 files changed, 250 insertions(+) create mode 100644 charts/dgraph/templates/networkpolicy.yaml create mode 100644 charts/dgraph/templates/pdb.yaml create mode 100644 charts/dgraph/templates/prometheusrule.yaml create mode 100644 charts/dgraph/templates/servicemonitor.yaml diff --git a/charts/dgraph/templates/networkpolicy.yaml b/charts/dgraph/templates/networkpolicy.yaml new file mode 100644 index 000000000..d5690fb4d --- /dev/null +++ b/charts/dgraph/templates/networkpolicy.yaml @@ -0,0 +1,49 @@ +{{- /* +Optional NetworkPolicy restricting ingress to the dgraph pods. Default off so +it can't silently break connectivity on clusters whose CNI does not enforce +NetworkPolicy. When enabled it: + - always allows intra-cluster dgraph traffic (alpha<->zero, peer gossip) by + selecting pods that carry this release's own app/release labels; + - grants client access to the alpha HTTP (8080) and gRPC (9080) ports only to + pods carrying every label in networkPolicy.clientPodLabels; + - appends any rules supplied in networkPolicy.extraIngress verbatim. +Egress is intentionally not restricted (backups reach S3/MinIO/NFS, etc.). +*/ -}} +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "dgraph.fullname" . }} + namespace: {{ include "dgraph.namespace" . }} + labels: + {{- include "dgraph.labels" (dict "ctx" .) | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "dgraph.name" . }} + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + # Intra-cluster: any dgraph pod from this release may reach any other. + - from: + - podSelector: + matchLabels: + app: {{ template "dgraph.name" . }} + release: {{ .Release.Name }} + {{- with .Values.networkPolicy.clientPodLabels }} + # Clients: pods carrying these labels may reach the alpha client ports. + - from: + - podSelector: + matchLabels: + {{- toYaml . | nindent 14 }} + ports: + - protocol: TCP + port: 8080 + - protocol: TCP + port: 9080 + {{- end }} + {{- with .Values.networkPolicy.extraIngress }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/dgraph/templates/pdb.yaml b/charts/dgraph/templates/pdb.yaml new file mode 100644 index 000000000..2e47e9bb6 --- /dev/null +++ b/charts/dgraph/templates/pdb.yaml @@ -0,0 +1,53 @@ +{{- /* +PodDisruptionBudgets for the alpha and zero Raft groups. Default off: setting +minAvailable (or maxUnavailable) too tight relative to replicaCount can block +voluntary evictions entirely. In particular, minAvailable >= replicaCount on a +single-replica group makes `kubectl drain` (and cluster-autoscaler scale-down) +hang forever, since Kubernetes will never evict a Pod that would violate the +budget. Review minAvailable/maxUnavailable against each group's replicaCount +before enabling. +Set maxUnavailable instead of minAvailable to express the budget the other way. +*/ -}} +{{- if .Values.zero.pdb.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ template "dgraph.zero.fullname" . }} + namespace: {{ include "dgraph.namespace" . }} + labels: + {{- include "dgraph.labels" (dict "ctx" . "component" .Values.zero.name) | nindent 4 }} +spec: + {{- if .Values.zero.pdb.maxUnavailable }} + maxUnavailable: {{ .Values.zero.pdb.maxUnavailable }} + {{- else }} + minAvailable: {{ .Values.zero.pdb.minAvailable }} + {{- end }} + selector: + matchLabels: + app: {{ template "dgraph.name" . }} + release: {{ .Release.Name }} + component: {{ .Values.zero.name }} +{{- end }} +{{- if .Values.alpha.pdb.enabled }} +{{- if .Values.zero.pdb.enabled }} +--- +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ template "dgraph.alpha.fullname" . }} + namespace: {{ include "dgraph.namespace" . }} + labels: + {{- include "dgraph.labels" (dict "ctx" . "component" .Values.alpha.name) | nindent 4 }} +spec: + {{- if .Values.alpha.pdb.maxUnavailable }} + maxUnavailable: {{ .Values.alpha.pdb.maxUnavailable }} + {{- else }} + minAvailable: {{ .Values.alpha.pdb.minAvailable }} + {{- end }} + selector: + matchLabels: + app: {{ template "dgraph.name" . }} + release: {{ .Release.Name }} + component: {{ .Values.alpha.name }} +{{- end }} diff --git a/charts/dgraph/templates/prometheusrule.yaml b/charts/dgraph/templates/prometheusrule.yaml new file mode 100644 index 000000000..a101608ce --- /dev/null +++ b/charts/dgraph/templates/prometheusrule.yaml @@ -0,0 +1,57 @@ +{{- /* +Prometheus Operator PrometheusRule. Default off. The default alerts below are +deliberately conservative and built only on metrics that are stable regardless +of this chart's internal metric names: + - `up`, emitted by Prometheus for every scrape target (works with the + ServiceMonitor above), keyed to the exact alpha/zero Service names; and + - `kube_job_status_failed` from kube-state-metrics, for backup CronJob failures. +Richer dgraph-specific alerts (raft leader churn, replication lag, predicate +move failures) depend on this image's exact metric names — add them via +prometheusRule.extraRules once you have validated the names against a live +target. Set prometheusRule.defaultRules=false to ship only your extraRules. +*/ -}} +{{- if .Values.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ template "dgraph.fullname" . }} + namespace: {{ include "dgraph.namespace" . }} + labels: + {{- include "dgraph.labels" (dict "ctx" .) | nindent 4 }} + {{- with .Values.prometheusRule.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + - name: {{ template "dgraph.fullname" . }} + rules: + {{- if .Values.prometheusRule.defaultRules }} + - alert: DgraphAlphaInstanceDown + expr: up{service="{{ template "dgraph.alpha.fullname" . }}"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Dgraph alpha instance {{`{{ $labels.instance }}`}} is down" + description: "An alpha pod has been unreachable to Prometheus for 5m. With a 3-node group, losing two alphas drops the group below quorum." + - alert: DgraphZeroInstanceDown + expr: up{service="{{ template "dgraph.zero.fullname" . }}"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Dgraph zero instance {{`{{ $labels.instance }}`}} is down" + description: "A zero pod has been unreachable to Prometheus for 5m. Zero is the cluster's Raft coordinator; losing quorum stalls the cluster." + - alert: DgraphBackupJobFailed + expr: kube_job_status_failed{namespace="{{ include "dgraph.namespace" . }}", job_name=~"{{ template "dgraph.backups.fullname" . }}-.*"} > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Dgraph backup job {{`{{ $labels.job_name }}`}} failed" + description: "A binary-backup CronJob run has failed. Backups may be stale — investigate before relying on point-in-time recovery." + {{- end }} + {{- with .Values.prometheusRule.extraRules }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/dgraph/templates/servicemonitor.yaml b/charts/dgraph/templates/servicemonitor.yaml new file mode 100644 index 000000000..6e291c024 --- /dev/null +++ b/charts/dgraph/templates/servicemonitor.yaml @@ -0,0 +1,44 @@ +{{- /* +Prometheus Operator ServiceMonitor for the alpha and zero metrics endpoints. +Default off. The alpha Service exposes the HTTP port as `http-alpha` (8080) and +the zero Service as `http-zero` (6080); dgraph serves Prometheus metrics on the +HTTP port at serviceMonitor.path. Listing both port names in one ServiceMonitor +is fine — an endpoint whose named port a given Service lacks is simply skipped. + +The selector requires the `monitor` label, which only the ClusterIP Services +carry (set from alpha.monitorLabel / zero.monitorLabel). The headless Services +share the same app/release labels but do not carry `monitor`, so they are +never selected. +*/ -}} +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "dgraph.fullname" . }} + namespace: {{ default (include "dgraph.namespace" .) .Values.serviceMonitor.namespace }} + labels: + {{- include "dgraph.labels" (dict "ctx" .) | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + matchNames: + - {{ include "dgraph.namespace" . }} + selector: + matchLabels: + app: {{ template "dgraph.name" . }} + release: {{ .Release.Name }} + matchExpressions: + - key: monitor + operator: Exists + endpoints: + - port: http-alpha + path: {{ .Values.serviceMonitor.path }} + interval: {{ .Values.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} + - port: http-zero + path: {{ .Values.serviceMonitor.path }} + interval: {{ .Values.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} +{{- end }} diff --git a/charts/dgraph/values.yaml b/charts/dgraph/values.yaml index 1743d5419..f13902669 100644 --- a/charts/dgraph/values.yaml +++ b/charts/dgraph/values.yaml @@ -72,6 +72,15 @@ zero: ## Value for the "monitor" label on the zero Service (not on pods or other resources). ## Used by Prometheus for service discovery. monitorLabel: zero-dgraph-io + + ## PodDisruptionBudget for the zero StatefulSet. Default off — see templates/pdb.yaml + ## for why minAvailable >= replicaCount can deadlock `kubectl drain` on a + ## single-replica group. Review against replicaCount before enabling. + pdb: + enabled: false + minAvailable: 2 + # maxUnavailable: + ## StatefulSet controller supports automated updates. There are two valid update strategies: RollingUpdate and OnDelete ## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets ## @@ -261,6 +270,15 @@ alpha: ## Value for the "monitor" label on the alpha Service (not on pods or other resources). ## Used by Prometheus for service discovery. monitorLabel: alpha-dgraph-io + + ## PodDisruptionBudget for the alpha StatefulSet. Default off — see templates/pdb.yaml + ## for why minAvailable >= replicaCount can deadlock `kubectl drain` on a + ## single-replica group. Review against replicaCount before enabling. + pdb: + enabled: false + minAvailable: 2 + # maxUnavailable: + ## StatefulSet controller supports automated updates. There are two valid update strategies: RollingUpdate and OnDelete ## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets ## @@ -721,6 +739,35 @@ backups: ## AWS_SECRET_ACCESS_KEY env var secret: "" +## Prometheus Operator ServiceMonitor for the alpha and zero metrics endpoints. +## Default off — requires the Prometheus Operator CRDs to be installed; a +## stock install with this disabled imposes no CRD dependency. +serviceMonitor: + enabled: false + # namespace: monitoring + labels: {} + interval: 30s + scrapeTimeout: 10s + path: /debug/prometheus_metrics + +## Prometheus Operator PrometheusRule with conservative default alerts for +## alpha/zero instance availability and backup CronJob failures. +## Default off — requires the Prometheus Operator CRDs to be installed; a +## stock install with this disabled imposes no CRD dependency. +prometheusRule: + enabled: false + labels: {} + defaultRules: true + extraRules: [] + +## Optional NetworkPolicy restricting ingress to the dgraph pods. Default off +## so it can't silently break connectivity on clusters whose CNI does not +## enforce NetworkPolicy. +networkPolicy: + enabled: false + clientPodLabels: {} + extraIngress: [] + global: domain: cluster.local ## Combined ingress resource for alpha and ratel services