WIP get service and pod metrics via our prometheus scraper

canonical-kubernetes/addons/prometheus/steps/01_install-prometheus/prometheus-scrape-k8s.yaml

@@ -1,4 +1,21 @@
-- job_name: 'k8s-api-endpoints'
+# A scrape configuration for running Prometheus on a Kubernetes cluster.
+# This uses separate scrape configs for cluster components (i.e. API server, node)
+# and services to allow each to use different authentication configs.
+#
+# Kubernetes labels will be added as Prometheus labels on metrics via the
+# `labelmap` relabeling action.
+#
+# If you are using Kubernetes 1.7.2 or earlier, please take note of the comments
+# for the kubernetes-cadvisor job; you will need to edit or remove this job.
+
+# Scrape config for API servers.
+#
+# Kubernetes exposes API servers as endpoints to the default/kubernetes
+# service so this uses `endpoints` role and uses relabelling to only keep
+# the endpoints associated with the default/kubernetes service using the
+# default named port `https`. This works for single API server deployments as
+# well as HA API server deployments.
+- job_name: 'kubernetes-apiservers'
   kubernetes_sd_configs:
   - api_server: K8S_API_ENDPOINT
     role: endpoints
@@ -18,6 +35,13 @@
   - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
     action: keep
     regex: default;kubernetes;https
+
+# Scrape config for nodes (kubelet).
+#
+# Rather than connecting directly to the node, the scrape is proxied though the
+# Kubernetes apiserver.  This means it will work if Prometheus is running out of
+# cluster, or can't connect to nodes for some other reason (e.g. because of
+# firewalling).
 - job_name: 'kubernetes-nodes'
   kubernetes_sd_configs:
   - api_server: K8S_API_ENDPOINT
@@ -43,6 +67,21 @@
     regex: (.+)
     target_label: __metrics_path__
     replacement: /api/v1/nodes/$1/proxy/metrics
+
+# Scrape config for Kubelet cAdvisor.
+#
+# This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics
+# (those whose names begin with 'container_') have been removed from the
+# Kubelet metrics endpoint.  This job scrapes the cAdvisor endpoint to
+# retrieve those metrics.
+#
+# In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor
+# HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics"
+# in that case (and ensure cAdvisor's HTTP server hasn't been disabled with
+# the --cadvisor-port=0 Kubelet flag).
+#
+# This job is not necessary and should be removed in Kubernetes 1.6 and
+# earlier versions, or it will cause the metrics to be scraped twice.
 - job_name: 'kubernetes-cadvisor'
   kubernetes_sd_configs:
   - api_server: K8S_API_ENDPOINT
@@ -68,3 +107,104 @@
     regex: (.+)
     target_label: __metrics_path__
     replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
+
+# Example scrape config for service endpoints.
+#
+# The relabeling allows the actual service scrape endpoint to be configured
+# for all or only some endpoints.
+- job_name: 'kubernetes-service-endpoints'
+  kubernetes_sd_configs:
+  - role: endpoints
+    api_server: K8S_API_ENDPOINT
+    tls_config:
+      insecure_skip_verify: true
+    basic_auth:
+      username: admin
+      password: K8S_PASSWORD
+  scheme: https
+  scrape_interval: 30s
+  tls_config:
+    insecure_skip_verify: true
+  basic_auth:
+    username: admin
+    password: K8S_PASSWORD
+  relabel_configs:
+  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+    action: keep
+    regex: true
+  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
+    action: replace
+    regex: (\d+)
+    target_label: __meta_kubernetes_pod_container_port_number
+  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+    action: replace
+    regex: ()
+    target_label: __meta_kubernetes_service_annotation_prometheus_io_path
+    replacement: /metrics
+  - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_pod_container_port_number, __meta_kubernetes_service_annotation_prometheus_io_path]
+    target_label: __metrics_path__
+    regex: (.+);(.+);(.+);(.+)
+    replacement: /api/v1/namespaces/$1/services/$2:$3/proxy$4
+  - target_label: __address__
+    replacement: K8S_API_ENDPOINT:443
+  - action: labelmap
+    regex: __meta_kubernetes_service_label_(.+)
+  - source_labels: [__meta_kubernetes_namespace]
+    action: replace
+    target_label: kubernetes_namespace
+  - source_labels: [__meta_kubernetes_service_name]
+    action: replace
+    target_label: kubernetes_name
+  - source_labels: [__meta_kubernetes_pod_node_name]
+    action: replace
+    target_label: instance
+
+
+
+# Example scrape config for pods
+#
+# The relabeling allows the actual pod scrape to be configured
+# for all the declared ports (or port-free target if none is declared)
+# or only some ports.
+- job_name: 'kubernetes-pods'
+  kubernetes_sd_configs:
+  - role: pod
+    api_server: K8S_API_ENDPOINT
+    tls_config:
+      insecure_skip_verify: true
+    basic_auth:
+      username: admin
+      password: K8S_PASSWORD
+  scheme: https
+  scrape_interval: 30s
+  tls_config:
+    insecure_skip_verify: true
+  basic_auth:
+    username: admin
+    password: K8S_PASSWORD
+  relabel_configs:
+  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+    action: keep
+    regex: true
+  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+    action: replace
+    regex: ()
+    target_label: __meta_kubernetes_pod_annotation_prometheus_io_path
+    replacement: /metrics
+  - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_name, __meta_kubernetes_pod_container_port_number, __meta_kubernetes_pod_annotation_prometheus_io_path]
+    target_label: __metrics_path__
+    regex: (.+);(.+);(.+);(.+)
+    replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4
+  - target_label: __address__
+    replacement: K8S_API_ENDPOINT:443
+  - action: labelmap
+    regex: __meta_kubernetes_pod_label_(.+)
+  - source_labels: [__meta_kubernetes_namespace]
+    action: replace
+    target_label: kubernetes_namespace
+  - source_labels: [__meta_kubernetes_pod_name]
+    action: replace
+    target_label: kubernetes_pod_name
+  - source_labels: [__meta_kubernetes_pod_node_name]
+    action: replace
+    target_label: instance