From f36dd8a2c78a2f2fb73a6774277b8a97bea99415 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 14:27:18 +0100 Subject: [PATCH 01/25] feat: supporting docker compose setup --- docker/docker-compose.yaml | 0 docker/volumes/grafana/.gitignore | 5 +++++ docker/volumes/prometheus/.gitignore | 5 +++++ 3 files changed, 10 insertions(+) create mode 100644 docker/docker-compose.yaml create mode 100644 docker/volumes/grafana/.gitignore create mode 100644 docker/volumes/prometheus/.gitignore diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000..e69de29 diff --git a/docker/volumes/grafana/.gitignore b/docker/volumes/grafana/.gitignore new file mode 100644 index 0000000..e769a26 --- /dev/null +++ b/docker/volumes/grafana/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* + +# But keep this .gitignore file +!.gitignore diff --git a/docker/volumes/prometheus/.gitignore b/docker/volumes/prometheus/.gitignore new file mode 100644 index 0000000..facdf81 --- /dev/null +++ b/docker/volumes/prometheus/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* + +# But keep this .gitignore +!.gitignore From 7eb3b40710151e1879d172f245a9e1931011741e Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 14:35:37 +0100 Subject: [PATCH 02/25] adding multiservice discovery --- docker/docker-compose.yaml | 14 ++++++++++++++ docker/volumes/msd/.gitignore | 5 +++++ 2 files changed, 19 insertions(+) create mode 100644 docker/volumes/msd/.gitignore diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index e69de29..cc4c1e3 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -0,0 +1,14 @@ + +services: + + multiservice-discovery: + image: ghcr.io/dfinity/dre/multiservice-discovery:cac3154c7868faf9acad14f8585d37df27169573 + command: + - --targets-dir + - /msd + - --network-state-file + - /msd/definitions.json + volume: + - ./volumes/msd:/targets + network: host # If the stack is behind a VPN + diff --git a/docker/volumes/msd/.gitignore b/docker/volumes/msd/.gitignore new file mode 100644 index 0000000..e67750e --- /dev/null +++ b/docker/volumes/msd/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* + +# But keep this .gitignore +!.gitignore From 7f58c1fa1be09daebde9145d1c0711ef36adc6e1 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 15:05:37 +0100 Subject: [PATCH 03/25] making a template --- docker/config/prometheus/.gitignore | 1 + docker/config/prometheus/config-template.yaml | 19 +++++++++++++++++++ docker/docker-compose.yaml | 19 +++++++++++++++---- 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 docker/config/prometheus/.gitignore create mode 100644 docker/config/prometheus/config-template.yaml diff --git a/docker/config/prometheus/.gitignore b/docker/config/prometheus/.gitignore new file mode 100644 index 0000000..5b6b072 --- /dev/null +++ b/docker/config/prometheus/.gitignore @@ -0,0 +1 @@ +config.yaml diff --git a/docker/config/prometheus/config-template.yaml b/docker/config/prometheus/config-template.yaml new file mode 100644 index 0000000..0694e35 --- /dev/null +++ b/docker/config/prometheus/config-template.yaml @@ -0,0 +1,19 @@ +global: + # Change this to control how frequently prometheus + # scrapes the targets. Prometheus accepts + # as the value for this field. + # More about duration: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#duration + scrape_interval: 30s + +scrape_configs: + - job_name: node-provider-dc-scrape + + tls_config: + insecure_skip_verify: true + + http_sd_configs: + - url: http://multiservice-discovery:8000/prom/targets + http_headers: + # Change these to tailor your setup + node_provider_id: + dc_id: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index cc4c1e3..add5c29 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -6,9 +6,20 @@ services: command: - --targets-dir - /msd - - --network-state-file + - --networks-state-file - /msd/definitions.json - volume: - - ./volumes/msd:/targets - network: host # If the stack is behind a VPN + volumes: + - ./volumes/msd:/msd + ports: + - 8000:8000 + + prometheus: + image: quay.io/prometheus/prometheus:v2.51.1 + ports: + - 32090:9090 + volumes: + - ./config/prometheus:/config + command: + - --config.file + - /config/config.yaml From e9b384247bce9d67c4aa771c138f950d67f104b3 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 17:43:51 +0100 Subject: [PATCH 04/25] configuring prometheus scraping --- docker/docker-compose.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index add5c29..4033440 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -15,11 +15,10 @@ services: prometheus: image: quay.io/prometheus/prometheus:v2.51.1 - ports: - - 32090:9090 volumes: - ./config/prometheus:/config command: - --config.file - /config/config.yaml - + ports: + - 9090:9090 From 30b5b8fe9e9f586a76716ce42a4e2aa2cb9a01a0 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 17:48:39 +0100 Subject: [PATCH 05/25] extending template --- docker/config/prometheus/config-template.yaml | 126 +++++++++++++++++- 1 file changed, 120 insertions(+), 6 deletions(-) diff --git a/docker/config/prometheus/config-template.yaml b/docker/config/prometheus/config-template.yaml index 0694e35..86bca9f 100644 --- a/docker/config/prometheus/config-template.yaml +++ b/docker/config/prometheus/config-template.yaml @@ -6,14 +6,128 @@ global: scrape_interval: 30s scrape_configs: - - job_name: node-provider-dc-scrape + - job_name: replica + metrics_path: "" tls_config: insecure_skip_verify: true http_sd_configs: - - url: http://multiservice-discovery:8000/prom/targets - http_headers: - # Change these to tailor your setup - node_provider_id: - dc_id: + - url: http://multiservice-discovery:8000/prom/targets?node_provider_id=&dc_id= + refresh_interval: 15s + + relabel_configs: + - # Strip the scheme from the multiservice-discovery-generated address. + source_labels: + - __address__ + target_label: __address__ + regex: (.*)://(.*) + replacement: ${2} + action: replace + - # Strip the metrics path from the multiservice-discovery-generated address. + source_labels: + - __address__ + target_label: __address__ + regex: (.*?)/(.*) + replacement: ${1} + action: replace + - # Prevent all scrapes that do not match this job. + source_labels: ["job"] + target_label: __dropped + regex: "replica" + action: keep + + - job_name: orchestrator + metrics_path: "" + + tls_config: + insecure_skip_verify: true + + http_sd_configs: + - url: http://multiservice-discovery:8000/prom/targets?node_provider_id=&dc_id= + refresh_interval: 15s + + relabel_configs: + - # Strip the scheme from the multiservice-discovery-generated address. + source_labels: + - __address__ + target_label: __address__ + regex: (.*)://(.*) + replacement: ${2} + action: replace + - # Strip the metrics path from the multiservice-discovery-generated address. + source_labels: + - __address__ + target_label: __address__ + regex: (.*?)/(.*) + replacement: ${1} + action: replace + - # Prevent all scrapes that do not match this job. + source_labels: ["job"] + target_label: __dropped + regex: "orchestrator" + action: keep + + - job_name: node_exporter + metrics_path: /metrics + scheme: https + + tls_config: + insecure_skip_verify: true + + http_sd_configs: + - url: http://multiservice-discovery:8000/prom/targets?node_provider_id=&dc_id= + refresh_interval: 15s + + relabel_configs: + - # Strip the scheme from the multiservice-discovery-generated address. + source_labels: + - __address__ + target_label: __address__ + regex: (.*)://(.*) + replacement: ${2} + action: replace + - # Strip the metrics path from the multiservice-discovery-generated address. + source_labels: + - __address__ + target_label: __address__ + regex: (.*?)/(.*) + replacement: ${1} + action: replace + - # Prevent all scrapes that do not match this job. + source_labels: ["job"] + target_label: __dropped + regex: "node_exporter" + action: keep + + - job_name: host_node_exporter + metrics_path: /metrics + scheme: https + + tls_config: + insecure_skip_verify: true + + http_sd_configs: + - url: http://multiservice-discovery:8000/prom/targets?node_provider_id=&dc_id= + refresh_interval: 15s + + relabel_configs: + - # Strip the scheme from the multiservice-discovery-generated address. + source_labels: + - __address__ + target_label: __address__ + regex: (.*)://(.*) + replacement: ${2} + action: replace + - # Strip the metrics path from the multiservice-discovery-generated address. + source_labels: + - __address__ + target_label: __address__ + regex: (.*?)/(.*) + replacement: ${1} + action: replace + - # Prevent all scrapes that do not match this job. + source_labels: ["job"] + target_label: __dropped + regex: "host_node_exporter" + action: keep From 812a43779ce1c4530dbc07e1557758055c4cd874 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 19:06:45 +0100 Subject: [PATCH 06/25] script for expanding prometheus config --- docker/config/prometheus/config-template.yaml | 136 ++++++------------ .../prometheus/prom-config-builder.Dockerfile | 5 + .../config/prometheus/prom_config_builder.py | 77 ++++++++++ docker/docker-compose.tools.yaml | 9 ++ docker/docker-compose.yaml | 3 + docker/volumes/prometheus/.gitignore | 0 6 files changed, 135 insertions(+), 95 deletions(-) create mode 100644 docker/config/prometheus/prom-config-builder.Dockerfile create mode 100644 docker/config/prometheus/prom_config_builder.py create mode 100644 docker/docker-compose.tools.yaml mode change 100644 => 100755 docker/volumes/prometheus/.gitignore diff --git a/docker/config/prometheus/config-template.yaml b/docker/config/prometheus/config-template.yaml index 86bca9f..25da4d1 100644 --- a/docker/config/prometheus/config-template.yaml +++ b/docker/config/prometheus/config-template.yaml @@ -1,30 +1,18 @@ -global: - # Change this to control how frequently prometheus - # scrapes the targets. Prometheus accepts - # as the value for this field. - # More about duration: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#duration - scrape_interval: 30s - -scrape_configs: - - job_name: replica - metrics_path: "" - +anchors: + tls: &tls tls_config: insecure_skip_verify: true - http_sd_configs: - - url: http://multiservice-discovery:8000/prom/targets?node_provider_id=&dc_id= - refresh_interval: 15s - + relabelings: &relabelings relabel_configs: - - # Strip the scheme from the multiservice-discovery-generated address. + - # Strip the scheme from the discovery generated address. source_labels: - __address__ target_label: __address__ regex: (.*)://(.*) replacement: ${2} action: replace - - # Strip the metrics path from the multiservice-discovery-generated address. + - # Strip the metrics path from the discovery generated address. source_labels: - __address__ target_label: __address__ @@ -34,100 +22,58 @@ scrape_configs: - # Prevent all scrapes that do not match this job. source_labels: ["job"] target_label: __dropped - regex: "replica" - action: keep + regex: "" + action: keep - - job_name: orchestrator + http_configs: &http_configs + http_sd_configs: + - url: http://multiservice-discovery:8000/prom/targets + refresh_interval: 15s + +global: + # Change this to control how frequently prometheus + # scrapes the targets. Prometheus accepts + # as the value for this field. + # More about duration: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#duration + scrape_interval: 30s + +scrape_configs: + - job_name: replica metrics_path: "" - tls_config: - insecure_skip_verify: true + <<: *tls - http_sd_configs: - - url: http://multiservice-discovery:8000/prom/targets?node_provider_id=&dc_id= - refresh_interval: 15s + <<: *http_configs - relabel_configs: - - # Strip the scheme from the multiservice-discovery-generated address. - source_labels: - - __address__ - target_label: __address__ - regex: (.*)://(.*) - replacement: ${2} - action: replace - - # Strip the metrics path from the multiservice-discovery-generated address. - source_labels: - - __address__ - target_label: __address__ - regex: (.*?)/(.*) - replacement: ${1} - action: replace - - # Prevent all scrapes that do not match this job. - source_labels: ["job"] - target_label: __dropped - regex: "orchestrator" - action: keep + <<: *relabelings + + - job_name: orchestrator + metrics_path: "" + + <<: *tls + + <<: *http_configs + + <<: *relabelings - job_name: node_exporter metrics_path: /metrics scheme: https + + <<: *tls - tls_config: - insecure_skip_verify: true + <<: *http_configs - http_sd_configs: - - url: http://multiservice-discovery:8000/prom/targets?node_provider_id=&dc_id= - refresh_interval: 15s + <<: *relabelings - relabel_configs: - - # Strip the scheme from the multiservice-discovery-generated address. - source_labels: - - __address__ - target_label: __address__ - regex: (.*)://(.*) - replacement: ${2} - action: replace - - # Strip the metrics path from the multiservice-discovery-generated address. - source_labels: - - __address__ - target_label: __address__ - regex: (.*?)/(.*) - replacement: ${1} - action: replace - - # Prevent all scrapes that do not match this job. - source_labels: ["job"] - target_label: __dropped - regex: "node_exporter" - action: keep - job_name: host_node_exporter metrics_path: /metrics scheme: https + + <<: *tls - tls_config: - insecure_skip_verify: true + <<: *http_configs - http_sd_configs: - - url: http://multiservice-discovery:8000/prom/targets?node_provider_id=&dc_id= - refresh_interval: 15s + <<: *relabelings - relabel_configs: - - # Strip the scheme from the multiservice-discovery-generated address. - source_labels: - - __address__ - target_label: __address__ - regex: (.*)://(.*) - replacement: ${2} - action: replace - - # Strip the metrics path from the multiservice-discovery-generated address. - source_labels: - - __address__ - target_label: __address__ - regex: (.*?)/(.*) - replacement: ${1} - action: replace - - # Prevent all scrapes that do not match this job. - source_labels: ["job"] - target_label: __dropped - regex: "host_node_exporter" - action: keep diff --git a/docker/config/prometheus/prom-config-builder.Dockerfile b/docker/config/prometheus/prom-config-builder.Dockerfile new file mode 100644 index 0000000..5be884d --- /dev/null +++ b/docker/config/prometheus/prom-config-builder.Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim + +RUN pip install --no-cache-dir pyyaml + +ENTRYPOINT ["python3"] diff --git a/docker/config/prometheus/prom_config_builder.py b/docker/config/prometheus/prom_config_builder.py new file mode 100644 index 0000000..3f9808b --- /dev/null +++ b/docker/config/prometheus/prom_config_builder.py @@ -0,0 +1,77 @@ +import argparse + +import yaml + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + "prom-config-builder", + description="Used to expand the configuration to actual prometheus config", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--template-path", + dest="template_path", + default="/config/prometheus/config-template.yaml", + help="Path to the config template that should be expanded", + ) + + parser.add_argument( + "--output-path", + dest="output_path", + default="/config/prometheus/config.yaml", + help="Path to where the output should be written", + ) + + parser.add_argument( + "--node-provider-id", + dest="node_provider_id", + help="The principal id for the node provider", + ) + + parser.add_argument( + "--dc-id", + dest="dc_id", + default=None, + help="The data center id represented with two letters and a number", + ) + + return parser.parse_args() + + +def expand_anchors_text(yaml_text: str) -> str: + data = yaml.safe_load(yaml_text) + + class NoAliasDumper(yaml.SafeDumper): + # always return True to *prevent* YAML from emitting aliases + def ignore_aliases(self, data): + return True + + return yaml.dump( + data, Dumper=NoAliasDumper, default_flow_style=False, sort_keys=False + ) + + +if __name__ == "__main__": + args = parse_args() + + with open(args.template_path, "r") as f: + text = f.read() + template = dict(yaml.safe_load(expand_anchors_text(text))) + + for scrape_config in template["scrape_configs"]: + job_name = scrape_config["job_name"] + + # Restrict job + scrape_config["relabel_configs"][2]["regex"] = job_name + + url = scrape_config["http_sd_configs"][0]["url"] + url = f"{url}?node_provider_id={args.node_provider_id}{'&dc_id={dc_id}'.format(dc_id=args.dc_id) if args.dc_id is not None else ''}" + scrape_config["http_sd_configs"][0]["url"] = url + + # Drop anchors + del template["anchors"] + + with open(args.output_path, "w") as f: + f.write(yaml.safe_dump(template)) diff --git a/docker/docker-compose.tools.yaml b/docker/docker-compose.tools.yaml new file mode 100644 index 0000000..4b6f913 --- /dev/null +++ b/docker/docker-compose.tools.yaml @@ -0,0 +1,9 @@ +services: + prom-config-builder: + build: + context: ./config/prometheus/ + dockerfile: ./config/prometheus/prom-config-builder.Dockerfile + volumes: + - ./config/prometheus/:/config/prometheus + working_dir: / + diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 4033440..e2bdc77 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -17,8 +17,11 @@ services: image: quay.io/prometheus/prometheus:v2.51.1 volumes: - ./config/prometheus:/config + - ./volumes/prometheus/:/prometheus command: - --config.file - /config/config.yaml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=6m ports: - 9090:9090 diff --git a/docker/volumes/prometheus/.gitignore b/docker/volumes/prometheus/.gitignore old mode 100644 new mode 100755 From bf2ef0f237c4b6eda4ddb5d1b32cdcc30145728d Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 22:33:04 +0100 Subject: [PATCH 07/25] Adding readme for docker tools --- docker/README.md | 4 ++++ docker/docker-compose.tools.yaml | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 docker/README.md diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..228491b --- /dev/null +++ b/docker/README.md @@ -0,0 +1,4 @@ +## Command for configuring prometheus: +```bash +docker compose -f ./docker-compose.tools.yaml run --rm prom-config-builder config/prometheus/prom_config_builder.py --node-provider-id [--dc-id ] +``` diff --git a/docker/docker-compose.tools.yaml b/docker/docker-compose.tools.yaml index 4b6f913..11a881d 100644 --- a/docker/docker-compose.tools.yaml +++ b/docker/docker-compose.tools.yaml @@ -2,8 +2,7 @@ services: prom-config-builder: build: context: ./config/prometheus/ - dockerfile: ./config/prometheus/prom-config-builder.Dockerfile + dockerfile: prom-config-builder.Dockerfile volumes: - - ./config/prometheus/:/config/prometheus - working_dir: / + - ./config/prometheus:/config/prometheus From aedae54c5cd50a216bde7dbbec91de2c5ec4ed38 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 22:49:24 +0100 Subject: [PATCH 08/25] Adding a way to override sd url --- docker/.dockerignore | 1 + docker/README.md | 4 +++- docker/docker-compose.tools.yaml | 5 +++-- .../prom-config-builder.Dockerfile | 0 .../prom_config_builder.py | 20 +++++++++++++++++-- 5 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 docker/.dockerignore rename docker/{config/prometheus => tools/prom-config-builder}/prom-config-builder.Dockerfile (100%) rename docker/{config/prometheus => tools/prom-config-builder}/prom_config_builder.py (75%) diff --git a/docker/.dockerignore b/docker/.dockerignore new file mode 100644 index 0000000..6995804 --- /dev/null +++ b/docker/.dockerignore @@ -0,0 +1 @@ +volumes diff --git a/docker/README.md b/docker/README.md index 228491b..a58373a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,4 +1,6 @@ ## Command for configuring prometheus: ```bash -docker compose -f ./docker-compose.tools.yaml run --rm prom-config-builder config/prometheus/prom_config_builder.py --node-provider-id [--dc-id ] +docker compose -f ./docker-compose.tools.yaml run --rm prom-config-builder \ + tools/prom-config-builder/prom_config_builder.py \ + --node-provider-id bvcsg-3od6r-jnydw-eysln-aql7w-td5zn-ay5m6-sibd2-jzojt-anwag-mqe ``` diff --git a/docker/docker-compose.tools.yaml b/docker/docker-compose.tools.yaml index 11a881d..1a48db6 100644 --- a/docker/docker-compose.tools.yaml +++ b/docker/docker-compose.tools.yaml @@ -1,8 +1,9 @@ services: prom-config-builder: build: - context: ./config/prometheus/ - dockerfile: prom-config-builder.Dockerfile + context: . + dockerfile: ./tools/prom-config-builder/prom-config-builder.Dockerfile volumes: - ./config/prometheus:/config/prometheus + - ./tools/prom-config-builder/:/tools/prom-config-builder diff --git a/docker/config/prometheus/prom-config-builder.Dockerfile b/docker/tools/prom-config-builder/prom-config-builder.Dockerfile similarity index 100% rename from docker/config/prometheus/prom-config-builder.Dockerfile rename to docker/tools/prom-config-builder/prom-config-builder.Dockerfile diff --git a/docker/config/prometheus/prom_config_builder.py b/docker/tools/prom-config-builder/prom_config_builder.py similarity index 75% rename from docker/config/prometheus/prom_config_builder.py rename to docker/tools/prom-config-builder/prom_config_builder.py index 3f9808b..3e190ff 100644 --- a/docker/config/prometheus/prom_config_builder.py +++ b/docker/tools/prom-config-builder/prom_config_builder.py @@ -1,4 +1,5 @@ import argparse +import urllib.parse import yaml @@ -37,6 +38,13 @@ def parse_args() -> argparse.Namespace: help="The data center id represented with two letters and a number", ) + parser.add_argument( + "--sd-url", + dest="sd_url", + default="multiservice-discovery", + help="Override the host for multiservice-discovery container. If running in network_mode: host it will be localhost", + ) + return parser.parse_args() @@ -67,8 +75,16 @@ def ignore_aliases(self, data): scrape_config["relabel_configs"][2]["regex"] = job_name url = scrape_config["http_sd_configs"][0]["url"] - url = f"{url}?node_provider_id={args.node_provider_id}{'&dc_id={dc_id}'.format(dc_id=args.dc_id) if args.dc_id is not None else ''}" - scrape_config["http_sd_configs"][0]["url"] = url + params = {"node_provider_id": args.node_provider_id} + if args.dc_id is not None: + params["dc_id"] = args.dc_id + + url_parts = urllib.parse.urlparse(url) + encoded_query = urllib.parse.urlencode(params) + + scrape_config["http_sd_configs"][0]["url"] = urllib.parse.urlunparse( + url_parts._replace(query=encoded_query, netloc=args.sd_url) + ) # Drop anchors del template["anchors"] From 63ec25c2ef6ed55d1d9289aa0e2b06c37e87912f Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 23:16:28 +0100 Subject: [PATCH 09/25] adding grafana --- docker/config/grafana/grafana.ini | 13 +++++++++++++ docker/docker-compose.yaml | 8 ++++++++ docker/volumes/grafana/.gitignore | 0 3 files changed, 21 insertions(+) create mode 100644 docker/config/grafana/grafana.ini mode change 100644 => 100755 docker/volumes/grafana/.gitignore diff --git a/docker/config/grafana/grafana.ini b/docker/config/grafana/grafana.ini new file mode 100644 index 0000000..64a4103 --- /dev/null +++ b/docker/config/grafana/grafana.ini @@ -0,0 +1,13 @@ +[server] +enable_gzip = true + +[analytics] +reporting_enabled = false + +[security] +admin_user = root +admin_password = password +admin_email = admin@localhost + +secret_key = strongPassword@!!123 + diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index e2bdc77..2d5e2bd 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -25,3 +25,11 @@ services: - --storage.tsdb.retention.time=6m ports: - 9090:9090 + + grafana: + image: grafana/grafana:12.2.1 + volumes: + - ./config/grafana:/etc/grafana + - ./volumes/grafana:/var/lib/grafana + ports: + - 3000:3000 diff --git a/docker/volumes/grafana/.gitignore b/docker/volumes/grafana/.gitignore old mode 100644 new mode 100755 From 6531b7648b04c0c7ab62f884e8ec18be6a46ba6e Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 23:26:16 +0100 Subject: [PATCH 10/25] Adding prometheus datasource --- .../grafana/provisioning/datasources/observability.yaml | 9 +++++++++ docker/docker-compose.yaml | 2 ++ 2 files changed, 11 insertions(+) create mode 100644 docker/config/grafana/provisioning/datasources/observability.yaml diff --git a/docker/config/grafana/provisioning/datasources/observability.yaml b/docker/config/grafana/provisioning/datasources/observability.yaml new file mode 100644 index 0000000..b13a459 --- /dev/null +++ b/docker/config/grafana/provisioning/datasources/observability.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://$PROM_HOST:9090 + jsonData: + httpMethod: POST diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 2d5e2bd..4efd837 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -28,6 +28,8 @@ services: grafana: image: grafana/grafana:12.2.1 + environment: + PROM_HOST: prometheus volumes: - ./config/grafana:/etc/grafana - ./volumes/grafana:/var/lib/grafana From d44b3bbe2ae3aec9877b8c6e0e3281b0de57fec4 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 6 Nov 2025 23:53:45 +0100 Subject: [PATCH 11/25] Adding sample node exporter dashboards --- .../dashboards/observability.yaml | 13 + .../dashboards/samples/node-exporter.json | 13013 ++++++++++++++++ .../dashboards/samples/power-usage.json | 324 + .../datasources/observability.yaml | 2 +- docker/docker-compose.yaml | 6 + 5 files changed, 13357 insertions(+), 1 deletion(-) create mode 100644 docker/config/grafana/provisioning/dashboards/observability.yaml create mode 100644 docker/config/grafana/provisioning/dashboards/samples/node-exporter.json create mode 100644 docker/config/grafana/provisioning/dashboards/samples/power-usage.json diff --git a/docker/config/grafana/provisioning/dashboards/observability.yaml b/docker/config/grafana/provisioning/dashboards/observability.yaml new file mode 100644 index 0000000..6648df9 --- /dev/null +++ b/docker/config/grafana/provisioning/dashboards/observability.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: 'Observability Samples' + folder: 'samples' + folderUid: 'samples' + type: file + disableDeletion: true + updateIntervalSeconds: 60 + allowUiUpdates: false + options: + path: /etc/grafana/provisioning/dashboards/samples + foldersFromFilesStructure: false diff --git a/docker/config/grafana/provisioning/dashboards/samples/node-exporter.json b/docker/config/grafana/provisioning/dashboards/samples/node-exporter.json new file mode 100644 index 0000000..15e4a83 --- /dev/null +++ b/docker/config/grafana/provisioning/dashboards/samples/node-exporter.json @@ -0,0 +1,13013 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:1058", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Machine stats", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 1860, + "graphTooltip": 0, + "id": 2, + "links": [ + { + "icon": "external link", + "tags": [], + "title": "GitHub", + "type": "link", + "url": "https://github.com/rfrail3/grafana-dashboards" + }, + { + "icon": "external link", + "tags": [], + "title": "Grafana", + "type": "link", + "url": "https://grafana.com/grafana/dashboards/1860" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 263, + "panels": [], + "title": "Basic CPU / Mem / Net / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Basic CPU info", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s/s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#1F78C1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "guest" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "irq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "nice" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "softirq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "steal" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "system" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "user" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 77, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum (rate(node_cpu_seconds_total{mode=\"system\",instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} sys", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum (rate(node_cpu_seconds_total{mode='user',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} user", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum (rate(node_cpu_seconds_total{mode='iowait',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} iowait", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum (rate(node_cpu_seconds_total{mode=~\".*irq\",instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} irq", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum (rate(node_cpu_seconds_total{mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} other", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum by (mode)(rate(node_cpu_seconds_total{mode='idle',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} idle", + "range": true, + "refId": "F", + "step": 240 + } + ], + "title": "CPU aggregate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Basic network info per interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 74, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum without (instance) (rate(node_network_receive_bytes_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])*8)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "recv {{device}}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum without (instance) (rate(node_network_transmit_bytes_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])*8)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "trans {{device}} ", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network aggregate", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 265, + "panels": [], + "title": "CPU / Memory / Net / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "percentage", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s/s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "guest" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "irq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "nice" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "softirq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "steal" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "system" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "user" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 3, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum by (mode, instance)(rate(node_cpu_seconds_total{mode=\"system\",instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} System - Processes executing in kernel mode", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum by (mode, instance)(rate(node_cpu_seconds_total{mode='user',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} User - Normal processes executing in user mode", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum by (mode, instance)(rate(node_cpu_seconds_total{mode='nice',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Nice - Niced processes executing in user mode", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum by (mode, instance)(rate(node_cpu_seconds_total{mode='idle',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Idle - Waiting for something to happen", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum by (mode, instance)(rate(node_cpu_seconds_total{mode='iowait',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Iowait - Waiting for I/O to complete", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum by (mode, instance)(rate(node_cpu_seconds_total{mode='irq',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Irq - Servicing interrupts", + "range": true, + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum by (mode, instance)(rate(node_cpu_seconds_total{mode='softirq',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Softirq - Servicing softirqs", + "range": true, + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum by (mode, instance)(rate(node_cpu_seconds_total{mode='steal',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Steal - Time spent in other operating systems when running in a virtualized environment", + "range": true, + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "1813a8c2-ee86-4cb9-b1a7-8b54a85c97d6" + }, + "editorMode": "code", + "expr": "sum by (mode, instance)(rate(node_cpu_seconds_total{mode='guest',instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Guest - Time spent running a virtual CPU for a guest operating system", + "range": true, + "refId": "I", + "step": 240 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap - Swap memory usage": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839", + "Unused - Free memory unassigned": "#052B51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Hardware Corrupted - *./", + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"} - node_memory_MemFree_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"} - node_memory_Buffers_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"} - node_memory_Cached_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"} - node_memory_Slab_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"} - node_memory_PageTables_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"} - node_memory_SwapCached_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Apps - Memory used by user-space applications", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_PageTables_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} PageTables - Memory used to map between virtual and physical memory addresses", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_SwapCached_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "refId": "C", + "step": 240 + }, + { + "expr": "node_memory_Slab_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "refId": "D", + "step": 240 + }, + { + "expr": "node_memory_Cached_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Cache - Parked file data (file content) cache", + "refId": "E", + "step": 240 + }, + { + "expr": "node_memory_Buffers_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Buffers - Block device (e.g. harddisk) cache", + "refId": "F", + "step": 240 + }, + { + "expr": "node_memory_MemFree_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Unused - Free memory unassigned", + "refId": "G", + "step": 240 + }, + { + "expr": "(node_memory_SwapTotal_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"} - node_memory_SwapFree_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Swap - Swap space used", + "refId": "H", + "step": 240 + }, + { + "expr": "node_memory_HardwareCorrupted_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "refId": "I", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Stack", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "receive_packets_eth0": "#7EB26D", + "receive_packets_lo": "#E24D42", + "transmit_packets_eth0": "#7EB26D", + "transmit_packets_lo": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 84, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:5871", + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_network_transmit_bytes_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:5884", + "format": "bps", + "label": "bits out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:5885", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 3, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 21 + }, + "height": "", + "hiddenSeries": false, + "id": 156, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_size_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk Space Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 229, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_reads_completed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{ instance }} {{device}} - Reads completed", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_disk_writes_completed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk IOps", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "IO read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "io time": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 3, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*read*./", + "transform": "negative-Y" + }, + { + "alias": "/.*sda.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde.*/", + "color": "#E24D42" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{instance=~\"[[instance]]\",job=\"host_node_exporter\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Successfully read bytes", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_disk_written_bytes_total{instance=~\"[[instance]]\",job=\"host_node_exporter\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Successfully written bytes", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "I/O Usage Read / Write", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": false, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:965", + "format": "Bps", + "label": "bytes read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:966", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "io time": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 3, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 127, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_io_time_seconds_total{instance=~\"[[instance]]\",job=\"host_node_exporter\",device=~\"$diskdevices\"} [$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "I/O Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": false, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1041", + "format": "percentunit", + "label": "%util", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1042", + "format": "s", + "label": "", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 266, + "panels": [ + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 136, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Inactive_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_Active_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Active / Inactive", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 70 + }, + "hiddenSeries": false, + "id": 135, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Committed_AS - *./" + }, + { + "alias": "/.*CommitLimit - *./", + "color": "#BF1B00", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Committed_AS_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Committed_AS - Amount of memory presently allocated on the system", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_CommitLimit_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} CommitLimit - Amount of memory currently available to be allocated on the system", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Commited", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 80 + }, + "hiddenSeries": false, + "id": 191, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Inactive_file_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Inactive_file - File-backed memory on inactive LRU list", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_Inactive_anon_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_Active_file_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Active_file - File-backed memory on active LRU list", + "refId": "C", + "step": 240 + }, + { + "expr": "node_memory_Active_anon_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", + "refId": "D", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Active / Inactive Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "bytes", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 80 + }, + "hiddenSeries": false, + "id": 130, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Writeback_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Writeback - Memory which is actively being written back to disk", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_WritebackTmp_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} WritebackTmp - Memory used by FUSE for temporary writeback buffers", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_Dirty_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Dirty - Memory which is waiting to get written back to the disk", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Writeback and Dirty", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 90 + }, + "hiddenSeries": false, + "id": 138, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:4131", + "alias": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "fill": 0 + }, + { + "$$hashKey": "object:4138", + "alias": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Mapped_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Mapped - Used memory in mapped pages files which have been mmaped, such as libraries", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_Shmem_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Shmem - Used shared memory (shared between several processes, thus including RAM disks)", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_ShmemHugePages_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "refId": "C", + "step": 240 + }, + { + "expr": "node_memory_ShmemPmdMapped_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} ShmemPmdMapped - Ammount of shared (shmem/tmpfs) memory backed by huge pages", + "refId": "D", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Shared and Mapped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:4106", + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:4107", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 90 + }, + "hiddenSeries": false, + "id": 131, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_SUnreclaim_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_SReclaimable_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} SReclaimable - Part of Slab, that might be reclaimed, such as caches", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Slab", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 100 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_VmallocChunk_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} VmallocChunk - Largest contigious block of vmalloc area which is free", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_VmallocTotal_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} VmallocTotal - Total size of vmalloc memory area", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_VmallocUsed_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} VmallocUsed - Amount of vmalloc area which is used", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Vmalloc", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 100 + }, + "hiddenSeries": false, + "id": 159, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Bounce_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Bounce - Memory used for block device bounce buffers", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Bounce", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 129, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Inactive *./", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_AnonHugePages_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} AnonHugePages - Memory in anonymous huge pages", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_AnonPages_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} AnonPages - Memory in user pages not backed by files", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Anonymous", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 110 + }, + "hiddenSeries": false, + "id": 160, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_KernelStack_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} KernelStack - Kernel memory stack. This is not reclaimable", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_Percpu_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} PerCPU - Per CPU memory allocated dynamically by loadable modules", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Kernel / CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#806EB7", + "Total RAM + Swap": "#806EB7", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 120 + }, + "hiddenSeries": false, + "id": 140, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_HugePages_Free{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} HugePages_Free - Huge pages in the pool that are not yet allocated", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_HugePages_Rsvd{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_HugePages_Surp{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory HugePages Counter", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "pages", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#806EB7", + "Total RAM + Swap": "#806EB7", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 120 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_HugePages_Total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} HugePages - Total size of the pool of huge pages", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_Hugepagesize_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Hugepagesize - Huge Page size", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory HugePages Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 130 + }, + "hiddenSeries": false, + "id": 128, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_DirectMap1G_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} DirectMap1G - Amount of pages mapped as this size", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_DirectMap2M_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} DirectMap2M - Amount of pages mapped as this size", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_DirectMap4k_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} DirectMap4K - Amount of pages mapped as this size", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory DirectMap", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 130 + }, + "hiddenSeries": false, + "id": 137, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Unevictable_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_Mlocked_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} MLocked - Size of pages locked to memory using the mlock() system call", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Unevictable and MLocked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 140 + }, + "hiddenSeries": false, + "id": 132, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_NFS_Unstable_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} NFS Unstable - Memory in NFS pages sent to the server, but not yet commited to the storage", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory NFS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Memory Meminfo", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 267, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 176, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*out/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_vmstat_pgpgin{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Pagesin - Page in operations", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_vmstat_pgpgout{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Pagesout - Page out operations", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Pages In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "pages out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*out/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_vmstat_pswpin{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Pswpin - Pages swapped in", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_vmstat_pswpout{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Pswpout - Pages swapped out", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Pages Swap In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "pages out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 175, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:6118", + "alias": "Pgfault - Page major and minor fault operations", + "fill": 0, + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_vmstat_pgfault{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Pgfault - Page major and minor fault operations", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_vmstat_pgmajfault{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Pgmajfault - Major page fault operations", + "refId": "B", + "step": 240 + }, + { + "expr": "rate(node_vmstat_pgfault{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]) - rate(node_vmstat_pgmajfault{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "P{{ instance }} gminfault - Minor page fault operations", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Page Faults", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:6133", + "format": "short", + "label": "faults", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:6134", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 2, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 307, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_vmstat_oom_kill{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} oom killer invocations ", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "OOM Killer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:5373", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:5374", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Memory Vmstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 293, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 260, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Variation*./", + "color": "#890F02" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_timex_estimated_error_seconds{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Estimated error in seconds", + "refId": "A", + "step": 240 + }, + { + "expr": "node_timex_offset_seconds{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Time offset in between local system and reference clock", + "refId": "B", + "step": 240 + }, + { + "expr": "node_timex_maxerror_seconds{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Maximum error in seconds", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time Syncronized Drift", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "seconds", + "logBase": 1, + "show": true + }, + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 291, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_timex_loop_time_constant{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Phase-locked loop time adjust", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time PLL Adjust", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 34 + }, + "hiddenSeries": false, + "id": 168, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Variation*./", + "color": "#890F02" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_timex_sync_status{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Is clock synchronized to a reliable server (1 = yes, 0 = no)", + "refId": "A", + "step": 240 + }, + { + "expr": "node_timex_frequency_adjustment_ratio{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Local clock frequency adjustment", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time Syncronized Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 34 + }, + "hiddenSeries": false, + "id": 294, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_timex_tick_seconds{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Seconds between clock ticks", + "refId": "A", + "step": 240 + }, + { + "expr": "node_timex_tai_offset_seconds{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} International Atomic Time (TAI) offset", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time Misc", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "seconds", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "System Timesync", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 312, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 62, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_procs_blocked{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Processes blocked waiting for I/O to complete", + "refId": "A", + "step": 240 + }, + { + "expr": "node_procs_running{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Processes in runnable state", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Processes Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:6500", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:6501", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 315, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_processes_state{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ state }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Processes State", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:6500", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:6501", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 148, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_forks_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Processes forks second", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Processes Forks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:6640", + "format": "short", + "label": "forks / sec", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:6641", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 149, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Max.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_virtual_memory_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Processes virtual memory size in bytes", + "refId": "A", + "step": 240 + }, + { + "expr": "process_resident_memory_max_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Maximum amount of virtual memory available in bytes", + "refId": "B", + "step": 240 + }, + { + "expr": "rate(process_virtual_memory_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Processes virtual memory size in bytes", + "refId": "C", + "step": 240 + }, + { + "expr": "rate(process_virtual_memory_max_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Maximum amount of virtual memory available in bytes", + "refId": "D", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Processes Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 313, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:709", + "alias": "PIDs limit", + "color": "#F2495C", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_processes_pids{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Number of PIDs", + "refId": "A", + "step": 240 + }, + { + "expr": "node_processes_max_processes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} PIDs limit", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "PIDs Number and Limit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:6500", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:6501", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 27 + }, + "hiddenSeries": false, + "id": 305, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:4963", + "alias": "/.*waiting.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_schedstat_running_seconds_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} CPU {{ cpu }} - seconds spent running a process", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_schedstat_waiting_seconds_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} CPU {{ cpu }} - seconds spent by processing waiting for this CPU", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Process schedule stats Running / Waiting", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:4860", + "format": "s", + "label": "seconds", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:4861", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 37 + }, + "hiddenSeries": false, + "id": 314, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:709", + "alias": "Threads limit", + "color": "#F2495C", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_processes_threads{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Allocated threads", + "refId": "A", + "step": 240 + }, + { + "expr": "node_processes_max_threads{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Threads limit", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Threads Number and Limit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:6500", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:6501", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "System Processes", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 269, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_context_switches_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Context switches", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_intr_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} Interrupts", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Context Switches / Interrupts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{ instance }} Load 1m", + "refId": "A", + "step": 240 + }, + { + "expr": "node_load5{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{ instance }} Load 5m", + "refId": "B", + "step": 240 + }, + { + "expr": "node_load15{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{ instance }} Load 15m", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "System Load", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:6261", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:6262", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 259, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Critical*./", + "color": "#E24D42", + "fill": 0 + }, + { + "alias": "/.*Max*./", + "color": "#EF843C", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_interrupts_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ type }} - {{ info }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Interrupts Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 306, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_schedstat_timeslices_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} CPU {{ cpu }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Schedule timeslices executed by each cpu", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:4860", + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:4861", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 151, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_entropy_available_bits{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Entropy available to random number generators", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Entropy", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:6568", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:6569", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 308, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Time spent", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU time spent in user and system contexts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:4860", + "format": "s", + "label": "seconds", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:4861", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 38 + }, + "hiddenSeries": false, + "id": 64, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:6323", + "alias": "/.*Max*./", + "color": "#890F02", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_max_fds{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Maximum open file descriptors", + "refId": "A", + "step": 240 + }, + { + "expr": "process_open_fds{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Open file descriptors", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:6338", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:6339", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "System Misc", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 304, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 158, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:6726", + "alias": "/.*Critical*./", + "color": "#E24D42", + "fill": 0 + }, + { + "$$hashKey": "object:6727", + "alias": "/.*Max*./", + "color": "#EF843C", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_hwmon_temp_celsius{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ chip }} {{ sensor }} temp", + "refId": "A", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ chip }} {{ sensor }} Critical Alarm", + "refId": "B", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_celsius{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ chip }} {{ sensor }} Critical", + "refId": "C", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ chip }} {{ sensor }} Critical Historical", + "refId": "D", + "step": 240 + }, + { + "expr": "node_hwmon_temp_max_celsius{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ chip }} {{ sensor }} Max", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Hardware temperature monitor", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:6750", + "format": "celsius", + "label": "temperature", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:6751", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 300, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1655", + "alias": "/.*Max*./", + "color": "#EF843C", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_cooling_device_cur_state{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Current {{ name }} in {{ type }}", + "refId": "A", + "step": 240 + }, + { + "expr": "node_cooling_device_max_state{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Max {{ name }} in {{ type }}", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Throttle cooling device", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1678", + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1679", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 302, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_power_supply_online{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ power_supply }} online", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Power supply", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1678", + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1679", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Hardware Misc", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 296, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 297, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_systemd_socket_accepted_connections_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ name }} Connections", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Systemd Sockets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 10 + }, + "hiddenSeries": false, + "id": 298, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Failed", + "color": "#F2495C" + }, + { + "alias": "Inactive", + "color": "#FF9830" + }, + { + "alias": "Active", + "color": "#73BF69" + }, + { + "alias": "Deactivating", + "color": "#FFCB7D" + }, + { + "alias": "Activating", + "color": "#C8F2C2" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_systemd_units{instance=~\"[[instance]]\",job=\"host_node_exporter\",state=\"activating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Activating", + "refId": "A", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=~\"[[instance]]\",job=\"host_node_exporter\",state=\"active\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Active", + "refId": "B", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=~\"[[instance]]\",job=\"host_node_exporter\",state=\"deactivating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Deactivating", + "refId": "C", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=~\"[[instance]]\",job=\"host_node_exporter\",state=\"failed\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Failed", + "refId": "D", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=~\"[[instance]]\",job=\"host_node_exporter\",state=\"inactive\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Inactive", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Systemd Units State", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Systemd", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 270, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number (after merges) of I/O requests completed per second for the device", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:2033", + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "$$hashKey": "object:2034", + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "$$hashKey": "object:2035", + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "$$hashKey": "object:2036", + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "$$hashKey": "object:2037", + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "$$hashKey": "object:2038", + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "$$hashKey": "object:2039", + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "$$hashKey": "object:2040", + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "$$hashKey": "object:2041", + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "$$hashKey": "object:2042", + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "$$hashKey": "object:2043", + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "$$hashKey": "object:2044", + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "$$hashKey": "object:2045", + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "$$hashKey": "object:2046", + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "$$hashKey": "object:2047", + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "$$hashKey": "object:2048", + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "$$hashKey": "object:2049", + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "$$hashKey": "object:2050", + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "$$hashKey": "object:2051", + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "$$hashKey": "object:2052", + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "$$hashKey": "object:2053", + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_reads_completed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{ instance }} {{device}} - Reads completed", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_disk_writes_completed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk IOps Completed", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2186", + "format": "iops", + "label": "IO read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2187", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of bytes read from or written to the device per second", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 29 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{ instance }} {{device}} - Read bytes", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_disk_written_bytes_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Written bytes", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk R/W Data", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:369", + "format": "Bps", + "label": "bytes read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:370", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_time_seconds_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{ instance }} {{device}} - Read wait time avg", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_disk_write_time_seconds_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Write wait time avg", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk Average Wait Time", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:441", + "format": "s", + "label": "time. read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:442", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The average queue length of the requests that were issued to the device", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 39 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_io_time_weighted_seconds_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{ instance }} {{device}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Average Queue Size", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:513", + "format": "none", + "label": "aqu-sz", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:514", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of read and write requests merged per second that were queued to the device", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 49 + }, + "hiddenSeries": false, + "id": 133, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_reads_merged_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Read merged", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_disk_writes_merged_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Write merged", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk R/W Merged", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:585", + "format": "iops", + "label": "I/Os", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:586", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 49 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_io_time_seconds_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{ instance }} {{device}} - IO", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_disk_discard_time_seconds_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{ instance }} {{device}} - discard", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time Spent Doing I/Os", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:657", + "format": "percentunit", + "label": "%util", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:658", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 59 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_disk_io_now{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{ instance }} {{device}} - IO now", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Instantaneous Queue Size", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:729", + "format": "none", + "label": "Outstanding req.", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:730", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 59 + }, + "hiddenSeries": false, + "id": 301, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:2034", + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "$$hashKey": "object:2035", + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "$$hashKey": "object:2036", + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "$$hashKey": "object:2037", + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "$$hashKey": "object:2038", + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "$$hashKey": "object:2039", + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "$$hashKey": "object:2040", + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "$$hashKey": "object:2041", + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "$$hashKey": "object:2042", + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "$$hashKey": "object:2043", + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "$$hashKey": "object:2044", + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "$$hashKey": "object:2045", + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "$$hashKey": "object:2046", + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "$$hashKey": "object:2047", + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "$$hashKey": "object:2048", + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "$$hashKey": "object:2049", + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "$$hashKey": "object:2050", + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "$$hashKey": "object:2051", + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "$$hashKey": "object:2052", + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "$$hashKey": "object:2053", + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_discards_completed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{ instance }} {{device}} - Discards completed", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_disk_discards_merged_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Discards merged", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk IOps Discards completed / merged", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2186", + "format": "iops", + "label": "IOs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2187", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Storage Disk", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 271, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 3, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "node_filesystem_avail_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{mountpoint}} - Available", + "metric": "", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filesystem space available", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3826", + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3827", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_files_free{instance=~\"[[instance]]\",job=\"host_node_exporter\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{mountpoint}} - Free file nodes", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Nodes Free", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3894", + "format": "short", + "label": "file nodes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3895", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filefd_maximum{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{ instance }} Max open files", + "refId": "A", + "step": 240 + }, + { + "expr": "node_filefd_allocated{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Open files", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Descriptor", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "files", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 22 + }, + "hiddenSeries": false, + "id": 219, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_files{instance=~\"[[instance]]\",job=\"host_node_exporter\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{mountpoint}} - File nodes total", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Nodes Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "file Nodes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "/ ReadOnly": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_readonly{instance=~\"[[instance]]\",job=\"host_node_exporter\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{mountpoint}} - ReadOnly", + "refId": "A", + "step": 240 + }, + { + "expr": "node_filesystem_device_error{instance=~\"[[instance]]\",job=\"host_node_exporter\",device!~'rootfs',fstype!~'tmpfs'}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{mountpoint}} - Device error", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filesystem in ReadOnly / Error", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3670", + "format": "short", + "label": "counter", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3671", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Storage Filesystem", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 272, + "panels": [ + { + "aliasColors": { + "receive_packets_eth0": "#7EB26D", + "receive_packets_lo": "#E24D42", + "transmit_packets_eth0": "#7EB26D", + "transmit_packets_lo": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_packets_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_network_transmit_packets_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic by Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "hiddenSeries": false, + "id": 142, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_errs_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Receive errors", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_network_transmit_errs_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Rransmit errors", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 143, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_drop_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Receive drop", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_network_transmit_drop_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Transmit drop", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Drop", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "hiddenSeries": false, + "id": 141, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_compressed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Receive compressed", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_network_transmit_compressed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Transmit compressed", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Compressed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 50 + }, + "hiddenSeries": false, + "id": 146, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_multicast_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Receive multicast", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Multicast", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 50 + }, + "hiddenSeries": false, + "id": 144, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_fifo_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Receive fifo", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_network_transmit_fifo_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Transmit fifo", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Fifo", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 60 + }, + "hiddenSeries": false, + "id": 145, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:576", + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_frame_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Receive frame", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Frame", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:589", + "format": "pps", + "label": "packets out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:590", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 60 + }, + "hiddenSeries": false, + "id": 231, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_carrier_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Statistic transmit_carrier", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Carrier", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 232, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_colls_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{device}} - Transmit colls", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic Colls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 70 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:663", + "alias": "NF conntrack limit", + "color": "#890F02", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_nf_conntrack_entries{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} NF conntrack entries", + "refId": "A", + "step": 240 + }, + { + "expr": "node_nf_conntrack_entries_limit{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} NF conntrack limit", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "NF Contrack", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:678", + "format": "short", + "label": "entries", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:679", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 80 + }, + "hiddenSeries": false, + "id": 230, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_arp_entries{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ device }} - ARP entries", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "ARP Entries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Entries", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 80 + }, + "hiddenSeries": false, + "id": 288, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_network_mtu_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ device }} - Bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "MTU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 90 + }, + "hiddenSeries": false, + "id": 280, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_network_speed_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ device }} - Speed", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Speed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 90 + }, + "hiddenSeries": false, + "id": 289, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_network_transmit_queue_length{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{ device }} - Interface transmit queue length", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Queue Length", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": "packets", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 100 + }, + "hiddenSeries": false, + "id": 290, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:232", + "alias": "/.*Dropped.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_softnet_processed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} CPU {{cpu}} - Processed", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_softnet_dropped_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} CPU {{cpu}} - Dropped", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Softnet Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:207", + "format": "short", + "label": "packetes drop (-) / process (+)", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:208", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 100 + }, + "hiddenSeries": false, + "id": 310, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_softnet_times_squeezed_total{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} CPU {{cpu}} - Squeezed", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Softnet Out of Quota", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:207", + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:208", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 309, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_network_up{operstate=\"up\",instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{interface}} - Operational state UP", + "refId": "A", + "step": 240 + }, + { + "expr": "node_network_carrier{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ instance }} {{device}} - Physical link state", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Operational Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Network Traffic", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 273, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 63, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_TCP_alloc{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} TCP_alloc - Allocated sockets", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_inuse{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} TCP_inuse - Tcp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_mem{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} TCP_mem - Used memory for tcp", + "refId": "C", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_orphan{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} TCP_orphan - Orphan sockets", + "refId": "D", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_tw{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} TCP_tw - Sockets wating close", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sockstat TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "hiddenSeries": false, + "id": 124, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_UDPLITE_inuse{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} UDPLITE_inuse - Udplite sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_inuse{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} UDP_inuse - Udp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_mem{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} UDP_mem - Used memory for udp", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sockstat UDP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 125, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_FRAG_inuse{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} FRAG_inuse - Frag sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_RAW_inuse{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} RAW_inuse - Raw sockets currently in use", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sockstat FRAG / RAW", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1572", + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1573", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 42 + }, + "hiddenSeries": false, + "id": 220, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_TCP_mem_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} mem_bytes - TCP sockets in that state", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_mem_bytes{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} mem_bytes - UDP sockets in that state", + "refId": "B", + "step": 240 + }, + { + "expr": "node_sockstat_FRAG_memory{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} FRAG_memory - Used memory for frag", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sockstat Memory Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 126, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_sockets_used{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Sockets_used - Sockets currently in use", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sockstat Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "sockets", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Network Sockstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 274, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 33 + }, + "height": "", + "hiddenSeries": false, + "id": 221, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1876", + "alias": "/.*Out.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_netstat_IpExt_InOctets{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} InOctets - Received octets", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_netstat_IpExt_OutOctets{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ instance }} OutOctets - Sent octets", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Netstat IP In / Out Octets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1889", + "format": "short", + "label": "octects out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1890", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 33 + }, + "height": "", + "hiddenSeries": false, + "id": 81, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_netstat_Ip_Forwarding{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} Forwarding - IP forwarding", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Netstat IP Forwarding", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1957", + "format": "short", + "label": "datagrams", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1958", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 43 + }, + "height": "", + "hiddenSeries": false, + "id": 115, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_netstat_Icmp_InMsgs{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_netstat_Icmp_OutMsgs{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "ICMP In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "messages out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 43 + }, + "height": "", + "hiddenSeries": false, + "id": 50, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_netstat_Icmp_InErrors{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "ICMP Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "messages out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 53 + }, + "height": "", + "hiddenSeries": false, + "id": 55, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Snd.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_netstat_Udp_InDatagrams{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} InDatagrams - Datagrams received", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_netstat_Udp_OutDatagrams{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} OutDatagrams - Datagrams sent", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "UDP In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "datagrams out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 53 + }, + "height": "", + "hiddenSeries": false, + "id": 109, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_netstat_Udp_InErrors{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} InErrors - UDP Datagrams that could not be delivered to an application", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_netstat_Udp_NoPorts{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} NoPorts - UDP Datagrams received on a port with no listener", + "refId": "B", + "step": 240 + }, + { + "expr": "rate(node_netstat_UdpLite_InErrors{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{ instance }} InErrors Lite - UDPLite Datagrams that could not be delivered to an application", + "refId": "C" + }, + { + "expr": "rate(node_netstat_Udp_RcvbufErrors{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} RcvbufErrors - UDP buffer errors received", + "refId": "D", + "step": 240 + }, + { + "expr": "rate(node_netstat_Udp_SndbufErrors{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} SndbufErrors - UDP buffer errors send", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "UDP Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:4232", + "format": "short", + "label": "datagrams", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:4233", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 63 + }, + "height": "", + "hiddenSeries": false, + "id": 299, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Snd.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_netstat_Tcp_InSegs{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_netstat_Tcp_OutSegs{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "datagrams out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 63 + }, + "height": "", + "hiddenSeries": false, + "id": 104, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_netstat_TcpExt_ListenOverflows{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} ListenOverflows - Times the listen queue of a socket overflowed", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_netstat_TcpExt_ListenDrops{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} ListenDrops - SYNs to LISTEN sockets ignored", + "refId": "B", + "step": 240 + }, + { + "expr": "rate(node_netstat_TcpExt_TCPSynRetrans{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", + "refId": "C", + "step": 240 + }, + { + "expr": "rate(node_netstat_Tcp_RetransSegs{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{ instance }} RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", + "refId": "D" + }, + { + "expr": "rate(node_netstat_Tcp_InErrs{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{ instance }} InErrs - Segments received in error (e.g., bad TCP checksums)", + "refId": "E" + }, + { + "expr": "rate(node_netstat_Tcp_OutRsts{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{ instance }} OutRsts - Segments sent with RST flag", + "refId": "F" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 73 + }, + "height": "", + "hiddenSeries": false, + "id": 85, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:454", + "alias": "/.*MaxConn *./", + "color": "#890F02", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "refId": "A", + "step": 240 + }, + { + "expr": "node_netstat_Tcp_MaxConn{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} MaxConn - Limit on the total number of TCP connections the entity can support (Dinamic is \"-1\")", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:469", + "format": "short", + "label": "connections", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:470", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 73 + }, + "height": "", + "hiddenSeries": false, + "id": 91, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Sent.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_netstat_TcpExt_SyncookiesFailed{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} SyncookiesFailed - Invalid SYN cookies received", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_netstat_TcpExt_SyncookiesRecv{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} SyncookiesRecv - SYN cookies received", + "refId": "B", + "step": 240 + }, + { + "expr": "rate(node_netstat_TcpExt_SyncookiesSent{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} SyncookiesSent - SYN cookies sent", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP SynCookie", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "counter out (-) / in (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 83 + }, + "height": "", + "hiddenSeries": false, + "id": 82, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 12, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_netstat_Tcp_ActiveOpens{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_netstat_Tcp_PassiveOpens{instance=~\"[[instance]]\",job=\"host_node_exporter\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP Direct Transition", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "connections", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Network Netstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 279, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_scrape_collector_duration_seconds{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{collector}} - Scrape duration", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Node Exporter Scrape Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "seconds", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 54 + }, + "hiddenSeries": false, + "id": 157, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1969", + "alias": "/.*error.*/", + "color": "#F2495C", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_scrape_collector_success{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{collector}} - Scrape success", + "refId": "A", + "step": 240 + }, + { + "expr": "node_textfile_scrape_error{instance=~\"[[instance]]\",job=\"host_node_exporter\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }} {{collector}} - Scrape textfile error (1 = true)", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Node Exporter Scrape", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1484", + "format": "short", + "label": "counter", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1485", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Node Exporter", + "type": "row" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [ + "linux", + "physical" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "IC Metrics (cluster local)", + "value": "IC Metrics (cluster local)" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "/IC.*/", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(up{job=\"host_node_exporter\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(up{job=\"host_node_exporter\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "diskdevice", + "options": [ + { + "selected": true, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + } + ], + "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Node Exporter", + "uid": "51412e5a92249978fe6e78d639a2c9fb", + "version": 9, + "weekStart": "" +} \ No newline at end of file diff --git a/docker/config/grafana/provisioning/dashboards/samples/power-usage.json b/docker/config/grafana/provisioning/dashboards/samples/power-usage.json new file mode 100644 index 0000000..789ee17 --- /dev/null +++ b/docker/config/grafana/provisioning/dashboards/samples/power-usage.json @@ -0,0 +1,324 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "power_average_watts{instance=~\"[[instance]]\", ic_subnet=~\"[[ic_subnet]]\"}", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Average power usage over time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "52fa8e1a-4143-4757-b4d7-aff2a7e4ae02" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 4, + "interval": "60s", + "options": { + "calculate": true, + "calculation": { + "xBuckets": { + "value": "$__interval" + }, + "yBuckets": { + "mode": "size", + "value": "20" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "min": 0, + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": "1e-9" + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "52fa8e1a-4143-4757-b4d7-aff2a7e4ae02" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(power_instantaneous_watts{instance=~\"[[instance]]\", ic_subnet=~\"[[ic_subnet]]\"}[$__interval])", + "format": "table", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Power draw distribution", + "transformations": [], + "type": "heatmap" + } + ], + "refresh": "1m", + "schemaVersion": 37, + "style": "dark", + "tags": [ + "linux", + "physical", + "power" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "IC Metrics (cluster local)", + "value": "IC Metrics (cluster local)" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "/IC.*/", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(up{job=\"host_node_exporter\"}, ic_subnet)", + "hide": 0, + "includeAll": true, + "label": "Subnet", + "multi": true, + "name": "ic_subnet", + "options": [], + "query": { + "query": "label_values(up{job=\"host_node_exporter\"}, ic_subnet)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(up{job=\"host_node_exporter\", ic_subnet=~\"[[ic_subnet]]\"}, instance)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(up{job=\"host_node_exporter\", ic_subnet=~\"[[ic_subnet]]\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Power usage", + "uid": "f50509061ae243e387ec9c42099c538a", + "version": 7, + "weekStart": "" +} \ No newline at end of file diff --git a/docker/config/grafana/provisioning/datasources/observability.yaml b/docker/config/grafana/provisioning/datasources/observability.yaml index b13a459..ae8d8ad 100644 --- a/docker/config/grafana/provisioning/datasources/observability.yaml +++ b/docker/config/grafana/provisioning/datasources/observability.yaml @@ -1,7 +1,7 @@ apiVersion: 1 datasources: - - name: Prometheus + - name: IC Metrics (cluster local) type: prometheus access: proxy url: http://$PROM_HOST:9090 diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 4efd837..004efaa 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -25,6 +25,12 @@ services: - --storage.tsdb.retention.time=6m ports: - 9090:9090 + deploy: + resources: + limits: + memory: 12G + reservations: + memory: 12G grafana: image: grafana/grafana:12.2.1 From 35a8d4c318916336bf54edbeed61432bac7e8a0c Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Fri, 7 Nov 2025 00:43:39 +0100 Subject: [PATCH 12/25] Adding alerting templates and notifications --- docker/README.md | 2 + .../grafana/provisioning/alerting/.gitignore | 1 + .../alerting/notification_policy.yaml | 10 + .../provisioning/alerting/templates.yaml | 39 +++ .../templates/template_contact_points.yaml | 284 ++++++++++++++++++ 5 files changed, 336 insertions(+) create mode 100644 docker/config/grafana/provisioning/alerting/.gitignore create mode 100644 docker/config/grafana/provisioning/alerting/notification_policy.yaml create mode 100644 docker/config/grafana/provisioning/alerting/templates.yaml create mode 100644 docker/config/grafana/templates/template_contact_points.yaml diff --git a/docker/README.md b/docker/README.md index a58373a..5f7ce1f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -4,3 +4,5 @@ docker compose -f ./docker-compose.tools.yaml run --rm prom-config-builder \ tools/prom-config-builder/prom_config_builder.py \ --node-provider-id bvcsg-3od6r-jnydw-eysln-aql7w-td5zn-ay5m6-sibd2-jzojt-anwag-mqe ``` + +# Explain how to configure contact points diff --git a/docker/config/grafana/provisioning/alerting/.gitignore b/docker/config/grafana/provisioning/alerting/.gitignore new file mode 100644 index 0000000..48d7f23 --- /dev/null +++ b/docker/config/grafana/provisioning/alerting/.gitignore @@ -0,0 +1 @@ +contact_points.yaml diff --git a/docker/config/grafana/provisioning/alerting/notification_policy.yaml b/docker/config/grafana/provisioning/alerting/notification_policy.yaml new file mode 100644 index 0000000..bf87b75 --- /dev/null +++ b/docker/config/grafana/provisioning/alerting/notification_policy.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +policies: + - receiver: Obs contact points + matchers: + - severity =~ "warning|critical" + group_by: + - alertname + - ic_node + - job diff --git a/docker/config/grafana/provisioning/alerting/templates.yaml b/docker/config/grafana/provisioning/alerting/templates.yaml new file mode 100644 index 0000000..7879e0c --- /dev/null +++ b/docker/config/grafana/provisioning/alerting/templates.yaml @@ -0,0 +1,39 @@ +apiVersion: 1 + +templates: + - name: obs.message + template: | + {{ define "alert_message" }} + **ALERT: {{ .Status | toUpper }}** + {{ range .Alerts }} + --- + **Alert Name:** {{ .Labels.alertname }} ({{ .Labels.severity | toUpper }}) + **Status:** {{ .Status }} + + {{ if .Annotations.description }} + **Description:** {{ .Annotations.description }} + {{ end }} + + **Labels:** + {{ range $key, $value := .Labels }} + • {{ $key }}: {{ $value }} + {{ end }} + + {{ if .Annotations }} + **Annotations:** + {{ range $key, $value := .Annotations }} + • {{ $key }}: {{ $value }} + {{ end }} + {{ end }} + + **Origin Generator URL:** + <{{ .GeneratorURL }}> + + {{ end }} + {{ end }} + + - name: obs.subject + template: | + {{ define "obs.subject" }} + {{ len .Alerts.Firing }} firing alert(s), {{ len .Alerts.Resolved }} resolved alert(s) + {{ end }} diff --git a/docker/config/grafana/templates/template_contact_points.yaml b/docker/config/grafana/templates/template_contact_points.yaml new file mode 100644 index 0000000..cd8f4cf --- /dev/null +++ b/docker/config/grafana/templates/template_contact_points.yaml @@ -0,0 +1,284 @@ +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: Obs contact points + receivers: + # - uid: alertmanager + # type: prometheus-alertmanager + # settings: + # url: + # basicAuthUser: # keep commented out if not needed + # basicAuthPassword: # keep commented out if not needed + # + # - uid: dingdong + # type: dingdong + # settings: + # url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxx + # msgType: link + # message: | + # {{ template "obs.message" . }} + # + # - uid: discord + # type: discord + # settings: + # url: https://discord/webhook + # avatar_url: https://my_avatar + # use_discord_username: false + # message: | + # {{ template "obs.message" . }} + # + # - uid: email + # type: email + # settings: + # addresses: me@example.com;you@example.com + # singleEmail: false + # subject: | + # {{ template "obs.title" . }} + # + # - uid: googlechat + # type: googlechat + # settings: + # url: https://google/webhook + # message: | + # {{ template "obs.message" . }} + # + # - uid: kafka + # type: kafka + # settings: + # kafkaRestProxy: http://localhost:8082 + # kafkaTopic: topic1 + # + # - uid: line + # type: line + # settings: + # token: xxx + # + # - uid: mqtt + # type: mqtt + # settings: + # # + # brokerUrl: tcp://127.0.0.1:1883 + # # + # clientId: grafana + # # + # topic: grafana/alerts + # # + # messageFormat: json + # # + # username: grafana + # # + # password: password1 + # # + # qos: 0 + # # + # retain: false + # # + # tlsConfig: + # # + # insecureSkipVerify: false + # # + # clientCertificate: certificate in PEM format + # # + # clientKey: key in PEM format + # # + # caCertificate: CA certificate in PEM format + # + # - uid: teams + # type: teams + # settings: + # # + # url: https://ms_teams_url + # # + # title: | + # {{ template "obs.title" . }} + # # + # sectiontitle: '' + # # + # message: | + # {{ template "obs.message" . }} + # + # - uid: opsgenie + # type: opsgenie + # settings: + # # + # apiKey: xxx + # # + # apiUrl: https://api.opsgenie.com/v2/alerts + # # + # message: | + # {{ template "obs.title" . }} + # # + # description: some descriptive description + # # + # autoClose: false + # # + # overridePriority: false + # # options: tags, details, both + # sendTagsAs: both + # + # - uid: pagerduty + # type: pagerduty + # settings: + # # the 32-character Events API key https://support.pagerduty.com/docs/api-access-keys#events-api-keys + # integrationKey: XXX + # # options: critical, error, warning, info + # severity: critical + # # + # class: ping failure + # # + # component: Grafana + # # + # group: app-stack + # # + # summary: | + # {{ template "obs.message" . }} + # + # - uid: pushover + # type: pushover + # settings: + # # + # apiToken: XXX + # # + # userKey: user1,user2 + # # + # device: device1,device2 + # # options (high to low): 2,1,0,-1,-2 + # priority: '2' + # # + # retry: '30' + # # + # expire: '120' + # # the number of seconds before a message expires and is deleted automatically. Examples: 10s, 5m30s, 8h. + # ttl: + # # + # sound: siren + # # + # okSound: magic + # # + # message: | + # {{ template "obs.message" . }} + # + # - uid: slack + # type: slack + # settings: + # # + # recipient: alerting-dev + # # + # token: xxx + # # + # username: grafana_bot + # # + # icon_emoji: heart + # # + # icon_url: https://icon_url + # # + # mentionUsers: user_1,user_2 + # # + # mentionGroups: group_1,group_2 + # # options: here, channel + # mentionChannel: here + # # Optionally provide a Slack incoming webhook URL for sending messages, in this case the token isn't necessary + # url: https://some_webhook_url + # # + # endpointUrl: https://custom_url/api/chat.postMessage + # # + # title: | + # {{ template "slack.obs.title" . }} + # text: | + # {{ template "slack.default.text" . }} + # + # - uid: sensugo + # type: sensugo + # settings: + # # + # url: http://sensu-api.local:8080 + # # + # apikey: xxx + # # + # entity: default + # # + # check: default + # # + # handler: some_handler + # # + # namespace: default + # # + # message: | + # {{ template "obs.message" . }} + # + # - uid: telegram + # type: telegram + # settings: + # # + # bottoken: xxx + # # + # chatid: some_chat_id + # # + # message: | + # {{ template "obs.message" . }} + # + # - uid: threema + # type: threema + # settings: + # # + # api_secret: xxx + # # + # gateway_id: A5K94S9 + # # + # recipient_id: A9R4KL4S + # + # - uid: victorops + # type: victorops + # settings: + # # + # url: XXX + # # options: CRITICAL, WARNING + # messageType: CRITICAL + # + # - uid: webhook + # type: webhook + # settings: + # # + # url: https://endpoint_url + # # options: POST, PUT + # httpMethod: POST + # # + # username: abc + # # + # password: abc123 + # # + # authorization_scheme: Bearer + # # + # authorization_credentials: abc123 + # # + # maxAlerts: '10' + # # + # tlsConfig: + # # + # insecureSkipVerify: false + # # + # clientCertificate: certificate in PEM format + # # + # clientKey: key in PEM format + # # + # caCertificate: CA certificate in PEM format + # hmacConfig: + # # + # secret: secret-key + # # + # header: X-Grafana-Alerting-Signature + # # + # timestampHeader: X-Grafana-Alerting-Signature-Timestamp + # + # - uid: wecom + # type: wecom + # settings: + # # + # url: https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxxxxxxx + # # + # message: | + # {{ template "obs.message" . }} + # # + # title: | + # {{ template "obs.title" . }} From 90e342ec4cf87ed07b591163635b55ef0b96778e Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Fri, 7 Nov 2025 01:01:32 +0100 Subject: [PATCH 13/25] fixing contact points --- .../grafana/provisioning/alerting/notification_policy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/config/grafana/provisioning/alerting/notification_policy.yaml b/docker/config/grafana/provisioning/alerting/notification_policy.yaml index bf87b75..d5be639 100644 --- a/docker/config/grafana/provisioning/alerting/notification_policy.yaml +++ b/docker/config/grafana/provisioning/alerting/notification_policy.yaml @@ -8,3 +8,4 @@ policies: - alertname - ic_node - job + From 91f6e21f799abea2dc40fc8484615e107333e82d Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Fri, 7 Nov 2025 01:47:43 +0100 Subject: [PATCH 14/25] adding explanations --- .../templates/template_contact_points.yaml | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/docker/config/grafana/templates/template_contact_points.yaml b/docker/config/grafana/templates/template_contact_points.yaml index cd8f4cf..6dc5f6d 100644 --- a/docker/config/grafana/templates/template_contact_points.yaml +++ b/docker/config/grafana/templates/template_contact_points.yaml @@ -1,9 +1,24 @@ apiVersion: 1 +# NOTE: +# After once enabling the contact point, later commenting out and restarting won't remove the +# contact point. Once provisioned, contact points are stored in the database and can't be +# deleted. To delete them, you have to: +# 1. stop grafana (docker compose down) +# 2. sudo rm -rf ./volumes/grafana +# 3. git checkout -- ./volumes/grafana +# 4. sudo chmod -R 777 ./volumes/grafana +# 5. make changes to the contact_points.yaml (the one in ./config/grafana/provisioning/alerting) +# 6. docker compose up -d + contactPoints: - orgId: 1 name: Obs contact points receivers: + # This receiver is useful if you have a bigger + # observability setup somewhere and just want to + # forward the alerts to that Alertmanager which + # will later route alerts to you in a prefered way. # - uid: alertmanager # type: prometheus-alertmanager # settings: @@ -11,14 +26,8 @@ contactPoints: # basicAuthUser: # keep commented out if not needed # basicAuthPassword: # keep commented out if not needed # - # - uid: dingdong - # type: dingdong - # settings: - # url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxx - # msgType: link - # message: | - # {{ template "obs.message" . }} - # + # This option is free. You can create your own server + # and in it, in settings, you can add "integrations" # - uid: discord # type: discord # settings: @@ -28,6 +37,9 @@ contactPoints: # message: | # {{ template "obs.message" . }} # + # To use this receiver you need to add smtp configuration + # in `./config/grafana/grafana.ini`. To read more visit: + # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#smtp # - uid: email # type: email # settings: @@ -36,6 +48,11 @@ contactPoints: # subject: | # {{ template "obs.title" . }} # + # This is only possible if you have a google organization. + # If you have an organization, in it you can create a new + # space. In Space Settings > Apps & Integrations press + # "Add webhooks", pick a name and an avatar and a URL will + # be generated which you should put in `url` part of `settings` # - uid: googlechat # type: googlechat # settings: @@ -49,11 +66,6 @@ contactPoints: # kafkaRestProxy: http://localhost:8082 # kafkaTopic: topic1 # - # - uid: line - # type: line - # settings: - # token: xxx - # # - uid: mqtt # type: mqtt # settings: From 9810d5824d344b095219d2a0cb02c0baa64d7bc1 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Fri, 7 Nov 2025 01:50:01 +0100 Subject: [PATCH 15/25] updating readme --- docker/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/README.md b/docker/README.md index 5f7ce1f..0fa19e6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,3 +6,6 @@ docker compose -f ./docker-compose.tools.yaml run --rm prom-config-builder \ ``` # Explain how to configure contact points + +- sudo chmod 777 -R ./volumes/grafana/ +- sudo chmod 777 -R ./volumes/prometheus/ From 9ff0f8b05bebe6feb6b1dd526846fc7e3ba6d025 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Sat, 8 Nov 2025 00:37:45 +0100 Subject: [PATCH 16/25] fixing alerts --- docker/README.md | 4 +- .../grafana/provisioning/alerting/alerts.yaml | 60 +++++++++++++++++++ .../alerting/notification_policy.yaml | 4 +- .../provisioning/alerting/templates.yaml | 13 +--- .../dashboards/samples/power-usage.json | 6 +- .../datasources/observability.yaml | 3 +- 6 files changed, 71 insertions(+), 19 deletions(-) create mode 100644 docker/config/grafana/provisioning/alerting/alerts.yaml diff --git a/docker/README.md b/docker/README.md index 0fa19e6..c1f8b94 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,8 +1,6 @@ ## Command for configuring prometheus: ```bash -docker compose -f ./docker-compose.tools.yaml run --rm prom-config-builder \ - tools/prom-config-builder/prom_config_builder.py \ - --node-provider-id bvcsg-3od6r-jnydw-eysln-aql7w-td5zn-ay5m6-sibd2-jzojt-anwag-mqe +docker compose -f ./docker-compose.tools.yaml run --rm prom-config-builder tools/prom-config-builder/prom_config_builder.py --node-provider-id bvcsg-3od6r-jnydw-eysln-aql7w-td5zn-ay5m6-sibd2-jzojt-anwag-mqe --dc-id se1 ``` # Explain how to configure contact points diff --git a/docker/config/grafana/provisioning/alerting/alerts.yaml b/docker/config/grafana/provisioning/alerting/alerts.yaml new file mode 100644 index 0000000..6ab2647 --- /dev/null +++ b/docker/config/grafana/provisioning/alerting/alerts.yaml @@ -0,0 +1,60 @@ +apiVersion: 1 +groups: + - orgId: 1 + name: obs alert evaluations + folder: obs alerts + interval: 1m + rules: + - uid: Orchestrator_TargetMissing + title: Orchestrator_TargetMissing + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: builder + expr: up{job="orchestrator"} + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: eq + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 5m + annotations: + description: | + The orchestrator cannot be reached for this node. It could mean that the node is offline or the orchestrator died. + labels: + severity: critical + isPaused: false + diff --git a/docker/config/grafana/provisioning/alerting/notification_policy.yaml b/docker/config/grafana/provisioning/alerting/notification_policy.yaml index d5be639..450be44 100644 --- a/docker/config/grafana/provisioning/alerting/notification_policy.yaml +++ b/docker/config/grafana/provisioning/alerting/notification_policy.yaml @@ -2,8 +2,8 @@ apiVersion: 1 policies: - receiver: Obs contact points - matchers: - - severity =~ "warning|critical" + object_matchers: + - ['severity', '=~', 'critical'] group_by: - alertname - ic_node diff --git a/docker/config/grafana/provisioning/alerting/templates.yaml b/docker/config/grafana/provisioning/alerting/templates.yaml index 7879e0c..b4e5503 100644 --- a/docker/config/grafana/provisioning/alerting/templates.yaml +++ b/docker/config/grafana/provisioning/alerting/templates.yaml @@ -3,7 +3,7 @@ apiVersion: 1 templates: - name: obs.message template: | - {{ define "alert_message" }} + {{ define "obs.message" }} **ALERT: {{ .Status | toUpper }}** {{ range .Alerts }} --- @@ -19,21 +19,14 @@ templates: • {{ $key }}: {{ $value }} {{ end }} - {{ if .Annotations }} - **Annotations:** - {{ range $key, $value := .Annotations }} - • {{ $key }}: {{ $value }} - {{ end }} - {{ end }} - **Origin Generator URL:** <{{ .GeneratorURL }}> {{ end }} {{ end }} - - name: obs.subject + - name: obs.title template: | - {{ define "obs.subject" }} + {{ define "obs.title" }} {{ len .Alerts.Firing }} firing alert(s), {{ len .Alerts.Resolved }} resolved alert(s) {{ end }} diff --git a/docker/config/grafana/provisioning/dashboards/samples/power-usage.json b/docker/config/grafana/provisioning/dashboards/samples/power-usage.json index 789ee17..af98f5e 100644 --- a/docker/config/grafana/provisioning/dashboards/samples/power-usage.json +++ b/docker/config/grafana/provisioning/dashboards/samples/power-usage.json @@ -232,8 +232,8 @@ { "current": { "selected": false, - "text": "IC Metrics (cluster local)", - "value": "IC Metrics (cluster local)" + "text": "prometheus", + "value": "prometheus" }, "hide": 0, "includeAll": false, @@ -321,4 +321,4 @@ "uid": "f50509061ae243e387ec9c42099c538a", "version": 7, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/config/grafana/provisioning/datasources/observability.yaml b/docker/config/grafana/provisioning/datasources/observability.yaml index ae8d8ad..825b528 100644 --- a/docker/config/grafana/provisioning/datasources/observability.yaml +++ b/docker/config/grafana/provisioning/datasources/observability.yaml @@ -1,7 +1,8 @@ apiVersion: 1 datasources: - - name: IC Metrics (cluster local) + - name: prometheus + uid: prometheus type: prometheus access: proxy url: http://$PROM_HOST:9090 From 1c45213d52a61b5e5899ad0035a127adfc676a11 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Sat, 8 Nov 2025 00:44:21 +0100 Subject: [PATCH 17/25] proper down for orchestrator --- docker/config/grafana/provisioning/alerting/alerts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/config/grafana/provisioning/alerting/alerts.yaml b/docker/config/grafana/provisioning/alerting/alerts.yaml index 6ab2647..cc15e62 100644 --- a/docker/config/grafana/provisioning/alerting/alerts.yaml +++ b/docker/config/grafana/provisioning/alerting/alerts.yaml @@ -29,7 +29,7 @@ groups: conditions: - evaluator: params: - - 1 + - 0 type: eq operator: type: and From 509ffb22a9cfad434c1be6da1162ac9f02355e5f Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Sat, 8 Nov 2025 20:08:37 +0100 Subject: [PATCH 18/25] Adding some node exporter alerts --- .../grafana/provisioning/alerting/alerts.yaml | 984 +++++++++++++++++- 1 file changed, 981 insertions(+), 3 deletions(-) diff --git a/docker/config/grafana/provisioning/alerting/alerts.yaml b/docker/config/grafana/provisioning/alerting/alerts.yaml index cc15e62..ab079d9 100644 --- a/docker/config/grafana/provisioning/alerting/alerts.yaml +++ b/docker/config/grafana/provisioning/alerting/alerts.yaml @@ -48,8 +48,8 @@ groups: maxDataPoints: 43200 refId: C type: threshold - noDataState: NoData - execErrState: Error + noDataState: OK + execErrState: OK for: 5m annotations: description: | @@ -57,4 +57,982 @@ groups: labels: severity: critical isPaused: false - + - uid: HostOutOfMemory + title: HostOutOfMemory + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 2m + annotations: + description: | + Node memory is filling up (< 10% left)\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostMemoryUnderMemoryPressure + title: HostMemoryUnderMemoryPressure + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: rate(node_vmstat_pgmajfault[5m]) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1000 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + labels: + severity: warning + annotations: + description: | + The node is under heavy memory pressure. High rate of loading memory pages from disk.\n VALUE = {{ $value }} + isPaused: false + - uid: HostUnusualNetworkThroughputIn + title: HostUnusualNetworkThroughputIn + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (rate(node_network_receive_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.8 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + annotations: + description: | + Host receive bandwidth is high (>80%).\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostUnusualNetworkThroughputOut + title: HostUnusualNetworkThroughputOut + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (rate(node_network_transmit_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.8 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + annotations: + description: Host transmit bandwidth is high (>80%)\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostUnusualDiskReadRate + title: HostUnusualDiskReadRate + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: rate(node_disk_io_time_seconds_total[5m]) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.8 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + annotations: + description: | + Disk is too busy (IO wait > 80%)\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostOutOfDiskSpace + title: HostOutOfDiskSpace + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (node_filesystem_avail_bytes{mountpoint="/var/lib/ic/data"} * 100) / node_filesystem_size_bytes{mountpoint="/var/lib/ic/data"} + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 10 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 5m + annotations: + description: | + Disk is almost full (< 10% left)\n VALUE = {{ $value }} + labels: + severity: critical + isPaused: false + - uid: HostFilesystemDeviceError + title: HostFilesystemDeviceError + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: node_filesystem_device_error{mountpoint="/var/lib/ic/data"} + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: eq + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 2m + annotations: + description: | + Error stat-ing the {{ $labels.mountpoint }} filesystem\n VALUE = {{ $value }} + labels: + severity: critical + isPaused: false + - uid: HostUnusualDiskReadLatency + title: HostUnusualDiskReadLatency + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 2m + annotations: + description: | + Disk latency is growing (read operations > 100ms)\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostUnusualDiskWriteLatency + title: HostUnusualDiskWriteLatency + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 2m + annotations: + description: | + Disk latency is growing (write operations > 100ms)\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostHighCpuLoad + title: HostHighCpuLoad + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: 1 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.8 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 5m + annotations: + description: | + CPU load is > 80%\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostCpuHighIOWait + title: HostCpuHighIOWait + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.1 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + annotations: + description: | + CPU iowait > 10%. Your CPU is idling waiting for storage to respond.\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostUnusualDiskIO + title: HostUnusualDiskIO + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: rate(node_disk_io_time_seconds_total[5m]) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.8 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 5m + annotations: + description: | + Disk usage >80%. Check storage for issues or increase IOPS capabilities. Check storage for issues.\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostSwapIsFillingUp + title: HostSwapIsFillingUp + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.8 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 2m + annotations: + description: | + Swap is filling up (>80%)\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostSystemdServiceCrashed + title: HostSystemdServiceCrashed + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: node_systemd_unit_state{state="failed"} + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: eq + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + annotations: + description: | + systemd service crashed\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostNodeOverTemperatureAlartm + title: HostNodeOverTemperatureAlartm + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (node_hwmon_temp_crit_alarm_celsius == 1) or (node_hwmon_temp_alarm == 1) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: eq + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + annotations: + description: | + Physical node temperature alarm triggered\n VALUE = {{ $value }} + labels: + severity: critical + isPaused: false + - uid: HostPhysicalComponentTooHot + title: HostPhysicalComponentTooHot + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: node_hwmon_temp_celsius > node_hwmon_temp_max_celsius + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: eq + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 5m + annotations: + description: | + Physical hardware component too hot\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostOomKillDetected + title: HostOomKillDetected + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: increase(node_vmstat_oom_kill[5m]) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + annotations: + description: | + OOM kill detected\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostNetworkReceiveErrors + title: HostNetworkReceiveErrors + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.01 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 2m + annotations: + description: | + Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: HostNetworkTransmitErrors + title: HostNetworkTransmitErrors + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.01 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 2m + annotations: + description: | + Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.\n VALUE = {{ $value }} + labels: + severity: warning + isPaused: false From 73b8f27b91029d3a08934547125da8b4a2bc8b0e Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Sat, 8 Nov 2025 23:46:59 +0100 Subject: [PATCH 19/25] Refactoring some alerts --- .../grafana/provisioning/alerting/alerts.yaml | 178 +++++++++++++++--- 1 file changed, 152 insertions(+), 26 deletions(-) diff --git a/docker/config/grafana/provisioning/alerting/alerts.yaml b/docker/config/grafana/provisioning/alerting/alerts.yaml index ab079d9..c88255b 100644 --- a/docker/config/grafana/provisioning/alerting/alerts.yaml +++ b/docker/config/grafana/provisioning/alerting/alerts.yaml @@ -102,10 +102,11 @@ groups: type: threshold noDataState: OK execErrState: OK - for: 2m + for: 5m annotations: description: | - Node memory is filling up (< 10% left)\n VALUE = {{ $value }} + Node memory is filling up (< 10% left) + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -158,7 +159,8 @@ groups: severity: warning annotations: description: | - The node is under heavy memory pressure. High rate of loading memory pages from disk.\n VALUE = {{ $value }} + The node is under heavy memory pressure. High rate of loading memory pages from disk. + VALUE = {{ $value }} isPaused: false - uid: HostUnusualNetworkThroughputIn title: HostUnusualNetworkThroughputIn @@ -207,7 +209,8 @@ groups: execErrState: OK annotations: description: | - Host receive bandwidth is high (>80%).\n VALUE = {{ $value }} + Host receive bandwidth is high (>80%). + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -257,7 +260,9 @@ groups: noDataState: OK execErrState: OK annotations: - description: Host transmit bandwidth is high (>80%)\n VALUE = {{ $value }} + description: | + Host transmit bandwidth is high (>80%) + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -308,7 +313,8 @@ groups: execErrState: OK annotations: description: | - Disk is too busy (IO wait > 80%)\n VALUE = {{ $value }} + Disk is too busy (IO wait > 80%) + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -323,7 +329,7 @@ groups: datasourceUid: prometheus model: editorMode: code - expr: (node_filesystem_avail_bytes{mountpoint="/var/lib/ic/data"} * 100) / node_filesystem_size_bytes{mountpoint="/var/lib/ic/data"} + expr: (node_filesystem_avail_bytes{mountpoint="/"} * 100) / node_filesystem_size_bytes{mountpoint="/"} instant: true intervalMs: 1000 legendFormat: __auto @@ -360,7 +366,8 @@ groups: for: 5m annotations: description: | - Disk is almost full (< 10% left)\n VALUE = {{ $value }} + Disk is almost full (< 10% left) + VALUE = {{ $value }} labels: severity: critical isPaused: false @@ -412,7 +419,8 @@ groups: for: 2m annotations: description: | - Error stat-ing the {{ $labels.mountpoint }} filesystem\n VALUE = {{ $value }} + Error stat-ing the {{ $labels.mountpoint }} filesystem + VALUE = {{ $value }} labels: severity: critical isPaused: false @@ -427,7 +435,7 @@ groups: datasourceUid: prometheus model: editorMode: code - expr: rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0 + expr: rate(node_disk_read_time_seconds_total[1m]) / (rate(node_disk_reads_completed_total[1m]) > 0) instant: true intervalMs: 1000 legendFormat: __auto @@ -461,10 +469,11 @@ groups: type: threshold noDataState: OK execErrState: OK - for: 2m + for: 10m annotations: description: | - Disk latency is growing (read operations > 100ms)\n VALUE = {{ $value }} + Disk latency is growing (read operations > 100ms) + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -479,7 +488,7 @@ groups: datasourceUid: prometheus model: editorMode: code - expr: rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0 + expr: rate(node_disk_write_time_seconds_total[1m]) / (rate(node_disk_writes_completed_total[1m]) > 0) instant: true intervalMs: 1000 legendFormat: __auto @@ -492,7 +501,7 @@ groups: conditions: - evaluator: params: - - 0 + - 0.1 type: gt operator: type: and @@ -513,10 +522,11 @@ groups: type: threshold noDataState: OK execErrState: OK - for: 2m + for: 10m annotations: description: | - Disk latency is growing (write operations > 100ms)\n VALUE = {{ $value }} + Disk latency is growing (write operations > 100ms) + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -568,7 +578,8 @@ groups: for: 5m annotations: description: | - CPU load is > 80%\n VALUE = {{ $value }} + CPU load is > 80% + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -619,7 +630,8 @@ groups: execErrState: OK annotations: description: | - CPU iowait > 10%. Your CPU is idling waiting for storage to respond.\n VALUE = {{ $value }} + CPU iowait > 10%. Your CPU is idling waiting for storage to respond. + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -671,7 +683,8 @@ groups: for: 5m annotations: description: | - Disk usage >80%. Check storage for issues or increase IOPS capabilities. Check storage for issues.\n VALUE = {{ $value }} + Disk usage >80%. Check storage for issues or increase IOPS capabilities. Check storage for issues. + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -723,7 +736,8 @@ groups: for: 2m annotations: description: | - Swap is filling up (>80%)\n VALUE = {{ $value }} + Swap is filling up (>80%) + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -774,7 +788,8 @@ groups: execErrState: OK annotations: description: | - systemd service crashed\n VALUE = {{ $value }} + systemd service crashed + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -825,7 +840,8 @@ groups: execErrState: OK annotations: description: | - Physical node temperature alarm triggered\n VALUE = {{ $value }} + Physical node temperature alarm triggered + VALUE = {{ $value }} labels: severity: critical isPaused: false @@ -877,7 +893,8 @@ groups: for: 5m annotations: description: | - Physical hardware component too hot\n VALUE = {{ $value }} + Physical hardware component too hot + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -928,7 +945,8 @@ groups: execErrState: OK annotations: description: | - OOM kill detected\n VALUE = {{ $value }} + OOM kill detected + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -980,7 +998,8 @@ groups: for: 2m annotations: description: | - Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.\n VALUE = {{ $value }} + Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ $value }} receive errors in the last two minutes. + VALUE = {{ $value }} labels: severity: warning isPaused: false @@ -1032,7 +1051,114 @@ groups: for: 2m annotations: description: | - Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.\n VALUE = {{ $value }} + Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ $value }} transmit errors in the last two minutes. + VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: IC_Node_DataPartitionAlmostFull + title: IC_Node_DataPartitionAlmostFull + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (node_filesystem_avail_bytes{mountpoint="/var/lib/ic/data"}) / node_filesystem_size_bytes{mountpoint="/var/lib/ic/data"} + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 5m + annotations: + description: | + This node's data partition is almost full (< 10% left) + VALUE = {{ $value }} + labels: + severity: warning + isPaused: false + - uid: IC_Node_BackupPartitionAlmostFull + title: IC_Node_BackupPartitionAlmostFull + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (node_filesystem_avail_bytes{mountpoint="/var/lib/ic/backup"}) / node_filesystem_size_bytes{mountpoint="/var/lib/ic/backup"} + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.3 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 5m + annotations: + description: | + This node's backup partition is almost full (< 30% left) + VALUE = {{ $value }} labels: severity: warning isPaused: false From dc77de7869007df48a90ced113eb526f308d2de5 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Mon, 10 Nov 2025 14:09:04 +0100 Subject: [PATCH 20/25] started readme --- docker/README.md | 162 +++++++++++++++++- .../dashboards/samples/node-exporter.json | 9 +- .../dashboards/samples/power-usage.json | 3 +- 3 files changed, 165 insertions(+), 9 deletions(-) diff --git a/docker/README.md b/docker/README.md index c1f8b94..8fea264 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,9 +1,167 @@ -## Command for configuring prometheus: +# IC observability stack + +This collection of software lets you use your own equipment to observe the health and +operation of the nodes in your datacenter that are part of the Internet Computer. It +is a collection of docker containers that, when deployed and configured, will collect +metrics about the performance of selected nodes and send alerts if something is off. + +## How does this work? + +The stack selects IC nodes and collects metrics from each node, saving them +in a local Prometheus database every few seconds. This database is +queryable through a Grafana deployed side-to-side with Prometheus. + +Nodes of the Internet Computer all make available to the public a series +of metrics (in [Prometheus text format](https://github.com/Prometheus/docs/blob/777846211d502a287ab2b304cb515dc779de3474/content/docs/instrumenting/exposition_formats.md#text-based-format)) +that can be collected and analyzed by software such as this. + +The stack has to be deployed within the same *network subnet* as the nodes so that +the scraping of the nodes can take place. This practically means that the nodes and the +machine that is running this stack have to be within the same data center and connected +to the same router. + +## Prerequisites + +* IPv6 connectivity. +* Root-equivalent access on your workstation, to deploy the software this stack + needs to be set up. +* Hardware. + * It is recommended to run this stack on a machine with at least 16GB ram and + 80-100GB storage. +* SSH access to the machine. + * This is needed to later remotely connect to the services and inspect issues. + * Setting up ssh server [guide](https://documentation.ubuntu.com/server/how-to/security/openssh-server/). + +## Preparation + +To start preparing your workstation for scraping and observing your nodes you have to +`ssh` into the machine (or be physically next to it in order to run commands on the +machine). + +Install [Docker](https://docs.docker.com/engine/install/) for your machine. There +shouldn't be visible differences across different operating systems. This setup has +been tested on Mac OS and Manjaro OS (Linux flavour) but hasn't been tested on +Windows machines. + +Once you can run docker commands and have `docker compose` you can proceed to the next step. + +## Configuration + +Now that you have `docker` you can proceed to configure the stack. + +### Scraping targets + +First thing that you have to configure is your scraping targets. To do that, find your +[principal id](https://support.dfinity.org/hc/en-us/articles/7365913875988-What-is-a-principal) +that correclates to your node provider id. You can find that from the [public dashboard](https://dashboard.internetcomputer.org/network/providers). After that you should find the id of the data center to which +you are deploying this stack. You can find that information also on the [public dashboard](https://dashboard.internetcomputer.org/network/providers). + +With that you can run the following command: +```bash +# NOTE: you have to be within the same directory as this README! +docker compose -f ./docker-compose.tools.yaml run --rm prom-config-builder tools/prom-config-builder/prom_config_builder.py --node-provider-id --dc-id +``` + +Example command for node provider Dfinity Stiftung for data center se1: ```bash docker compose -f ./docker-compose.tools.yaml run --rm prom-config-builder tools/prom-config-builder/prom_config_builder.py --node-provider-id bvcsg-3od6r-jnydw-eysln-aql7w-td5zn-ay5m6-sibd2-jzojt-anwag-mqe --dc-id se1 ``` -# Explain how to configure contact points +Once that executes, you should be able to see a new file at `./config/prometheus/config.yaml`. +This file contains the definitions for the scraping targets. It will be slightly different +for each node provider and each data center. It is not versione controlled and you can +always recreate it with running the above command if you lose it, or delete it. + +### Contact points + +This stack uses [Grafana](https://grafana.com/) to present the dashboards and to send +alerts. Sending alerts is done using the [contact points](https://grafana.com/docs/grafana/latest/alerting/fundamentals/notifications/contact-points/) which need to be configured. Grafana supports +various contact points, some of them can be used for free, some of them are paid. +This stack was tested with the following contact points: +* Discord [free] +* Slack +* Google chat + +To setup your prefered contact point copy over the template with the following command: +```bash +cp ./config/grafana/templates/template_contact_points.yaml ./config/grafana/provisioning/alerting/contact_points.yaml +``` + +After that, edit the new file on path `./config/grafana/provisioning/alerting/contact_points.yaml` +and uncomment your prefered contact point and configure it (to _uncomment_ you should +remove the initial `#` from the contact point definition. + +### Volumes + +Since prometheus and grafana have their own users which probably differ from yours +you need to make the volumes readable and writiable by those users. There are number +of ways how you can do this, but the easiest is to make it readable and writable +for all users on the machine: +```bash +sudo chmod 777 -R ./volumes/grafana/ +sudo chmod 777 -R ./volumes/prometheus/ +``` + +## Running the stack + +To deploy the stack run the following command: +```bash +# This will spawn the containers +docker compose -f ./docker-compose.yaml up -d +``` + +It will take some time for the services to start and to sync between one another. +You can monitor if containers are failing by running `docker ps` and see if +there are some restarts happening in the containers. To see more about Troubleshooting +read *Troubleshooting* + +## Usage + +Once started, you will see the following applications: +* Prometheus - http://localhost:9090 +* Grafana - http://localhost:3000 - default creds can be see in `./config/grafana/grafana.ini` + +After 5-10 minutes you should see targets discovered in prometheus on the [targets page](http://localhost:9090/targets?search=). Initially, they might apear in red and if you keep monitoring they should +slowly start getting blue, which means that the targets are successfuly scraped. + +You should also see some data incoming in the grafana [sample dashboard for the +node exporter](http://localhost:3000/d/1/node-exporter?orgId=1&from=now-3h&to=now&timezone=utc&var-datasource=prometheus&var-instance=$__all&var-diskdevice=%5Ba-z%5D%2B%7Cnvme%5B0-9%5D%2Bn%5B0-9%5D%2B%7Cmmcblk%5B0-9%5D%2B). +Here, you can see various information about the performance of nodes and their health. + +[Alerting tab](http://localhost:3000/alerting/list) will show `obs alert evaluations` +which will contain a list of preconfigured alerts. Here you can see if any of the +alerts are in in a problematic state. Most of the time they should be in `Normal` +state, this means that everything is fine. Some of them may occasionally go into a +`Pending` state, which means that they crossed the threshold for an alert but still +isn't happening for long enough to consider this a problem. When a `Pending` alert +is happening for long enough it will go into `Error` mode and you should receive an +alert on your preconfigured contact point. + +*NOTE*: Not all alerts are configured to send notifications if they fire, usually +because they are just warnings and cannot be acted upon. To see which are and +which aren't you can see `./config/grafana/provisioning/alerting/alerts.yaml` and +see which have the lavel `severity: critical` attached to them because only they +will send alerts to your contact point. If you wish to send all of them, just replace +the other ones that contain `severity: warning` with `severity: critical`. + +### Access to the services of the stack + +To access the stack remotely you can do the following: +```bash +ssh -L 3000:localhost:3000 -L 9090:localhost:9090 +``` + +Example command with all parameters: +```bash +ssh -L 3000:localhost:3000 -L 9090:localhost:9090 -i ~/.ssh/priv_key.pem myuser@192.168.15.15 +``` + +## Extending +### Dashboards +### Alerts + +## Troubleshooting + - sudo chmod 777 -R ./volumes/grafana/ - sudo chmod 777 -R ./volumes/prometheus/ diff --git a/docker/config/grafana/provisioning/dashboards/samples/node-exporter.json b/docker/config/grafana/provisioning/dashboards/samples/node-exporter.json index 15e4a83..4d21a62 100644 --- a/docker/config/grafana/provisioning/dashboards/samples/node-exporter.json +++ b/docker/config/grafana/provisioning/dashboards/samples/node-exporter.json @@ -12905,8 +12905,8 @@ { "current": { "selected": false, - "text": "IC Metrics (cluster local)", - "value": "IC Metrics (cluster local)" + "text": "prometheus", + "value": "prometheus" }, "hide": 0, "includeAll": false, @@ -12915,7 +12915,6 @@ "options": [], "query": "prometheus", "refresh": 1, - "regex": "/IC.*/", "skipUrlSync": false, "type": "datasource" }, @@ -13007,7 +13006,7 @@ }, "timezone": "utc", "title": "Node Exporter", - "uid": "51412e5a92249978fe6e78d639a2c9fb", + "uid": "1", "version": 9, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/config/grafana/provisioning/dashboards/samples/power-usage.json b/docker/config/grafana/provisioning/dashboards/samples/power-usage.json index af98f5e..ab12fa8 100644 --- a/docker/config/grafana/provisioning/dashboards/samples/power-usage.json +++ b/docker/config/grafana/provisioning/dashboards/samples/power-usage.json @@ -243,7 +243,6 @@ "query": "prometheus", "queryValue": "", "refresh": 1, - "regex": "/IC.*/", "skipUrlSync": false, "type": "datasource" }, @@ -318,7 +317,7 @@ "timepicker": {}, "timezone": "", "title": "Power usage", - "uid": "f50509061ae243e387ec9c42099c538a", + "uid": "2", "version": 7, "weekStart": "" } From ae94b225efee71c3e2a7e73d3117261ef8478c71 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Mon, 10 Nov 2025 14:49:43 +0100 Subject: [PATCH 21/25] Adding docs --- docker/README.md | 122 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 119 insertions(+), 3 deletions(-) diff --git a/docker/README.md b/docker/README.md index 8fea264..5e5a21d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -102,6 +102,13 @@ sudo chmod 777 -R ./volumes/grafana/ sudo chmod 777 -R ./volumes/prometheus/ ``` +*NOTE*: `./volumes/prometheus/` is your database and you should take special care not to +delete it, because if you do, you will lose all historical data that has been stored. +*NOTE*: `./volumes/grafana/` is also quite important, but can be recreated. Your first +start will generate a lot of things using grafana provisoning, but things that you later +modify or add using the grafana ui will be lost if you haven't exported them and codified +them into `provisoning`. + ## Running the stack To deploy the stack run the following command: @@ -120,6 +127,7 @@ read *Troubleshooting* Once started, you will see the following applications: * Prometheus - http://localhost:9090 * Grafana - http://localhost:3000 - default creds can be see in `./config/grafana/grafana.ini` +* Service discovery - http://localhost:8000 After 5-10 minutes you should see targets discovered in prometheus on the [targets page](http://localhost:9090/targets?search=). Initially, they might apear in red and if you keep monitoring they should slowly start getting blue, which means that the targets are successfuly scraped. @@ -148,7 +156,7 @@ the other ones that contain `severity: warning` with `severity: critical`. To access the stack remotely you can do the following: ```bash -ssh -L 3000:localhost:3000 -L 9090:localhost:9090 +ssh -L 3000:localhost:3000 -L 9090:localhost:9090 -L 8000:localhost:8000 ``` Example command with all parameters: @@ -157,11 +165,119 @@ ssh -L 3000:localhost:3000 -L 9090:localhost:9090 -i ~/.ssh/priv_key.pem myuser@ ``` ## Extending + +Extending this stack usually means adding new alerts or dashboards. Those might +be your own modifications or the ones that come from someone else. + ### Dashboards + +Building grafana dashboards is usually done via [grafana ui](http://localhost:3000/dashboard/new?orgId=1&from=now-6h&to=now&timezone=browser). You can follow +[this tutorial](https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/create-dashboard/) from grafana to make your custom dashboards. + +After creating the dashboard it is suggested to export it and save it +`./config/grafana/provisioning/dashboards/`. This will make sure that +if you later need to restore the dashboard or do a fresh deployment +of grafana it will be persisted. + +*NOTE*: Storing a dashboard and restarting can make lead to errors +due to overlapping dashboard `uids`. If you export the dashboard and +save it you should, be sure to do a clean deployment of grafana. + ### Alerts +Adding grafana alerts can also be done via [grafana ui](https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-grafana-managed-rule/). + +Similarly to dashboards, it is suggested to export alerts and save them +in `./config/grafana/provisioning/alerting/` which will make sure that +they persist after full grafana redeployments. + ## Troubleshooting +### Service discovery failing + +It is possible that usually due to networking issues, service discovery component +may fail. To debug why you can run the following command to inspect the logs: +```bash +# From the same folder of this README +docker compose logs multiservice-discovery +``` + +You may see something along the lines of: +```bash +multiservice-discovery-1 | Nov 10 13:25:07.044 WARN Failed to sync registry for mercury @ interval Instant { tv_sec: 9050, tv_nsec: 707832535 }: SyncWithNnsFailed { failures: [("targets", RegistryTransportError { source: UnknownError("Failed to query get_certified_changes_since on canister rwlgt-iiaaa-aaaaa-aaaaa-cai: Request failed for http://[2606:fb40:201:1001:6801:2fff:fef5:b129]:8080/api/v2/canister/rwlgt-iiaaa-aaaaa-aaaaa-cai/query: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [2606:fb40:201:1001:6801:2fff:fef5:b129]:8080, Os { code: 101, kind: NetworkUnreachable, message: \"Network is unreachable\" }))") })] } +``` + +This is usually a transient error and may happen from time to time. What this +means is that the discovery cannot sync with the registry canister and may be +serving stale targets. Usually this is acceptable but if that happens when +deploying and the initial sync fails you may be unable to see any of your +nodes. As long as you can see your nodes on the following link you can safely +ignore the transient failures. +```bash +curl http://localhost:8000/prom/targets?node_provider_id=&dc_id= +``` + +*NOTE*: The initial sync of service discovery may take up to 15 minutes! Syncing +will be clearly logged in the multiservice discovery. + +### Prometheus + +#### No targets visible in targets view + +If you don't see anything in the [prometheus targets view](http://localhost:9090/targets?search=), +that means that prometheus failed to receive targets from the service discovery. + +To check the logs run: +```bash +# From the same folder of this README +docker compose logs prometheus +``` + +Check if you can see your nodes by running the following command: +```bash +curl http://localhost:8000/prom/targets?node_provider_id=&dc_id= +``` + +#### Targets visible but are being shown in read + +You should now see 4 jobs: +* `host_node_exporter` +* `node_exporter` +* `orchestrator` +* `replica` + +If any of them are shown in read it means that some of the targets (or all of them) +are failing to be scraped. You can see that from the logs as well: +```bash +# From the same folder of this README +docker compose logs prometheus +``` + +This means that the prometheus scraper cannot reach the nodes it is trying +to scrape. It can be because the workstation for this observability stack +isn't in the same network subnet as the nodes, or due to other network +issues. + +### Stack restart + +To make a full clean restart (or partial) you can do the following: +* Ensure that everything you created through grafana ui is exported + * Dashboards (save them into `./config/grafana/provisioning/dashboards/`) + * Alerts (save them into `./config/grafana/provisioning/alerting/`) + * Contact points (save them into `./config/grafana/provisioning/alerting/`) + * Message templates (save them into `./config/grafana/provisioning/alerting/`) + * Notification policies (save them into `./config/grafana/provisioning/alerting/`) +* If you haven't created the resources, or don't mind losing them + you can proceed. +* Stop the stack: `docker compose -f ./docker-compose.yaml down` +* Clean the volumes. You don't have to clean everything, pick just + ones that you wish to restart fully: + * prometheus: `sudo rm -rf ./volumes/prometheus/` + * grafana: `sudo rm -rf ./volumes/grafana/` + * multiservice discovery: `sudo rm -rf ./volumes/msd/` +* Reset the folder structure: `git checkout -- ./volumes/` +* Change permissions: + * prometheus: `sudo chmod 777 -R ./volumes/prometheus/` + * grafana: `sudo chmod 777 -R ./volumes/grafana/` +* Run the stack again: `docker compose -f ./docker-compose.yaml up -d` -- sudo chmod 777 -R ./volumes/grafana/ -- sudo chmod 777 -R ./volumes/prometheus/ From 54355e57a51251d9130a79d352fcbe2583a076bd Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Mon, 10 Nov 2025 15:02:11 +0100 Subject: [PATCH 22/25] fixing default port --- docker/tools/prom-config-builder/prom_config_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/tools/prom-config-builder/prom_config_builder.py b/docker/tools/prom-config-builder/prom_config_builder.py index 3e190ff..a7b7ecd 100644 --- a/docker/tools/prom-config-builder/prom_config_builder.py +++ b/docker/tools/prom-config-builder/prom_config_builder.py @@ -41,7 +41,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--sd-url", dest="sd_url", - default="multiservice-discovery", + default="multiservice-discovery:8000", help="Override the host for multiservice-discovery container. If running in network_mode: host it will be localhost", ) From 9791a147d73c470f7bede0491d1b6be885165231 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Mon, 10 Nov 2025 15:07:35 +0100 Subject: [PATCH 23/25] network mode host by default --- docker/docker-compose.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 004efaa..29a0d74 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -3,6 +3,7 @@ services: multiservice-discovery: image: ghcr.io/dfinity/dre/multiservice-discovery:cac3154c7868faf9acad14f8585d37df27169573 + network_mode: host command: - --targets-dir - /msd @@ -10,11 +11,10 @@ services: - /msd/definitions.json volumes: - ./volumes/msd:/msd - ports: - - 8000:8000 prometheus: image: quay.io/prometheus/prometheus:v2.51.1 + network_mode: host volumes: - ./config/prometheus:/config - ./volumes/prometheus/:/prometheus @@ -23,8 +23,6 @@ services: - /config/config.yaml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=6m - ports: - - 9090:9090 deploy: resources: limits: @@ -34,10 +32,9 @@ services: grafana: image: grafana/grafana:12.2.1 + network_mode: host environment: - PROM_HOST: prometheus + PROM_HOST: localhost volumes: - ./config/grafana:/etc/grafana - ./volumes/grafana:/var/lib/grafana - ports: - - 3000:3000 From 225d5da79b1a3b98ef08a69fa55aa903c497b2b5 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Mon, 10 Nov 2025 15:09:15 +0100 Subject: [PATCH 24/25] changing the defaults in the script --- docker/tools/prom-config-builder/prom_config_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/tools/prom-config-builder/prom_config_builder.py b/docker/tools/prom-config-builder/prom_config_builder.py index a7b7ecd..8108efa 100644 --- a/docker/tools/prom-config-builder/prom_config_builder.py +++ b/docker/tools/prom-config-builder/prom_config_builder.py @@ -41,8 +41,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--sd-url", dest="sd_url", - default="multiservice-discovery:8000", - help="Override the host for multiservice-discovery container. If running in network_mode: host it will be localhost", + default="localhost:8000", + help="Override the host for multiservice-discovery container. If not running in network_mode: host it will be multiservice-discovery:8000", ) return parser.parse_args() From 82bcd558ad85a4ff1ce32242bcfa507122f8af15 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Mon, 10 Nov 2025 16:51:37 +0100 Subject: [PATCH 25/25] removing the need for chmod --- docker/README.md | 27 +++---------------- .../templates/template_contact_points.yaml | 3 +-- docker/docker-compose.yaml | 2 ++ 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/docker/README.md b/docker/README.md index 5e5a21d..c99b8e7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -91,24 +91,6 @@ After that, edit the new file on path `./config/grafana/provisioning/alerting/co and uncomment your prefered contact point and configure it (to _uncomment_ you should remove the initial `#` from the contact point definition. -### Volumes - -Since prometheus and grafana have their own users which probably differ from yours -you need to make the volumes readable and writiable by those users. There are number -of ways how you can do this, but the easiest is to make it readable and writable -for all users on the machine: -```bash -sudo chmod 777 -R ./volumes/grafana/ -sudo chmod 777 -R ./volumes/prometheus/ -``` - -*NOTE*: `./volumes/prometheus/` is your database and you should take special care not to -delete it, because if you do, you will lose all historical data that has been stored. -*NOTE*: `./volumes/grafana/` is also quite important, but can be recreated. Your first -start will generate a lot of things using grafana provisoning, but things that you later -modify or add using the grafana ui will be lost if you haven't exported them and codified -them into `provisoning`. - ## Running the stack To deploy the stack run the following command: @@ -272,12 +254,9 @@ To make a full clean restart (or partial) you can do the following: * Stop the stack: `docker compose -f ./docker-compose.yaml down` * Clean the volumes. You don't have to clean everything, pick just ones that you wish to restart fully: - * prometheus: `sudo rm -rf ./volumes/prometheus/` - * grafana: `sudo rm -rf ./volumes/grafana/` - * multiservice discovery: `sudo rm -rf ./volumes/msd/` + * prometheus: `rm -rf ./volumes/prometheus/` + * grafana: `rm -rf ./volumes/grafana/` + * multiservice discovery: `rm -rf ./volumes/msd/` * Reset the folder structure: `git checkout -- ./volumes/` -* Change permissions: - * prometheus: `sudo chmod 777 -R ./volumes/prometheus/` - * grafana: `sudo chmod 777 -R ./volumes/grafana/` * Run the stack again: `docker compose -f ./docker-compose.yaml up -d` diff --git a/docker/config/grafana/templates/template_contact_points.yaml b/docker/config/grafana/templates/template_contact_points.yaml index 6dc5f6d..7a8fb52 100644 --- a/docker/config/grafana/templates/template_contact_points.yaml +++ b/docker/config/grafana/templates/template_contact_points.yaml @@ -5,9 +5,8 @@ apiVersion: 1 # contact point. Once provisioned, contact points are stored in the database and can't be # deleted. To delete them, you have to: # 1. stop grafana (docker compose down) -# 2. sudo rm -rf ./volumes/grafana +# 2. rm -rf ./volumes/grafana # 3. git checkout -- ./volumes/grafana -# 4. sudo chmod -R 777 ./volumes/grafana # 5. make changes to the contact_points.yaml (the one in ./config/grafana/provisioning/alerting) # 6. docker compose up -d diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 29a0d74..910997c 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -23,6 +23,7 @@ services: - /config/config.yaml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=6m + user: "${UID}:${GID}" deploy: resources: limits: @@ -38,3 +39,4 @@ services: volumes: - ./config/grafana:/etc/grafana - ./volumes/grafana:/var/lib/grafana + user: "${UID}:${GID}"