diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cc81092..87a4344 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,5 +24,12 @@ // "remoteUser": "root" "workspaceFolder": "${localWorkspaceFolder}", "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind", + "customizations": { + "vscode": { + "extensions": [ + "Grafana.grafana-alloy" + ] + } + }, } diff --git a/.github/workflows/observability-docker.yml b/.github/workflows/observability-docker.yml index 04e8fe4..95927e9 100644 --- a/.github/workflows/observability-docker.yml +++ b/.github/workflows/observability-docker.yml @@ -29,6 +29,10 @@ jobs: context: observability/traefik dockerfile: observability/traefik/Dockerfile image: cogstacksystems/cogstack-observability-traefik + - name: alloy + context: observability/grafana-alloy + dockerfile: observability/grafana-alloy/Dockerfile + image: cogstacksystems/cogstack-observability-alloy steps: - name: Checkout branch uses: actions/checkout@v4 diff --git a/docs/observability/customization/custom-prometheus-configs.md b/docs/observability/customization/custom-prometheus-configs.md index 5ec75c2..6f2c34f 100644 --- a/docs/observability/customization/custom-prometheus-configs.md +++ b/docs/observability/customization/custom-prometheus-configs.md @@ -1,4 +1,33 @@ # Custom Prometheus Configuration + +Prometheus can be fully customized, still using the provided defaults. To do this you can mount files in the right directories in docker. + +## Custom Prometheus Exporters + +Grafana Alloy is used by default in the stack to get metrics for telemetry. If desired, it is also possible to get metrics by having prometheus scrape APIs + +To do this, add the host and IPs into a yaml file in `prometheus/scrape-configs/exporters/` to tell prometheus where to scrape. + +Mount these files under `site/prometheus/scrape-configs/exporters/*.yml` in docker + + +### Add Exporters to Prometheus +- `prometheus/scrape-configs/exporters/` - + Add yaml files into this folder. These file should contain all exporter prometheus metrics, for example from node_exporter or CAdvisor. Add any hosts and ip addresses you want to collect /metrics from will be retrieved + +```yaml +# Exporter example yml +- targets: + - 123.0.0.1:9100 # Enter your IP address and port of a target + labels: + job: node_exporter # Mandatory - Enter the type of metric being collected + host: my-host-name # (Optional) A readable hostname + custom_label: a_custom_label # (Optional) + # __metrics_path__: /path/metrics # Optionally override the metrics path, the default is just /metrics +# ... add all targets +``` +## Custom Prometheus Scrape Configs + You can add compeltely custom prometheus scrape configs and recording rules by mounting in docker. - `site/prometheus/scrape-configs/*.yml`. This is for advanced configuration. @@ -14,4 +43,4 @@ scrape_configs: - my-custom-target:9114 labels: custom_label: custom # (Optional) -``` \ No newline at end of file +``` diff --git a/docs/observability/setup/production-setup.md b/docs/observability/setup/production-setup.md index 6448f3f..2c01ac2 100644 --- a/docs/observability/setup/production-setup.md +++ b/docs/observability/setup/production-setup.md @@ -1,4 +1,5 @@ # Production Setup Tutorial + This tutorial guides you through setting up the **CogStack Observability Stack** for production use. If you're new, we recommend completing the [Quickstart Tutorial](../get-started/quickstart.md) first to get a simplified setup running. @@ -14,12 +15,13 @@ Your project configuration should follow this structure: ``` observability.docker-compose.yml exporters.docker-compose.yml +alloy/ + probers/ # HTTP endpoints to check availability + blackbox-exporter/ # (Optional) Custom Probe configuration prometheus/ scrape-configs/ exporters/ # Targets that expose metrics (e.g. Elasticsearch, Docker, VMs) - probers/ # HTTP endpoints to check availability recording-rules/ # Prometheus recording rules (e.g. for SLOs, summaries) - blackbox-exporter/ # (Optional) Custom Probe configuration grafana/ # (Optional) Custom Grafana dashboards and config ``` @@ -40,13 +42,14 @@ Downloads the example docker compose files: - [exporters.docker-compose.yml](../../../observability/examples/full/exporters.docker-compose.yml) - [exporters.elastic.docker-compose.yml](../../../observability/examples/full/exporters.elastic.docker-compose.yml) -Downloads the prometheus configurations: +Downloads the configurations: +- [alloy/probers/probe-external.yml](../../../observability/examples/full/alloy/probers/probe-external.yml) +- [alloy/probers/probe-internal.yml ](../../../observability/examples/full/alloy/probers/probe-internal.yml) - [prometheus/scrape-configs/exporters/exporters.yml](../../../observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml) -- [prometheus/scrape-configs/probers/probe-external.ymll](../../../observability/examples/full/prometheus/scrape-configs/probers/probe-external.yml) -- [prometheus/scrape-configs/probers/probe-internal.yml ](../../../observability/examples/full/prometheus/scrape-configs/probers/probe-internal.yml) - [prometheus/scrape-configs/recording-rules/slo.yml](../../../observability/examples/full/prometheus/scrape-configs/recording-rules/slo.yml) + Inspect the results in your local directory, and see that it matches the folder layout defined in step 1. ## Step 3: Run the Stack diff --git a/docs/observability/setup/telemetry.md b/docs/observability/setup/telemetry.md index e282c22..acbdabf 100644 --- a/docs/observability/setup/telemetry.md +++ b/docs/observability/setup/telemetry.md @@ -4,32 +4,15 @@ We can get telemetry from our services and VMs displayed in our dashboards. This Using telemetry lets us get feedback from the stack, diagnose problems, and predict issues before they occur. -## Prometheus Exporters -Prometheus gets metrics from "Exporters". These need to be run on each VM you want to get metrics from - -Run these exporters on each virtual machine. +Grafana Alloy is used to get telemetry. These features are configured by default in this project inside the created image - Node Exporter: This gives host metrics eg disk usage, memory - Elastic Search Exporter: Get ES metrics like index size - CAdvisor: This gives docker metrics, eg what containers are running - Then add the host and IPs into a yaml file in `scrape-configs/exporters/` to tell prometheus where to scrape. - - -## How to run Exporters -## Add Exporters to Prometheus -- `scrape-configs/exporters/` - - Add yaml files into this folder. These file should contain all exporter prometheus metrics, for example from node_exporter or CAdvisor. Add any hosts and ip addresses you want to collect /metrics from will be retrieved +## How to get Telemetry -```yaml -# Exporter example yml -- targets: - - 123.0.0.1:9100 # Enter your IP address and port of a target - labels: - job: node_exporter # Mandatory - Enter the type of metric being collected - host: my-host-name # (Optional) A readable hostname - custom_label: a_custom_label # (Optional) - # __metrics_path__: /path/metrics # Optionally override the metrics path, the default is just /metrics -# ... add all targets -``` \ No newline at end of file +- Copy this docker compose file: (exporters.docker-compose.yml)[observability/examples/full/exporters.docker-compose.yml] +- Edit the environment variables to point to your prometheus URL +- Run `docker compose -f exporters.docker-compose.yml up -d ` on every VM you want metrics from diff --git a/observability/build-docker.sh b/observability/build-docker.sh index d3521cc..a8db297 100644 --- a/observability/build-docker.sh +++ b/observability/build-docker.sh @@ -5,4 +5,6 @@ docker build -t cogstacksystems/cogstack-observability-blackbox-exporter:latest docker build -t cogstacksystems/cogstack-observability-grafana:latest -f grafana/Dockerfile ./grafana -docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik \ No newline at end of file +docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik + +docker build -t cogstacksystems/cogstack-observability-alloy:latest -f grafana-alloy/Dockerfile ./grafana-alloy --debug \ No newline at end of file diff --git a/observability/examples/full/alloy/elasticsearch.alloy b/observability/examples/full/alloy/elasticsearch.alloy new file mode 100644 index 0000000..0084479 --- /dev/null +++ b/observability/examples/full/alloy/elasticsearch.alloy @@ -0,0 +1,29 @@ + +prometheus.exporter.elasticsearch "elasticsearch" { + address = sys.env("ELASTICSEARCH_URL") + basic_auth { + username = sys.env("ELASTICSEARCH_USERNAME") + password = sys.env("ELASTICSEARCH_PASSWORD") + } + ssl_skip_verify = true + +} +discovery.relabel "elasticsearch" { + targets = prometheus.exporter.elasticsearch.elasticsearch.targets + + rule { + target_label = "cluster" + replacement = "elasticsearch-cogstack-cluster" + } + rule { + target_label = "host" + replacement = "elasticsearch_host" + } +} + +prometheus.scrape "elasticsearch_exporter" { + scrape_interval = "15s" + targets = discovery.relabel.elasticsearch.output + forward_to = [prometheus.remote_write.default.receiver] + +} diff --git a/observability/examples/full/prometheus/scrape-configs/probers/probe-external.yml b/observability/examples/full/alloy/probers/probe-external.yml similarity index 100% rename from observability/examples/full/prometheus/scrape-configs/probers/probe-external.yml rename to observability/examples/full/alloy/probers/probe-external.yml diff --git a/observability/examples/full/alloy/probers/probe-internal.yml b/observability/examples/full/alloy/probers/probe-internal.yml new file mode 100644 index 0000000..fa16233 --- /dev/null +++ b/observability/examples/full/alloy/probers/probe-internal.yml @@ -0,0 +1,6 @@ +# Example of probe targets in a different file. +- targets: + - https://cogstack.org + labels: + name: cogstack-homepage + job: probe-internal-services \ No newline at end of file diff --git a/observability/examples/full/docker-compose.yml b/observability/examples/full/docker-compose.yml index ab994fa..87b6a8a 100755 --- a/observability/examples/full/docker-compose.yml +++ b/observability/examples/full/docker-compose.yml @@ -2,6 +2,21 @@ # Depends on docker-compose.exporters.yml for the network name: "cogstack-observability" services: + alloy: + image: cogstacksystems/cogstack-observability-alloy:latest + ports: + - "12345:12345" + volumes: + - ${BASE_DIR-.}/alloy/probers:/etc/alloy/probers + # CAdvisor + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + environment: + - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus + - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics + - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics prometheus: image: cogstacksystems/cogstack-observability-prometheus:latest restart: unless-stopped @@ -29,11 +44,7 @@ services: - "80:80" volumes: - /var/run/docker.sock:/var/run/docker.sock:ro # So that Traefik can listen to the Docker events - blackbox-exporter: - image: cogstacksystems/cogstack-observability-blackbox-exporter:latest - restart: unless-stopped - networks: - - observability + networks: observability: driver: bridge diff --git a/observability/examples/full/exporters.docker-compose.yml b/observability/examples/full/exporters.docker-compose.yml index 0cae3bf..a16f194 100755 --- a/observability/examples/full/exporters.docker-compose.yml +++ b/observability/examples/full/exporters.docker-compose.yml @@ -1,39 +1,20 @@ -# Exporter deployment on each VM for exporting metrics. +# Run Exporters on every VM you want metrics from. Grafana Alloy provides multiple components for this name: "cogstack-observability-exporters" services: - node-exporter: - image: prom/node-exporter - restart: unless-stopped - networks: - - observability-exporters - ports: - - 9100:9100 - labels: - - "traefik.enable=true" - - "traefik.http.routers.node-exporter.rule=PathPrefix(`/node-exporter`)" - - "traefik.http.middlewares.node-exporter-stripprefix.stripprefix.prefixes=/node-exporter" - - "traefik.http.routers.node-exporter.middlewares=node-exporter-stripprefix@docker" - cadvisor: - image: gcr.io/cadvisor/cadvisor:latest + alloy: + image: cogstacksystems/cogstack-observability-alloy:latest volumes: + # CAdvisor - /:/rootfs:ro - /var/run:/var/run:rw - /sys:/sys:ro - /var/lib/docker/:/var/lib/docker:ro - ports: - - 9116:8080 + environment: + - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus + - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics + - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics networks: - observability-exporters - command: - - '-housekeeping_interval=10s' - - '-docker_only=true' - security_opt: - - no-new-privileges:true - labels: - - "traefik.enable=true" - - "traefik.http.routers.cadvisor.rule=PathPrefix(`/cadvisor`)" - - "traefik.http.middlewares.cadvisor-stripprefix.stripprefix.prefixes=/cadvisor" - - "traefik.http.routers.cadvisor.middlewares=cadvisor-stripprefix@docker" networks: observability-exporters: diff --git a/observability/examples/full/exporters.elastic.docker-compose.yml b/observability/examples/full/exporters.elastic.docker-compose.yml index 3a02bd5..0b5c69f 100644 --- a/observability/examples/full/exporters.elastic.docker-compose.yml +++ b/observability/examples/full/exporters.elastic.docker-compose.yml @@ -1,21 +1,26 @@ -# Exporter deployment on each VM for exporting metrics. +# Run to get elasticsearch metrics into prometheus name: "cogstack-observability-exporters" services: - elasticsearch_exporter: - image: quay.io/prometheuscommunity/elasticsearch-exporter:latest + alloy: + image: cogstacksystems/cogstack-observability-alloy:latest restart: unless-stopped - command: - - '--es.uri=${ES_URL}' - - '--es.ssl-skip-verify' - ports: - - "9114:9114" + volumes: + - ${BASE_DIR-.}/alloy/elasticsearch.alloy:/etc/alloy/elasticsearch.alloy # Enable Elastic Exporter + # CAdvisor + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro environment: - - ES_USERNAME=${ES_USERNAME} - - ES_PASSWORD=${ES_PASSWORD} + - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus + - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics + - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics + - ELASTICSEARCH_URL=${ELASTICSEARCH_URL-https://elassticsearch-1:9200} + - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME-user} # Used to get metrics from Elasticsearch + - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD-pass} # Used to get metrics from Elasticsearch networks: - observability-exporters networks: observability-exporters: - name: cogstack-observability-exporters_observability-exporters - external: true \ No newline at end of file + driver: bridge diff --git a/observability/examples/full/full-quickstart.sh b/observability/examples/full/full-quickstart.sh index 3f77c9a..c811f3d 100644 --- a/observability/examples/full/full-quickstart.sh +++ b/observability/examples/full/full-quickstart.sh @@ -19,8 +19,8 @@ download_to docker-compose.yml download_to exporters.docker-compose.yml download_to exporters.elastic.docker-compose.yml -download_to prometheus/scrape-configs/probers/probe-internal.yml -download_to prometheus/scrape-configs/probers/probe-external.yml +download_to alloy/probers/probe-internal.yml +download_to alloy/probers/probe-external.yml download_to prometheus/scrape-configs/exporters/exporters.yml download_to prometheus/scrape-configs/recording-rules/slo.yml diff --git a/observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml b/observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml index 8036d08..004fc0e 100644 --- a/observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml +++ b/observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml @@ -1,3 +1,4 @@ +# Scrape metrics from any targets that are exposing Prometheus formatted data. Default is to call the /metrics API. - targets: - cogstack-observability-node-exporter-1:9100 labels: diff --git a/observability/examples/full/prometheus/scrape-configs/probers/probe-internal.yml b/observability/examples/full/prometheus/scrape-configs/probers/probe-internal.yml deleted file mode 100644 index b3d7353..0000000 --- a/observability/examples/full/prometheus/scrape-configs/probers/probe-internal.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Example of probe targets -- targets: - - https://cogstack.org - labels: - name: cogstack-homepage - job: probe-services \ No newline at end of file diff --git a/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml b/observability/examples/simple/alloy/probers/probe-simple.yml similarity index 100% rename from observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml rename to observability/examples/simple/alloy/probers/probe-simple.yml diff --git a/observability/examples/simple/docker-compose.yml b/observability/examples/simple/docker-compose.yml index 7f2f8d1..3f22573 100755 --- a/observability/examples/simple/docker-compose.yml +++ b/observability/examples/simple/docker-compose.yml @@ -2,11 +2,17 @@ # Depends on docker-compose.exporters.yml for the network name: "cogstack-observability" services: + alloy: + image: cogstacksystems/cogstack-observability-alloy:latest + restart: unless-stopped + volumes: + - ${BASE_DIR-.}/alloy/probers:/etc/alloy/probers + networks: + - observability prometheus: image: cogstacksystems/cogstack-observability-prometheus:latest restart: unless-stopped volumes: - - ./prometheus:/etc/prometheus/cogstack/site/ - prometheus-data:/prometheus networks: - observability @@ -29,16 +35,7 @@ services: - "80:80" volumes: - /var/run/docker.sock:/var/run/docker.sock:ro # So that Traefik can listen to the Docker events - blackbox-exporter: - image: cogstacksystems/cogstack-observability-blackbox-exporter:latest - restart: unless-stopped - networks: - - observability - node-exporter: - image: prom/node-exporter - restart: unless-stopped - networks: - - observability + networks: observability: driver: bridge diff --git a/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml b/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml deleted file mode 100644 index 8036d08..0000000 --- a/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml +++ /dev/null @@ -1,5 +0,0 @@ -- targets: - - cogstack-observability-node-exporter-1:9100 - labels: - job: node_exporter - host: localhost \ No newline at end of file diff --git a/observability/examples/simple/quickstart.sh b/observability/examples/simple/quickstart.sh index a252b82..ecc7569 100644 --- a/observability/examples/simple/quickstart.sh +++ b/observability/examples/simple/quickstart.sh @@ -1,21 +1,16 @@ #!/bin/bash set -e -mkdir -p observability-simple/prometheus/scrape-configs/probers -mkdir -p observability-simple/prometheus/scrape-configs/exporters +mkdir -p observability-simple/alloy/probers cd observability-simple echo "Downloading docker-compose.yml..." curl -fsSL -o docker-compose.yml \ https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/docker-compose.yml -echo "Downloading probe-simple.yml into prometheus/scrape-configs/probers/..." -curl -fsSL -o prometheus/scrape-configs/probers/probe-simple.yml \ - https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml - -echo "Downloading exporters-simple.yml into prometheus/scrape-configs/exporters/..." -curl -fsSL -o prometheus/scrape-configs/exporters/exporters-simple.yml \ - https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/refs/heads/main/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml +echo "Downloading probe-simple.yml into alloy/probers/..." +curl -fsSL -o probers/probe-simple.yml \ + https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/probers/probe-simple.yml echo "Setup complete in observability-simple/" diff --git a/observability/grafana-alloy/Dockerfile b/observability/grafana-alloy/Dockerfile new file mode 100644 index 0000000..affb334 --- /dev/null +++ b/observability/grafana-alloy/Dockerfile @@ -0,0 +1,17 @@ +FROM grafana/alloy:latest + +LABEL traefik.enable="true" \ + traefik.http.routers.alloy.rule="PathPrefix(`/alloy`)" + +COPY ./defaults /etc/alloy + +CMD [ \ + "run", \ + "--server.http.listen-addr=0.0.0.0:12345", \ + "--storage.path=/var/lib/alloy/data", \ + "--server.http.ui-path-prefix=/alloy", \ + "/etc/alloy" \ + ] + +ENV PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus + diff --git a/observability/grafana-alloy/defaults/blackbox-exporter.yml b/observability/grafana-alloy/defaults/blackbox-exporter.yml new file mode 100644 index 0000000..58bb84e --- /dev/null +++ b/observability/grafana-alloy/defaults/blackbox-exporter.yml @@ -0,0 +1,24 @@ +modules: + http_get_200: + prober: http + timeout: 5s + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + valid_status_codes: [200] # Defaults to 2xx + method: GET + preferred_ip_protocol: "ip4" # defaults to "ip6" + tls_config: + insecure_skip_verify: true + http_admin_get_200: + prober: http + timeout: 5s + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + valid_status_codes: [200] # Defaults to 2xx + method: GET + preferred_ip_protocol: "ip4" # defaults to "ip6" + tls_config: + insecure_skip_verify: true + basic_auth: + username: admin + password: admin diff --git a/observability/grafana-alloy/defaults/config.alloy b/observability/grafana-alloy/defaults/config.alloy new file mode 100644 index 0000000..34f8c7a --- /dev/null +++ b/observability/grafana-alloy/defaults/config.alloy @@ -0,0 +1,38 @@ +logging { + level = "info" + format = "logfmt" +} + +prometheus.remote_write "default" { + endpoint { + url = sys.env("PROMETHEUS_URL") + "/api/v1/write" + } + external_labels = { + host = sys.env("ALLOY_HOSTNAME"), + ip_address = sys.env("ALLOY_IP_ADDRESS"), + } +} + +prometheus.scrape "exporter" { + scrape_interval = "15s" + targets = array.concat( + prometheus.exporter.self.alloy.targets, + prometheus.exporter.cadvisor.local_cadvisor.targets, + prometheus.exporter.unix.local_node_exporter.targets, + ) + forward_to = [prometheus.remote_write.default.receiver] +} + +// Alloys internal metrics +prometheus.exporter.self "alloy" { +} + +// CAdvisor +prometheus.exporter.cadvisor "local_cadvisor" { + docker_host = "unix:///var/run/docker.sock" + storage_duration = "5m" +} + +// Node exporter +prometheus.exporter.unix "local_node_exporter" { +} diff --git a/observability/grafana-alloy/defaults/probe-default-config.alloy b/observability/grafana-alloy/defaults/probe-default-config.alloy new file mode 100644 index 0000000..8c55d47 --- /dev/null +++ b/observability/grafana-alloy/defaults/probe-default-config.alloy @@ -0,0 +1,114 @@ + +// Blackbox Exporter - Probe Observability Stack +prometheus.exporter.blackbox "probe_observability_stack" { + config_file = "/etc/alloy/blackbox-exporter.yml" + target { + name = "grafana" + address = "cogstack-observability-traefik-1/grafana/api/health" + module = "http_get_200" + labels = { + "name" = "grafana", + "host" = "localhost", + } + } + + target { + name = "prometheus" + address = "cogstack-observability-traefik-1/prometheus/-/healthy" + module = "http_get_200" + labels = { + "name" = "prometheus", + "host" = "localhost", + } + } +} +discovery.relabel "probe_observability_stack_results" { + targets = prometheus.exporter.blackbox.probe_observability_stack.targets + + rule { + source_labels = ["__param_target"] + target_label = "instance" + regex = "([^/]+)/(.*)" + replacement = "prometheus-host/$2" + } + + rule { + target_label = "job" + replacement = "probe-observability-stack" + } + rule { + target_label = "host" + replacement = "prometheus-host" + } +} + +// Blackbox Exporter - Probe External Services +discovery.file "probe_target_files" { + files = ["/etc/alloy/probers/*.yml"] + refresh_interval = "5m" +} + +discovery.relabel "probe_target_files_with_defaults" { + targets = discovery.file.probe_target_files.targets + + rule { + source_labels = ["job"] // Alloy overrides the job field, this works around it. + target_label = "group" + } + rule { + // Add default module to all calls + source_labels = ["module"] + target_label = "module" + regex = "" + replacement = "http_get_200" + } + rule { + // Alloy takes the "name" field from the yml file, puts it into the job field as a suffix, then removes name. This brings name back. + source_labels = ["name"] + target_label = "service_name" + } +} + +prometheus.exporter.blackbox "probe_discovered_targets" { + config_file = "/etc/alloy/blackbox-exporter.yml" + targets = discovery.relabel.probe_target_files_with_defaults.output +} + + +discovery.relabel "probe_discovered_targets_results" { + targets = prometheus.exporter.blackbox.probe_discovered_targets.targets + + rule { + source_labels = ["__param_target"] + target_label = "instance" + } + rule { + // Bring name back + source_labels = ["service_name"] + target_label = "name" + } + rule { + // Bring job back + source_labels = ["group"] + target_label = "job" + } + + rule { + regex = "group" + action = "labeldrop" + } + rule { + regex = "service_name" + action = "labeldrop" + } +} + +// Scrape Probe targets +prometheus.scrape "blackbox_exporter" { + scrape_interval = "15s" + targets = array.concat( + discovery.relabel.probe_discovered_targets_results.output, + discovery.relabel.probe_observability_stack_results.output, + ) + forward_to = [prometheus.remote_write.default.receiver] +} \ No newline at end of file diff --git a/observability/grafana/Dockerfile b/observability/grafana/Dockerfile index 0dc0bc5..bef90d9 100644 --- a/observability/grafana/Dockerfile +++ b/observability/grafana/Dockerfile @@ -17,4 +17,4 @@ ENV GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboar ALERTING_PAUSE_BURN_RATE=true \ ALERTING_DEFAULT_CONTACT=Slack \ SLACK_WEBHOOK_URL=http://localhost:3000/missing-SLACK_WEBHOOK_URL-env-var \ - PROMETHEUS_DATASOURCE_URL=http://cogstack-observability-prometheus-1:9090/prometheus \ No newline at end of file + PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus \ No newline at end of file diff --git a/observability/grafana/provisioning/datasources/default.yml b/observability/grafana/provisioning/datasources/default.yml index 06f9927..80b52cb 100644 --- a/observability/grafana/provisioning/datasources/default.yml +++ b/observability/grafana/provisioning/datasources/default.yml @@ -24,7 +24,7 @@ datasources: uid: datasource-prometheus-local-docker # Sets the data source's URL, including the # port. - url: ${PROMETHEUS_DATASOURCE_URL} + url: ${PROMETHEUS_URL} isDefault: true # Allows users to edit data sources from the # Grafana UI. diff --git a/observability/prometheus/defaults/scrape-config.local.yml b/observability/prometheus/defaults/scrape-config.local.yml deleted file mode 100644 index 379e0bf..0000000 --- a/observability/prometheus/defaults/scrape-config.local.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Scrape configs for running local NiFi Stack -scrape_configs: - - job_name: elasticsearch-exporter # Scrape configuration to get metrics from elasticsearch, eg index size. - static_configs: - - targets: - - cogstack-observability-elasticsearch-exporter:9114 - labels: - cluster: elasticsearch-cogstack-cluster - - job_name: "probe-local-stack" # Scrape configuration for using prometheus to probe locally hosted apps, eg for development. - metrics_path: /probe - params: - module: [http_get_200] - static_configs: - - targets: - - https://cogstack-nifi-nginx:8443/nifi-api/access - - https://cogstack-nifi:8443/nifi-api/access - labels: - name: cogstack-nifi - - targets: - - cogstack-medcat-trainer-nginx:8000/admin/login - - cogstack-medcat-trainer-ui:8000/admin/login - labels: - name: cogstack-medcat-trainer-ui - - targets: - - cogstack-medcat-service-production:5000/api/info - labels: - name: cogstack-medcat-service - - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: cogstack-observability-blackbox-exporter-1:9115 # The blackbox exporter's real hostname:port. - - - job_name: "blackbox-elastic" - metrics_path: /probe - params: - module: [http_elastic_get_200] - static_configs: - - targets: - - https://elasticsearch-1:9200/_cat/health - labels: - name: elasticsearch-1 - - targets: - - https://elasticsearch-2:9200/_cat/health - labels: - name: elasticsearch-2 - - targets: - - https://cogstack-kibana:5601/app/kibana - labels: - name: kibana - relabel_configs: - - target_label: env - replacement: local - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: cogstack-observability-blackbox-exporter-1:9115 # The blackbox exporter's real hostname:port. diff --git a/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml b/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml index 94feb92..2e39c79 100644 --- a/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml +++ b/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml @@ -12,11 +12,6 @@ scrape_configs: labels: name: grafana # collect grafana metrics like dashboard usage __metrics_path__: /metrics - - targets: - - cogstack-observability-traefik-1 # Collect blackbox-exporter metrics like number of probes - labels: - name: blackbox-exporter - __metrics_path__: /blackbox-exporter/metrics - targets: - cogstack-observability-traefik-1:8080 # Collect Traefik metrics like number of success calls labels: @@ -25,33 +20,4 @@ scrape_configs: - target_label: env replacement: observability-stack - target_label: host - replacement: prometheus-host - - job_name: "probe-observability-stack" # Probe if the observability services are running - metrics_path: /blackbox-exporter/probe - params: - module: [http_get_200] - static_configs: - - targets: - - cogstack-observability-traefik-1/grafana/api/health - labels: - name: grafana - - targets: - - cogstack-observability-traefik-1/prometheus/-/healthy - labels: - name: prometheus - relabel_configs: - - source_labels: [__address__] - regex: ([0-9.]+):[0-9]+ - target_label: ip_address - - target_label: env - replacement: observability-stack - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__address__] - regex: ([^/]+)/(.*) - target_label: instance - replacement: prometheus-host/$2 - - target_label: host - replacement: prometheus-host - - target_label: __address__ - replacement: cogstack-observability-traefik-1 # The blackbox exporter's real host:port. + replacement: prometheus-host \ No newline at end of file