Use Grafana Alloy to replace individual exporters

.devcontainer/devcontainer.json

@@ -24,5 +24,12 @@
 	// "remoteUser": "root"
 	"workspaceFolder": "${localWorkspaceFolder}",
 	"workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind",
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"Grafana.grafana-alloy"
+			]
+		}
+	},
 
 }

.github/workflows/observability-docker.yml

@@ -29,6 +29,10 @@ jobs:
             context: observability/traefik
             dockerfile: observability/traefik/Dockerfile
             image: cogstacksystems/cogstack-observability-traefik
+          - name: alloy
+            context: observability/grafana-alloy
+            dockerfile: observability/grafana-alloy/Dockerfile
+            image: cogstacksystems/cogstack-observability-alloy
     steps:
       - name: Checkout branch
         uses: actions/checkout@v4

docs/observability/customization/custom-prometheus-configs.md

@@ -1,4 +1,33 @@
 # Custom Prometheus Configuration
+
+Prometheus can be fully customized, still using the provided defaults. To do this you can mount files in the right directories in docker. 
+
+## Custom Prometheus Exporters
+
+Grafana Alloy is used by default in the stack to get metrics for telemetry. If desired, it is also possible to get metrics by having prometheus scrape APIs
+
+To do this, add the host and IPs into a yaml file in `prometheus/scrape-configs/exporters/` to tell prometheus where to scrape.
+
+Mount these files under `site/prometheus/scrape-configs/exporters/*.yml` in docker
+
+
+### Add Exporters to Prometheus
+- `prometheus/scrape-configs/exporters/` -
+ Add yaml files into this folder. These file should contain all exporter prometheus metrics, for example from node_exporter or CAdvisor. Add any hosts and ip addresses you want to collect /metrics from will be retrieved
+
+```yaml 
+# Exporter example yml
+- targets:
+    - 123.0.0.1:9100 # Enter your IP address and port of a target
+  labels:
+    job: node_exporter # Mandatory  - Enter the type of metric being collected
+    host: my-host-name  # (Optional) A readable hostname
+    custom_label: a_custom_label # (Optional)
+    # __metrics_path__: /path/metrics  # Optionally override the metrics path, the default is just /metrics
+# ... add all targets
+```
+## Custom Prometheus Scrape Configs
+
 You can add compeltely custom prometheus scrape configs and recording rules by mounting in docker.
 
 - `site/prometheus/scrape-configs/*.yml`. This is for advanced configuration. 
@@ -14,4 +43,4 @@ scrape_configs:
           - my-custom-target:9114
         labels:
           custom_label: custom # (Optional)
-```
+```

docs/observability/setup/production-setup.md

@@ -1,4 +1,5 @@
 # Production Setup Tutorial
+
 This tutorial guides you through setting up the **CogStack Observability Stack** for production use.
 
 If you're new, we recommend completing the [Quickstart Tutorial](../get-started/quickstart.md) first to get a simplified setup running.
@@ -14,12 +15,13 @@ Your project configuration should follow this structure:
 ```
 observability.docker-compose.yml
 exporters.docker-compose.yml
+alloy/
+    probers/           # HTTP endpoints to check availability
+        blackbox-exporter/     # (Optional) Custom Probe configuration
 prometheus/
     scrape-configs/
         exporters/         # Targets that expose metrics (e.g. Elasticsearch, Docker, VMs)
-        probers/           # HTTP endpoints to check availability
     recording-rules/       # Prometheus recording rules (e.g. for SLOs, summaries)
-    blackbox-exporter/     # (Optional) Custom Probe configuration
 grafana/                   # (Optional) Custom Grafana dashboards and config
 ```
 
@@ -40,13 +42,14 @@ Downloads the example docker compose files:
 - [exporters.docker-compose.yml](../../../observability/examples/full/exporters.docker-compose.yml)
 - [exporters.elastic.docker-compose.yml](../../../observability/examples/full/exporters.elastic.docker-compose.yml)
 
-Downloads the prometheus configurations:
+Downloads the configurations:
+- [alloy/probers/probe-external.yml](../../../observability/examples/full/alloy/probers/probe-external.yml)
+- [alloy/probers/probe-internal.yml ](../../../observability/examples/full/alloy/probers/probe-internal.yml)
 - [prometheus/scrape-configs/exporters/exporters.yml](../../../observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml)
-- [prometheus/scrape-configs/probers/probe-external.ymll](../../../observability/examples/full/prometheus/scrape-configs/probers/probe-external.yml)
-- [prometheus/scrape-configs/probers/probe-internal.yml ](../../../observability/examples/full/prometheus/scrape-configs/probers/probe-internal.yml)
 - [prometheus/scrape-configs/recording-rules/slo.yml](../../../observability/examples/full/prometheus/scrape-configs/recording-rules/slo.yml)
 
 
+
 Inspect the results in your local directory, and see that it matches the folder layout defined in step 1. 
 
 ## Step 3: Run the Stack

docs/observability/setup/telemetry.md

@@ -4,32 +4,15 @@ We can get telemetry from our services and VMs displayed in our dashboards. This
 
 Using telemetry lets us get feedback from the stack, diagnose problems, and predict issues before they occur.
 
-## Prometheus Exporters
-Prometheus gets metrics from "Exporters". These need to be run on each VM you want to get metrics from
-
-Run these exporters on each virtual machine.
+Grafana Alloy is used to get telemetry. These features are configured by default in this project inside the created image
 
 - Node Exporter: This gives host metrics eg disk usage, memory
 - Elastic Search Exporter: Get ES metrics like index size
 - CAdvisor: This gives docker metrics, eg what containers are running
 
- Then add the host and IPs into a yaml file in `scrape-configs/exporters/` to tell prometheus where to scrape.
-
-
-## How to run Exporters
 
-## Add Exporters to Prometheus
-- `scrape-configs/exporters/` -
- Add yaml files into this folder. These file should contain all exporter prometheus metrics, for example from node_exporter or CAdvisor. Add any hosts and ip addresses you want to collect /metrics from will be retrieved
+## How to get Telemetry
 
-```yaml 
-# Exporter example yml
-- targets:
-    - 123.0.0.1:9100 # Enter your IP address and port of a target
-  labels:
-    job: node_exporter # Mandatory  - Enter the type of metric being collected
-    host: my-host-name  # (Optional) A readable hostname
-    custom_label: a_custom_label # (Optional)
-    # __metrics_path__: /path/metrics  # Optionally override the metrics path, the default is just /metrics
-# ... add all targets
-```
+- Copy this docker compose file: (exporters.docker-compose.yml)[observability/examples/full/exporters.docker-compose.yml]
+- Edit the environment variables to point to your prometheus URL
+- Run `docker compose -f exporters.docker-compose.yml up -d ` on every VM you want metrics from

observability/build-docker.sh

@@ -5,4 +5,6 @@ docker build -t cogstacksystems/cogstack-observability-blackbox-exporter:latest
 
 docker build -t cogstacksystems/cogstack-observability-grafana:latest -f grafana/Dockerfile ./grafana
 
-docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik
+docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik
+
+docker build -t cogstacksystems/cogstack-observability-alloy:latest -f grafana-alloy/Dockerfile ./grafana-alloy --debug

observability/examples/full/alloy/elasticsearch.alloy

@@ -0,0 +1,29 @@
+
+prometheus.exporter.elasticsearch "elasticsearch" {
+    address = sys.env("ELASTICSEARCH_URL")
+    basic_auth {
+        username = sys.env("ELASTICSEARCH_USERNAME")
+        password =  sys.env("ELASTICSEARCH_PASSWORD")
+    }
+    ssl_skip_verify = true
+
+}
+discovery.relabel "elasticsearch" {
+    targets = prometheus.exporter.elasticsearch.elasticsearch.targets
+
+    rule {
+        target_label    = "cluster"
+        replacement     = "elasticsearch-cogstack-cluster"
+    }
+    rule {
+        target_label    = "host"
+        replacement     = "elasticsearch_host"
+    }
+}
+
+prometheus.scrape "elasticsearch_exporter" {
+    scrape_interval = "15s"
+    targets    =  discovery.relabel.elasticsearch.output
+    forward_to = [prometheus.remote_write.default.receiver]
+
+}

observability/examples/full/alloy/probers/probe-internal.yml

@@ -0,0 +1,6 @@
+# Example of probe targets in a different file. 
+- targets:
+    - https://cogstack.org
+  labels:
+    name: cogstack-homepage
+    job: probe-internal-services

observability/examples/full/docker-compose.yml

@@ -2,6 +2,21 @@
 # Depends on docker-compose.exporters.yml for the network
 name: "cogstack-observability"
 services:
+  alloy:
+    image: cogstacksystems/cogstack-observability-alloy:latest
+    ports:
+      - "12345:12345"
+    volumes:
+      - ${BASE_DIR-.}/alloy/probers:/etc/alloy/probers
+      # CAdvisor
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    environment:
+      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
+      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics
+      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics
   prometheus:
     image: cogstacksystems/cogstack-observability-prometheus:latest
     restart: unless-stopped
@@ -29,11 +44,7 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
-  blackbox-exporter:
-    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
-    restart: unless-stopped
-    networks:
-      - observability
+
 networks:
   observability:
     driver: bridge

observability/examples/full/exporters.docker-compose.yml

@@ -1,39 +1,20 @@
-# Exporter deployment on each VM for exporting metrics.
+# Run Exporters on every VM you want metrics from. Grafana Alloy provides multiple components for this
 name: "cogstack-observability-exporters"
 services:
-  node-exporter:
-    image: prom/node-exporter
-    restart: unless-stopped
-    networks:
-      - observability-exporters
-    ports:
-      - 9100:9100
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.node-exporter.rule=PathPrefix(`/node-exporter`)"
-      - "traefik.http.middlewares.node-exporter-stripprefix.stripprefix.prefixes=/node-exporter"
-      - "traefik.http.routers.node-exporter.middlewares=node-exporter-stripprefix@docker"
-  cadvisor:
-    image: gcr.io/cadvisor/cadvisor:latest
+  alloy:
+    image: cogstacksystems/cogstack-observability-alloy:latest
     volumes:
+      # CAdvisor
       - /:/rootfs:ro
       - /var/run:/var/run:rw
       - /sys:/sys:ro
       - /var/lib/docker/:/var/lib/docker:ro
-    ports:
-      - 9116:8080
+    environment:
+      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
+      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics
+      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics
     networks:
       - observability-exporters
-    command:
-      - '-housekeeping_interval=10s'
-      - '-docker_only=true'
-    security_opt:
-      - no-new-privileges:true
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.cadvisor.rule=PathPrefix(`/cadvisor`)"
-      - "traefik.http.middlewares.cadvisor-stripprefix.stripprefix.prefixes=/cadvisor"
-      - "traefik.http.routers.cadvisor.middlewares=cadvisor-stripprefix@docker"
 
 networks:
   observability-exporters:

observability/examples/full/exporters.elastic.docker-compose.yml

@@ -1,21 +1,26 @@
-# Exporter deployment on each VM for exporting metrics.
+# Run to get elasticsearch metrics into prometheus
 name: "cogstack-observability-exporters"
 services:
-  elasticsearch_exporter:
-    image: quay.io/prometheuscommunity/elasticsearch-exporter:latest
+  alloy:
+    image: cogstacksystems/cogstack-observability-alloy:latest
     restart: unless-stopped
-    command:
-     - '--es.uri=${ES_URL}'
-     - '--es.ssl-skip-verify'
-    ports:
-    - "9114:9114"
+    volumes:
+      - ${BASE_DIR-.}/alloy/elasticsearch.alloy:/etc/alloy/elasticsearch.alloy # Enable Elastic Exporter
+      # CAdvisor
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
     environment:
-      - ES_USERNAME=${ES_USERNAME}
-      - ES_PASSWORD=${ES_PASSWORD}
+      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
+      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics
+      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics
+      - ELASTICSEARCH_URL=${ELASTICSEARCH_URL-https://elassticsearch-1:9200}
+      - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME-user} # Used to get metrics from Elasticsearch
+      - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD-pass} # Used to get metrics from Elasticsearch
     networks:
       - observability-exporters
 
 networks:
   observability-exporters:
-    name: cogstack-observability-exporters_observability-exporters
-    external: true
+    driver: bridge

observability/examples/full/full-quickstart.sh

@@ -19,8 +19,8 @@ download_to docker-compose.yml
 download_to exporters.docker-compose.yml
 download_to exporters.elastic.docker-compose.yml
 
-download_to prometheus/scrape-configs/probers/probe-internal.yml
-download_to prometheus/scrape-configs/probers/probe-external.yml
+download_to alloy/probers/probe-internal.yml
+download_to alloy/probers/probe-external.yml
 download_to prometheus/scrape-configs/exporters/exporters.yml
 download_to prometheus/scrape-configs/recording-rules/slo.yml
 

observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml

@@ -1,3 +1,4 @@
+# Scrape metrics from any targets that are exposing Prometheus formatted data. Default is to call the /metrics API. 
 - targets:
     - cogstack-observability-node-exporter-1:9100
   labels:

observability/examples/simple/docker-compose.yml

@@ -2,11 +2,17 @@
 # Depends on docker-compose.exporters.yml for the network
 name: "cogstack-observability"
 services:
+  alloy:
+    image: cogstacksystems/cogstack-observability-alloy:latest
+    restart: unless-stopped
+    volumes:
+      - ${BASE_DIR-.}/alloy/probers:/etc/alloy/probers
+    networks:
+      - observability
   prometheus:
     image: cogstacksystems/cogstack-observability-prometheus:latest
     restart: unless-stopped
     volumes:
-      - ./prometheus:/etc/prometheus/cogstack/site/
       - prometheus-data:/prometheus
     networks:
       - observability
@@ -29,16 +35,7 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
-  blackbox-exporter:
-    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
-    restart: unless-stopped
-    networks:
-      - observability
-  node-exporter:
-    image: prom/node-exporter
-    restart: unless-stopped
-    networks:
-      - observability
+
 networks:
   observability:
     driver: bridge