From 5de738e95f2425c5b09b4b81a98843600b6d1308 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Wed, 4 Jun 2025 14:51:19 +0000
Subject: [PATCH 01/11] Run Grafana alloy to replace exporters

---
 observability/build-docker.sh                 |  8 +++++
 .../examples/full/docker-compose.yml          | 26 ++++++++++++++
 .../examples/full/grafana-alloy/config.alloy  | 36 +++++++++++++++++++
 .../prometheus/Dockerfile.prometheus          |  3 +-
 4 files changed, 72 insertions(+), 1 deletion(-)
 create mode 100644 observability/build-docker.sh
 create mode 100644 observability/examples/full/grafana-alloy/config.alloy

diff --git a/observability/build-docker.sh b/observability/build-docker.sh
new file mode 100644
index 0000000..d3521cc
--- /dev/null
+++ b/observability/build-docker.sh
@@ -0,0 +1,8 @@
+
+docker build -t cogstacksystems/cogstack-observability-prometheus:latest -f prometheus/Dockerfile.prometheus ./prometheus
+
+docker build -t cogstacksystems/cogstack-observability-blackbox-exporter:latest -f prometheus/Dockerfile.blackbox ./prometheus
+
+docker build -t cogstacksystems/cogstack-observability-grafana:latest -f grafana/Dockerfile ./grafana
+
+docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik
\ No newline at end of file
diff --git a/observability/examples/full/docker-compose.yml b/observability/examples/full/docker-compose.yml
index 8fb9362..79322b0 100755
--- a/observability/examples/full/docker-compose.yml
+++ b/observability/examples/full/docker-compose.yml
@@ -5,6 +5,8 @@ services:
   prometheus:
     image: cogstacksystems/cogstack-observability-prometheus:latest
     restart: unless-stopped
+    ports:
+      - "9090:9090"
     volumes:
       - ./prometheus:/etc/prometheus/cogstack/site/
       - prometheus-data:/prometheus
@@ -29,6 +31,30 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
+  alloy:
+    image: grafana/alloy:latest
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - --server.http.ui-path-prefix=/alloy
+      - /etc/alloy/config.alloy
+    ports:
+      - "12345:12345"
+    volumes:
+      - ./grafana-alloy/config.alloy:/etc/alloy/config.alloy
+      # CAdvisor
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.cadvisor.rule=PathPrefix(`/alloy`)"
+    environment:
+      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write
+    networks:
+      - observability
 networks:
   observability:
     driver: bridge
diff --git a/observability/examples/full/grafana-alloy/config.alloy b/observability/examples/full/grafana-alloy/config.alloy
new file mode 100644
index 0000000..d3af4a2
--- /dev/null
+++ b/observability/examples/full/grafana-alloy/config.alloy
@@ -0,0 +1,36 @@
+logging {
+  level  = "debug"
+  format = "logfmt"
+}
+
+prometheus.remote_write "default" {
+  endpoint {
+    url = sys.env("PROMETHEUS_URL")
+  }
+}
+
+prometheus.scrape "exporter" {
+
+    scrape_interval = "15s"
+
+    targets    = array.concat(
+        prometheus.exporter.self.alloy.targets,
+        prometheus.exporter.cadvisor.local_cadvisor.targets,
+        prometheus.exporter.unix.local_node_exporter.targets,
+    )
+    forward_to = [prometheus.remote_write.default.receiver]
+}
+
+// Alloys internal metrics
+prometheus.exporter.self "alloy" {
+}
+
+// CAdvisor
+prometheus.exporter.cadvisor "local_cadvisor" {
+  docker_host = "unix:///var/run/docker.sock"
+  storage_duration = "5m"
+}
+
+// Node exporter
+prometheus.exporter.unix "local_node_exporter" {
+}
diff --git a/observability/prometheus/Dockerfile.prometheus b/observability/prometheus/Dockerfile.prometheus
index 0db9a47..a743fcb 100644
--- a/observability/prometheus/Dockerfile.prometheus
+++ b/observability/prometheus/Dockerfile.prometheus
@@ -10,5 +10,6 @@ CMD [ \
     "--storage.tsdb.path=/prometheus", \
     "--storage.tsdb.retention.time=30d", \
     "--web.external-url=/prometheus", \
-    "--web.route-prefix=/prometheus" \
+    "--web.route-prefix=/prometheus", \
+    "--web.enable-remote-write-receiver" \
     ]
\ No newline at end of file

From 4584769aff82252f97500502b9b488bf83322cdd Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Wed, 4 Jun 2025 15:11:44 +0000
Subject: [PATCH 02/11] Make sample folder using alloy

---
 .../examples/alloy/docker-compose.yml         | 71 +++++++++++++++++++
 .../grafana-alloy/config.alloy                |  0
 .../examples/full/docker-compose.yml          | 26 -------
 3 files changed, 71 insertions(+), 26 deletions(-)
 create mode 100755 observability/examples/alloy/docker-compose.yml
 rename observability/examples/{full => alloy}/grafana-alloy/config.alloy (100%)

diff --git a/observability/examples/alloy/docker-compose.yml b/observability/examples/alloy/docker-compose.yml
new file mode 100755
index 0000000..c2088b1
--- /dev/null
+++ b/observability/examples/alloy/docker-compose.yml
@@ -0,0 +1,71 @@
+# Observability main stack. Prometheus and Grafana. 
+# Depends on docker-compose.exporters.yml for the network
+name: "cogstack-observability"
+services:
+  prometheus:
+    image: cogstacksystems/cogstack-observability-prometheus:latest
+    restart: unless-stopped
+    ports:
+      - "9090:9090"
+    volumes:
+      - ${BASE_PATH-.}/prometheus:/etc/prometheus/cogstack/site/
+      - prometheus-data:/prometheus
+    networks:
+      - observability
+    command: 
+    - "--config.file=/etc/prometheus/cogstack/defaults/prometheus.yml"
+    - "--storage.tsdb.path=/prometheus"
+    - "--storage.tsdb.retention.time=30d"
+    - "--web.external-url=/prometheus"
+    - "--web.route-prefix=/prometheus"
+    - "--web.enable-remote-write-receiver"
+  grafana:
+    image: cogstacksystems/cogstack-observability-grafana:latest
+    restart: unless-stopped
+    volumes:
+      - grafana-data:/var/lib/grafana
+    networks:
+      - observability
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true # Allows use of grafana without sign in 
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
+  traefik:
+    image: cogstacksystems/cogstack-observability-traefik:latest
+    networks:
+     - observability
+    restart: unless-stopped
+    ports:
+      - "80:80"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
+  alloy:
+    image: grafana/alloy:latest
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - --server.http.ui-path-prefix=/alloy
+      - /etc/alloy/config.alloy
+    ports:
+      - "12345:12345"
+    volumes:
+      - ${BASE_PATH-.}/grafana-alloy/config.alloy:/etc/alloy/config.alloy
+      # CAdvisor
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.cadvisor.rule=PathPrefix(`/alloy`)"
+    environment:
+      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write
+    networks:
+      - observability
+networks:
+  observability:
+    driver: bridge
+
+volumes:
+  prometheus-data:
+  grafana-data:
\ No newline at end of file
diff --git a/observability/examples/full/grafana-alloy/config.alloy b/observability/examples/alloy/grafana-alloy/config.alloy
similarity index 100%
rename from observability/examples/full/grafana-alloy/config.alloy
rename to observability/examples/alloy/grafana-alloy/config.alloy
diff --git a/observability/examples/full/docker-compose.yml b/observability/examples/full/docker-compose.yml
index 79322b0..8fb9362 100755
--- a/observability/examples/full/docker-compose.yml
+++ b/observability/examples/full/docker-compose.yml
@@ -5,8 +5,6 @@ services:
   prometheus:
     image: cogstacksystems/cogstack-observability-prometheus:latest
     restart: unless-stopped
-    ports:
-      - "9090:9090"
     volumes:
       - ./prometheus:/etc/prometheus/cogstack/site/
       - prometheus-data:/prometheus
@@ -31,30 +29,6 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
-  alloy:
-    image: grafana/alloy:latest
-    command:
-      - run
-      - --server.http.listen-addr=0.0.0.0:12345
-      - --storage.path=/var/lib/alloy/data
-      - --server.http.ui-path-prefix=/alloy
-      - /etc/alloy/config.alloy
-    ports:
-      - "12345:12345"
-    volumes:
-      - ./grafana-alloy/config.alloy:/etc/alloy/config.alloy
-      # CAdvisor
-      - /:/rootfs:ro
-      - /var/run:/var/run:rw
-      - /sys:/sys:ro
-      - /var/lib/docker/:/var/lib/docker:ro
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.cadvisor.rule=PathPrefix(`/alloy`)"
-    environment:
-      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write
-    networks:
-      - observability
 networks:
   observability:
     driver: bridge

From c352673b9352d3be8dfcb840bdfc199ffa4e48cc Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Wed, 4 Jun 2025 15:52:32 +0000
Subject: [PATCH 03/11] Add default labels to alloy scraped metrics

---
 observability/examples/alloy/docker-compose.yml     | 13 ++++++++++---
 .../examples/alloy/grafana-alloy/config.alloy       |  6 ++++--
 .../scrape-configs/probers/probe-internal.yml       |  6 ++++++
 .../scrape-configs/recording-rules/slo.yml          |  8 ++++++++
 observability/examples/full/docker-compose.yml      |  5 +++++
 5 files changed, 33 insertions(+), 5 deletions(-)
 create mode 100644 observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml
 create mode 100644 observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml

diff --git a/observability/examples/alloy/docker-compose.yml b/observability/examples/alloy/docker-compose.yml
index c2088b1..a2f53c6 100755
--- a/observability/examples/alloy/docker-compose.yml
+++ b/observability/examples/alloy/docker-compose.yml
@@ -8,7 +8,7 @@ services:
     ports:
       - "9090:9090"
     volumes:
-      - ${BASE_PATH-.}/prometheus:/etc/prometheus/cogstack/site/
+      - ${BASE_DIR-.}/prometheus:/etc/prometheus/cogstack/site/
       - prometheus-data:/prometheus
     networks:
       - observability
@@ -38,6 +38,11 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
+  blackbox-exporter:
+    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
+    restart: unless-stopped
+    networks:
+      - observability
   alloy:
     image: grafana/alloy:latest
     command:
@@ -49,7 +54,7 @@ services:
     ports:
       - "12345:12345"
     volumes:
-      - ${BASE_PATH-.}/grafana-alloy/config.alloy:/etc/alloy/config.alloy
+      - ${BASE_DIR-.}/grafana-alloy/config.alloy:/etc/alloy/config.alloy
       # CAdvisor
       - /:/rootfs:ro
       - /var/run:/var/run:rw
@@ -59,7 +64,9 @@ services:
       - "traefik.enable=true"
       - "traefik.http.routers.cadvisor.rule=PathPrefix(`/alloy`)"
     environment:
-      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write
+      - PROMETHEUS_URL=${PROMETHEUS_URL-http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write}
+      - ALLOY_HOSTNAME=my-custom-host
+      - ALLOY_IP_ADDRESS=my-custom-ip
     networks:
       - observability
 networks:
diff --git a/observability/examples/alloy/grafana-alloy/config.alloy b/observability/examples/alloy/grafana-alloy/config.alloy
index d3af4a2..eaa2c5c 100644
--- a/observability/examples/alloy/grafana-alloy/config.alloy
+++ b/observability/examples/alloy/grafana-alloy/config.alloy
@@ -7,12 +7,14 @@ prometheus.remote_write "default" {
   endpoint {
     url = sys.env("PROMETHEUS_URL")
   }
+  external_labels = {
+    host = sys.env("ALLOY_HOSTNAME"),
+    ip_address = sys.env("ALLOY_IP_ADDRESS"),
+  }
 }
 
 prometheus.scrape "exporter" {
-
     scrape_interval = "15s"
-
     targets    = array.concat(
         prometheus.exporter.self.alloy.targets,
         prometheus.exporter.cadvisor.local_cadvisor.targets,
diff --git a/observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml b/observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml
new file mode 100644
index 0000000..b3d7353
--- /dev/null
+++ b/observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml
@@ -0,0 +1,6 @@
+# Example of probe targets
+- targets:
+    - https://cogstack.org
+  labels:
+    name: cogstack-homepage
+    job: probe-services
\ No newline at end of file
diff --git a/observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml b/observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml
new file mode 100644
index 0000000..440913c
--- /dev/null
+++ b/observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml
@@ -0,0 +1,8 @@
+groups:
+  - name:  slo-target-rules
+    rules:
+      # What SLO am I targeting
+      - record: slo_target_over_30_days
+        expr: 0.95 # We target 95% uptime over 30 days
+        labels:
+          job: "probe-external-demo-apps" #Job here must match the job in the probe targets
\ No newline at end of file
diff --git a/observability/examples/full/docker-compose.yml b/observability/examples/full/docker-compose.yml
index 8fb9362..ab994fa 100755
--- a/observability/examples/full/docker-compose.yml
+++ b/observability/examples/full/docker-compose.yml
@@ -29,6 +29,11 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
+  blackbox-exporter:
+    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
+    restart: unless-stopped
+    networks:
+      - observability
 networks:
   observability:
     driver: bridge

From bfe035cfc2bf39a9f8b6287fc8c802699c46aa6f Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Wed, 4 Jun 2025 16:34:51 +0000
Subject: [PATCH 04/11] Add stub for making alloy the default

---
 .../examples/alloy/docker-compose.yml         |  8 ++--
 observability/grafana/Dockerfile.alloy        |  1 +
 .../grafana/alloy/default-config.alloy        | 38 +++++++++++++++++++
 3 files changed, 43 insertions(+), 4 deletions(-)
 create mode 100644 observability/grafana/Dockerfile.alloy
 create mode 100644 observability/grafana/alloy/default-config.alloy

diff --git a/observability/examples/alloy/docker-compose.yml b/observability/examples/alloy/docker-compose.yml
index a2f53c6..4795443 100755
--- a/observability/examples/alloy/docker-compose.yml
+++ b/observability/examples/alloy/docker-compose.yml
@@ -50,7 +50,7 @@ services:
       - --server.http.listen-addr=0.0.0.0:12345
       - --storage.path=/var/lib/alloy/data
       - --server.http.ui-path-prefix=/alloy
-      - /etc/alloy/config.alloy
+      - /etc/alloy
     ports:
       - "12345:12345"
     volumes:
@@ -62,11 +62,11 @@ services:
       - /var/lib/docker/:/var/lib/docker:ro
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.cadvisor.rule=PathPrefix(`/alloy`)"
+      - "traefik.http.routers.alloy.rule=PathPrefix(`/alloy`)"
     environment:
       - PROMETHEUS_URL=${PROMETHEUS_URL-http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write}
-      - ALLOY_HOSTNAME=my-custom-host
-      - ALLOY_IP_ADDRESS=my-custom-ip
+      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME} # Used to add a label to metrics
+      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS} # Used to add a label to metrics
     networks:
       - observability
 networks:
diff --git a/observability/grafana/Dockerfile.alloy b/observability/grafana/Dockerfile.alloy
new file mode 100644
index 0000000..f87f5c1
--- /dev/null
+++ b/observability/grafana/Dockerfile.alloy
@@ -0,0 +1 @@
+# TODO
\ No newline at end of file
diff --git a/observability/grafana/alloy/default-config.alloy b/observability/grafana/alloy/default-config.alloy
new file mode 100644
index 0000000..eaa2c5c
--- /dev/null
+++ b/observability/grafana/alloy/default-config.alloy
@@ -0,0 +1,38 @@
+logging {
+  level  = "debug"
+  format = "logfmt"
+}
+
+prometheus.remote_write "default" {
+  endpoint {
+    url = sys.env("PROMETHEUS_URL")
+  }
+  external_labels = {
+    host = sys.env("ALLOY_HOSTNAME"),
+    ip_address = sys.env("ALLOY_IP_ADDRESS"),
+  }
+}
+
+prometheus.scrape "exporter" {
+    scrape_interval = "15s"
+    targets    = array.concat(
+        prometheus.exporter.self.alloy.targets,
+        prometheus.exporter.cadvisor.local_cadvisor.targets,
+        prometheus.exporter.unix.local_node_exporter.targets,
+    )
+    forward_to = [prometheus.remote_write.default.receiver]
+}
+
+// Alloys internal metrics
+prometheus.exporter.self "alloy" {
+}
+
+// CAdvisor
+prometheus.exporter.cadvisor "local_cadvisor" {
+  docker_host = "unix:///var/run/docker.sock"
+  storage_duration = "5m"
+}
+
+// Node exporter
+prometheus.exporter.unix "local_node_exporter" {
+}

From 12086f2f8664e978ff64d9f0e751317edcfaadf6 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Mon, 9 Jun 2025 12:57:55 +0000
Subject: [PATCH 05/11] Add Elasticsearch Exporter to Alloy

---
 .devcontainer/devcontainer.json               |  7 +++++
 .../examples/alloy/docker-compose.yml         |  3 +-
 .../advanced/elasticsearch.alloy              | 29 +++++++++++++++++++
 .../examples/alloy/grafana-alloy/config.alloy |  6 +++-
 .../alloy/grafana-alloy/elasticsearch.alloy   |  0
 5 files changed, 43 insertions(+), 2 deletions(-)
 create mode 100644 observability/examples/alloy/grafana-alloy/advanced/elasticsearch.alloy
 create mode 100644 observability/examples/alloy/grafana-alloy/elasticsearch.alloy

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index cc81092..87a4344 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -24,5 +24,12 @@
 	// "remoteUser": "root"
 	"workspaceFolder": "${localWorkspaceFolder}",
 	"workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind",
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"Grafana.grafana-alloy"
+			]
+		}
+	},
 
 }
diff --git a/observability/examples/alloy/docker-compose.yml b/observability/examples/alloy/docker-compose.yml
index 4795443..826622d 100755
--- a/observability/examples/alloy/docker-compose.yml
+++ b/observability/examples/alloy/docker-compose.yml
@@ -54,7 +54,8 @@ services:
     ports:
       - "12345:12345"
     volumes:
-      - ${BASE_DIR-.}/grafana-alloy/config.alloy:/etc/alloy/config.alloy
+      - ${BASE_DIR-.}/grafana-alloy:/etc/alloy
+      # - ${BASE_DIR-.}/grafana-alloy/advanced/elasticsearch.alloy:/etc/alloy/elasticsearch.alloy # Enable Elastic Exporter
       # CAdvisor
       - /:/rootfs:ro
       - /var/run:/var/run:rw
diff --git a/observability/examples/alloy/grafana-alloy/advanced/elasticsearch.alloy b/observability/examples/alloy/grafana-alloy/advanced/elasticsearch.alloy
new file mode 100644
index 0000000..9020fb7
--- /dev/null
+++ b/observability/examples/alloy/grafana-alloy/advanced/elasticsearch.alloy
@@ -0,0 +1,29 @@
+
+prometheus.exporter.elasticsearch "elasticsearch" {
+    address = "https://host.docker.internal:9200"
+    basic_auth {
+        username = sys.env("ELASTICSEARCH_USERNAME")
+        password =  sys.env("ELASTICSEARCH_PASSWORD")
+    }
+    ssl_skip_verify = true
+
+}
+discovery.relabel "elasticsearch" {
+    targets = prometheus.exporter.elasticsearch.elasticsearch.targets
+
+    rule {
+        target_label    = "cluster"
+        replacement     = "elasticsearch-cogstack-cluster"
+    }
+    rule {
+        target_label    = "host"
+        replacement     = "elasticsearch_host"
+    }
+}
+
+prometheus.scrape "elasticsearch_exporter" {
+    scrape_interval = "15s"
+    targets    =  discovery.relabel.elasticsearch.output
+    forward_to = [prometheus.remote_write.default.receiver]
+
+}
diff --git a/observability/examples/alloy/grafana-alloy/config.alloy b/observability/examples/alloy/grafana-alloy/config.alloy
index eaa2c5c..faad925 100644
--- a/observability/examples/alloy/grafana-alloy/config.alloy
+++ b/observability/examples/alloy/grafana-alloy/config.alloy
@@ -3,6 +3,10 @@ logging {
   format = "logfmt"
 }
 
+livedebugging {
+  enabled = true
+}
+
 prometheus.remote_write "default" {
   endpoint {
     url = sys.env("PROMETHEUS_URL")
@@ -10,7 +14,7 @@ prometheus.remote_write "default" {
   external_labels = {
     host = sys.env("ALLOY_HOSTNAME"),
     ip_address = sys.env("ALLOY_IP_ADDRESS"),
-  }
+  } 
 }
 
 prometheus.scrape "exporter" {
diff --git a/observability/examples/alloy/grafana-alloy/elasticsearch.alloy b/observability/examples/alloy/grafana-alloy/elasticsearch.alloy
new file mode 100644
index 0000000..e69de29

From f1b45cab69a0e7e5a0c67fbec9890d6a022e0168 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Mon, 9 Jun 2025 15:10:36 +0000
Subject: [PATCH 06/11] Use Alloy for Blackbox Probing

---
 .../examples/alloy/grafana-alloy/blackbox.yml |  24 ++++
 .../alloy/grafana-alloy/elasticsearch.alloy   |   0
 .../grafana-alloy/probe-default-config.alloy  | 114 ++++++++++++++++++
 .../grafana-alloy/probes/probe-external.yml   |  12 ++
 4 files changed, 150 insertions(+)
 create mode 100644 observability/examples/alloy/grafana-alloy/blackbox.yml
 delete mode 100644 observability/examples/alloy/grafana-alloy/elasticsearch.alloy
 create mode 100644 observability/examples/alloy/grafana-alloy/probe-default-config.alloy
 create mode 100644 observability/examples/alloy/grafana-alloy/probes/probe-external.yml

diff --git a/observability/examples/alloy/grafana-alloy/blackbox.yml b/observability/examples/alloy/grafana-alloy/blackbox.yml
new file mode 100644
index 0000000..58bb84e
--- /dev/null
+++ b/observability/examples/alloy/grafana-alloy/blackbox.yml
@@ -0,0 +1,24 @@
+modules:
+  http_get_200:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+      valid_status_codes: [200]  # Defaults to 2xx
+      method: GET
+      preferred_ip_protocol: "ip4" # defaults to "ip6"
+      tls_config:
+        insecure_skip_verify: true
+  http_admin_get_200:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+      valid_status_codes: [200]  # Defaults to 2xx
+      method: GET
+      preferred_ip_protocol: "ip4" # defaults to "ip6"
+      tls_config:
+        insecure_skip_verify: true
+      basic_auth:
+        username: admin
+        password: admin
diff --git a/observability/examples/alloy/grafana-alloy/elasticsearch.alloy b/observability/examples/alloy/grafana-alloy/elasticsearch.alloy
deleted file mode 100644
index e69de29..0000000
diff --git a/observability/examples/alloy/grafana-alloy/probe-default-config.alloy b/observability/examples/alloy/grafana-alloy/probe-default-config.alloy
new file mode 100644
index 0000000..ccd0fef
--- /dev/null
+++ b/observability/examples/alloy/grafana-alloy/probe-default-config.alloy
@@ -0,0 +1,114 @@
+
+// Blackbox Exporter - Probe Observability Stack
+prometheus.exporter.blackbox "probe_observability_stack" {
+  config_file = "/etc/alloy/blackbox.yml"
+  target {
+    name    = "grafana"
+    address = "cogstack-observability-traefik-1/grafana/api/health"
+    module  = "http_get_200"
+    labels = {
+      "name" = "grafana",
+      "host" = "localhost",
+    }
+  }
+
+  target {
+    name    = "prometheus"
+    address = "cogstack-observability-traefik-1/prometheus/-/healthy"
+    module  = "http_get_200"
+    labels = {
+      "name" = "prometheus",
+      "host" = "localhost",
+    }
+  }
+}
+discovery.relabel "probe_observability_stack_results" {
+  targets = prometheus.exporter.blackbox.probe_observability_stack.targets
+
+  rule {
+    source_labels = ["__param_target"]
+    target_label = "instance"
+    regex = "([^/]+)/(.*)"
+    replacement = "prometheus-host/$2"
+  }
+
+  rule {
+    target_label = "job"
+    replacement = "probe-observability-stack"
+  }
+  rule {
+    target_label = "host"
+    replacement = "prometheus-host"
+  }
+}
+
+// Blackbox Exporter - Probe External Services
+discovery.file "probe_target_files" {
+  files = ["/etc/alloy/probes/*.yml"]
+  refresh_interval = "5m"
+}
+
+discovery.relabel "probe_target_files_with_defaults" {
+  targets = discovery.file.probe_target_files.targets
+
+  rule {
+    source_labels = ["job"] // Alloy overrides the job field, this works around it. 
+    target_label = "group"
+  }
+  rule {
+   // Add default module to all calls
+   source_labels =  ["module"]
+   target_label =  "module"
+   regex =  ""
+   replacement =  "http_200_ok"
+  }
+  rule {
+    // Alloy takes the "name" field from the yml file, puts it into the job field as a suffix, then removes name. This brings name back.
+    source_labels = ["name"]
+    target_label = "service_name"
+  }
+}
+
+prometheus.exporter.blackbox "probe_discovered_targets" {
+  config_file = "/etc/alloy/blackbox.yml"
+  targets = discovery.relabel.probe_target_files_with_defaults.output
+}
+
+
+discovery.relabel "probe_discovered_targets_results" {
+  targets = prometheus.exporter.blackbox.probe_discovered_targets.targets
+
+  rule {
+    source_labels = ["__param_target"]
+    target_label = "instance"
+  }
+  rule {
+    // Bring name back
+    source_labels = ["service_name"]
+    target_label = "name"
+  }
+    rule {
+    // Bring job back
+    source_labels = ["group"]
+    target_label = "job"
+  }
+
+  rule {
+    regex   = "group"
+    action  = "labeldrop"
+  }
+  rule {
+    regex   = "service_name"
+    action  = "labeldrop"
+  }
+}
+
+// Scrape Probe targets
+prometheus.scrape "blackbox_exporter" {
+    scrape_interval = "15s"
+    targets    = array.concat(
+        discovery.relabel.probe_discovered_targets_results.output,
+        discovery.relabel.probe_observability_stack_results.output,
+    )
+    forward_to = [prometheus.remote_write.default.receiver]
+}
\ No newline at end of file
diff --git a/observability/examples/alloy/grafana-alloy/probes/probe-external.yml b/observability/examples/alloy/grafana-alloy/probes/probe-external.yml
new file mode 100644
index 0000000..0dc8059
--- /dev/null
+++ b/observability/examples/alloy/grafana-alloy/probes/probe-external.yml
@@ -0,0 +1,12 @@
+# Example of probe targets
+- targets:
+    - https://cogstack.org
+  labels:
+    name: cogstack-homepage
+    job: probe-services-alloy
+- targets:
+    - https://cogstack.org
+  labels:
+    name: admin_probe
+    job: probe-services-alloy-2
+    module: http_admin_get_200
\ No newline at end of file

From a9b78862d7829d434af52c936bf81d9c1e9d341d Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Mon, 9 Jun 2025 16:00:44 +0000
Subject: [PATCH 07/11] Create docker image for Alloy

---
 .github/workflows/observability-docker.yml    |  4 ++
 observability/build-docker.sh                 |  4 +-
 .../examples/alloy/docker-compose.yml         | 35 ++---------
 .../grafana-alloy/probes/probe-external.yml   | 10 +--
 .../scrape-configs/probers/probe-internal.yml |  6 --
 observability/grafana-alloy/Dockerfile        | 16 +++++
 .../defaults/blackbox-exporter.yml}           |  0
 .../defaults}/config.alloy                    |  2 +-
 .../defaults}/probe-default-config.alloy      |  6 +-
 observability/grafana/Dockerfile              |  2 +-
 observability/grafana/Dockerfile.alloy        |  1 -
 .../grafana/alloy/default-config.alloy        | 38 ------------
 .../provisioning/datasources/default.yml      |  2 +-
 .../defaults/scrape-config.local.yml          | 62 -------------------
 .../scrape-configs/scrape-config.self.yml     | 36 +----------
 15 files changed, 38 insertions(+), 186 deletions(-)
 delete mode 100644 observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml
 create mode 100644 observability/grafana-alloy/Dockerfile
 rename observability/{examples/alloy/grafana-alloy/blackbox.yml => grafana-alloy/defaults/blackbox-exporter.yml} (100%)
 rename observability/{examples/alloy/grafana-alloy => grafana-alloy/defaults}/config.alloy (94%)
 rename observability/{examples/alloy/grafana-alloy => grafana-alloy/defaults}/probe-default-config.alloy (95%)
 delete mode 100644 observability/grafana/Dockerfile.alloy
 delete mode 100644 observability/grafana/alloy/default-config.alloy
 delete mode 100644 observability/prometheus/defaults/scrape-config.local.yml

diff --git a/.github/workflows/observability-docker.yml b/.github/workflows/observability-docker.yml
index 04e8fe4..95927e9 100644
--- a/.github/workflows/observability-docker.yml
+++ b/.github/workflows/observability-docker.yml
@@ -29,6 +29,10 @@ jobs:
             context: observability/traefik
             dockerfile: observability/traefik/Dockerfile
             image: cogstacksystems/cogstack-observability-traefik
+          - name: alloy
+            context: observability/grafana-alloy
+            dockerfile: observability/grafana-alloy/Dockerfile
+            image: cogstacksystems/cogstack-observability-alloy
     steps:
       - name: Checkout branch
         uses: actions/checkout@v4
diff --git a/observability/build-docker.sh b/observability/build-docker.sh
index d3521cc..a8db297 100644
--- a/observability/build-docker.sh
+++ b/observability/build-docker.sh
@@ -5,4 +5,6 @@ docker build -t cogstacksystems/cogstack-observability-blackbox-exporter:latest
 
 docker build -t cogstacksystems/cogstack-observability-grafana:latest -f grafana/Dockerfile ./grafana
 
-docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik
\ No newline at end of file
+docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik
+
+docker build -t cogstacksystems/cogstack-observability-alloy:latest -f grafana-alloy/Dockerfile ./grafana-alloy --debug
\ No newline at end of file
diff --git a/observability/examples/alloy/docker-compose.yml b/observability/examples/alloy/docker-compose.yml
index 826622d..5b4ca6b 100755
--- a/observability/examples/alloy/docker-compose.yml
+++ b/observability/examples/alloy/docker-compose.yml
@@ -5,20 +5,11 @@ services:
   prometheus:
     image: cogstacksystems/cogstack-observability-prometheus:latest
     restart: unless-stopped
-    ports:
-      - "9090:9090"
     volumes:
-      - ${BASE_DIR-.}/prometheus:/etc/prometheus/cogstack/site/
+      - ./prometheus:/etc/prometheus/cogstack/site/
       - prometheus-data:/prometheus
     networks:
       - observability
-    command: 
-    - "--config.file=/etc/prometheus/cogstack/defaults/prometheus.yml"
-    - "--storage.tsdb.path=/prometheus"
-    - "--storage.tsdb.retention.time=30d"
-    - "--web.external-url=/prometheus"
-    - "--web.route-prefix=/prometheus"
-    - "--web.enable-remote-write-receiver"
   grafana:
     image: cogstacksystems/cogstack-observability-grafana:latest
     restart: unless-stopped
@@ -38,36 +29,22 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
-  blackbox-exporter:
-    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
-    restart: unless-stopped
-    networks:
-      - observability
   alloy:
-    image: grafana/alloy:latest
-    command:
-      - run
-      - --server.http.listen-addr=0.0.0.0:12345
-      - --storage.path=/var/lib/alloy/data
-      - --server.http.ui-path-prefix=/alloy
-      - /etc/alloy
+    image: cogstacksystems/cogstack-observability-alloy:latest
     ports:
       - "12345:12345"
     volumes:
-      - ${BASE_DIR-.}/grafana-alloy:/etc/alloy
+      - ${BASE_DIR-.}/grafana-alloy/probes:/etc/alloy/probes
       # - ${BASE_DIR-.}/grafana-alloy/advanced/elasticsearch.alloy:/etc/alloy/elasticsearch.alloy # Enable Elastic Exporter
       # CAdvisor
       - /:/rootfs:ro
       - /var/run:/var/run:rw
       - /sys:/sys:ro
       - /var/lib/docker/:/var/lib/docker:ro
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.alloy.rule=PathPrefix(`/alloy`)"
     environment:
-      - PROMETHEUS_URL=${PROMETHEUS_URL-http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write}
-      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME} # Used to add a label to metrics
-      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS} # Used to add a label to metrics
+      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
+      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics
+      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics
     networks:
       - observability
 networks:
diff --git a/observability/examples/alloy/grafana-alloy/probes/probe-external.yml b/observability/examples/alloy/grafana-alloy/probes/probe-external.yml
index 0dc8059..cb61134 100644
--- a/observability/examples/alloy/grafana-alloy/probes/probe-external.yml
+++ b/observability/examples/alloy/grafana-alloy/probes/probe-external.yml
@@ -2,11 +2,5 @@
 - targets:
     - https://cogstack.org
   labels:
-    name: cogstack-homepage
-    job: probe-services-alloy
-- targets:
-    - https://cogstack.org
-  labels:
-    name: admin_probe
-    job: probe-services-alloy-2
-    module: http_admin_get_200
\ No newline at end of file
+    name: homepage
+    job: probe-services
\ No newline at end of file
diff --git a/observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml b/observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml
deleted file mode 100644
index b3d7353..0000000
--- a/observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-# Example of probe targets
-- targets:
-    - https://cogstack.org
-  labels:
-    name: cogstack-homepage
-    job: probe-services
\ No newline at end of file
diff --git a/observability/grafana-alloy/Dockerfile b/observability/grafana-alloy/Dockerfile
new file mode 100644
index 0000000..d431ed1
--- /dev/null
+++ b/observability/grafana-alloy/Dockerfile
@@ -0,0 +1,16 @@
+FROM grafana/alloy:latest
+
+LABEL traefik.enable="true" \
+    traefik.http.routers.alloy.rule="PathPrefix(`/alloy`)" 
+
+COPY ./defaults /etc/alloy
+
+CMD [ \
+    "run", \
+    "--server.http.listen-addr=0.0.0.0:12345", \
+    "--storage.path=/var/lib/alloy/data", \
+    "--server.http.ui-path-prefix=/alloy", \
+    "/etc/alloy" \
+    ] 
+
+ENV PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
diff --git a/observability/examples/alloy/grafana-alloy/blackbox.yml b/observability/grafana-alloy/defaults/blackbox-exporter.yml
similarity index 100%
rename from observability/examples/alloy/grafana-alloy/blackbox.yml
rename to observability/grafana-alloy/defaults/blackbox-exporter.yml
diff --git a/observability/examples/alloy/grafana-alloy/config.alloy b/observability/grafana-alloy/defaults/config.alloy
similarity index 94%
rename from observability/examples/alloy/grafana-alloy/config.alloy
rename to observability/grafana-alloy/defaults/config.alloy
index faad925..41482ff 100644
--- a/observability/examples/alloy/grafana-alloy/config.alloy
+++ b/observability/grafana-alloy/defaults/config.alloy
@@ -9,7 +9,7 @@ livedebugging {
 
 prometheus.remote_write "default" {
   endpoint {
-    url = sys.env("PROMETHEUS_URL")
+    url = sys.env("PROMETHEUS_URL") + "/api/v1/write"
   }
   external_labels = {
     host = sys.env("ALLOY_HOSTNAME"),
diff --git a/observability/examples/alloy/grafana-alloy/probe-default-config.alloy b/observability/grafana-alloy/defaults/probe-default-config.alloy
similarity index 95%
rename from observability/examples/alloy/grafana-alloy/probe-default-config.alloy
rename to observability/grafana-alloy/defaults/probe-default-config.alloy
index ccd0fef..9b75043 100644
--- a/observability/examples/alloy/grafana-alloy/probe-default-config.alloy
+++ b/observability/grafana-alloy/defaults/probe-default-config.alloy
@@ -1,7 +1,7 @@
 
 // Blackbox Exporter - Probe Observability Stack
 prometheus.exporter.blackbox "probe_observability_stack" {
-  config_file = "/etc/alloy/blackbox.yml"
+  config_file = "/etc/alloy/blackbox-exporter.yml"
   target {
     name    = "grafana"
     address = "cogstack-observability-traefik-1/grafana/api/health"
@@ -60,7 +60,7 @@ discovery.relabel "probe_target_files_with_defaults" {
    source_labels =  ["module"]
    target_label =  "module"
    regex =  ""
-   replacement =  "http_200_ok"
+   replacement =  "http_get_200"
   }
   rule {
     // Alloy takes the "name" field from the yml file, puts it into the job field as a suffix, then removes name. This brings name back.
@@ -70,7 +70,7 @@ discovery.relabel "probe_target_files_with_defaults" {
 }
 
 prometheus.exporter.blackbox "probe_discovered_targets" {
-  config_file = "/etc/alloy/blackbox.yml"
+  config_file = "/etc/alloy/blackbox-exporter.yml"
   targets = discovery.relabel.probe_target_files_with_defaults.output
 }
 
diff --git a/observability/grafana/Dockerfile b/observability/grafana/Dockerfile
index 0dc0bc5..bef90d9 100644
--- a/observability/grafana/Dockerfile
+++ b/observability/grafana/Dockerfile
@@ -17,4 +17,4 @@ ENV GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboar
     ALERTING_PAUSE_BURN_RATE=true \
     ALERTING_DEFAULT_CONTACT=Slack \
     SLACK_WEBHOOK_URL=http://localhost:3000/missing-SLACK_WEBHOOK_URL-env-var \
-    PROMETHEUS_DATASOURCE_URL=http://cogstack-observability-prometheus-1:9090/prometheus
\ No newline at end of file
+    PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
\ No newline at end of file
diff --git a/observability/grafana/Dockerfile.alloy b/observability/grafana/Dockerfile.alloy
deleted file mode 100644
index f87f5c1..0000000
--- a/observability/grafana/Dockerfile.alloy
+++ /dev/null
@@ -1 +0,0 @@
-# TODO
\ No newline at end of file
diff --git a/observability/grafana/alloy/default-config.alloy b/observability/grafana/alloy/default-config.alloy
deleted file mode 100644
index eaa2c5c..0000000
--- a/observability/grafana/alloy/default-config.alloy
+++ /dev/null
@@ -1,38 +0,0 @@
-logging {
-  level  = "debug"
-  format = "logfmt"
-}
-
-prometheus.remote_write "default" {
-  endpoint {
-    url = sys.env("PROMETHEUS_URL")
-  }
-  external_labels = {
-    host = sys.env("ALLOY_HOSTNAME"),
-    ip_address = sys.env("ALLOY_IP_ADDRESS"),
-  }
-}
-
-prometheus.scrape "exporter" {
-    scrape_interval = "15s"
-    targets    = array.concat(
-        prometheus.exporter.self.alloy.targets,
-        prometheus.exporter.cadvisor.local_cadvisor.targets,
-        prometheus.exporter.unix.local_node_exporter.targets,
-    )
-    forward_to = [prometheus.remote_write.default.receiver]
-}
-
-// Alloys internal metrics
-prometheus.exporter.self "alloy" {
-}
-
-// CAdvisor
-prometheus.exporter.cadvisor "local_cadvisor" {
-  docker_host = "unix:///var/run/docker.sock"
-  storage_duration = "5m"
-}
-
-// Node exporter
-prometheus.exporter.unix "local_node_exporter" {
-}
diff --git a/observability/grafana/provisioning/datasources/default.yml b/observability/grafana/provisioning/datasources/default.yml
index 06f9927..80b52cb 100644
--- a/observability/grafana/provisioning/datasources/default.yml
+++ b/observability/grafana/provisioning/datasources/default.yml
@@ -24,7 +24,7 @@ datasources:
     uid: datasource-prometheus-local-docker
     # <string> Sets the data source's URL, including the
     # port.
-    url: ${PROMETHEUS_DATASOURCE_URL}
+    url: ${PROMETHEUS_URL}
     isDefault: true
     # <bool> Allows users to edit data sources from the
     # Grafana UI.
diff --git a/observability/prometheus/defaults/scrape-config.local.yml b/observability/prometheus/defaults/scrape-config.local.yml
deleted file mode 100644
index 379e0bf..0000000
--- a/observability/prometheus/defaults/scrape-config.local.yml
+++ /dev/null
@@ -1,62 +0,0 @@
-# Scrape configs for running local NiFi Stack
-scrape_configs:
-  - job_name: elasticsearch-exporter # Scrape configuration to get metrics from elasticsearch, eg index size.
-    static_configs:
-      - targets:
-          - cogstack-observability-elasticsearch-exporter:9114
-        labels:
-          cluster: elasticsearch-cogstack-cluster
-  - job_name: "probe-local-stack" # Scrape configuration for using prometheus to probe locally hosted apps, eg for development.
-    metrics_path: /probe
-    params:
-      module: [http_get_200]
-    static_configs:
-      - targets:
-          - https://cogstack-nifi-nginx:8443/nifi-api/access
-          - https://cogstack-nifi:8443/nifi-api/access
-        labels:
-          name: cogstack-nifi
-      - targets:
-          - cogstack-medcat-trainer-nginx:8000/admin/login
-          - cogstack-medcat-trainer-ui:8000/admin/login
-        labels:
-          name: cogstack-medcat-trainer-ui
-      - targets:
-          - cogstack-medcat-service-production:5000/api/info
-        labels:
-          name: cogstack-medcat-service
-
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: cogstack-observability-blackbox-exporter-1:9115 # The blackbox exporter's real hostname:port.
-
-  - job_name: "blackbox-elastic"
-    metrics_path: /probe
-    params:
-      module: [http_elastic_get_200]
-    static_configs:
-      - targets:
-          - https://elasticsearch-1:9200/_cat/health
-        labels:
-          name: elasticsearch-1
-      - targets:
-          - https://elasticsearch-2:9200/_cat/health
-        labels:
-          name: elasticsearch-2
-      - targets:
-          - https://cogstack-kibana:5601/app/kibana
-        labels:
-          name: kibana
-    relabel_configs:
-      - target_label: env
-        replacement: local
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: cogstack-observability-blackbox-exporter-1:9115 # The blackbox exporter's real hostname:port.
diff --git a/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml b/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml
index 94feb92..2e39c79 100644
--- a/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml
+++ b/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml
@@ -12,11 +12,6 @@ scrape_configs:
         labels:
           name: grafana # collect grafana metrics like dashboard usage
           __metrics_path__: /metrics
-      - targets:
-          - cogstack-observability-traefik-1 # Collect blackbox-exporter metrics like number of probes
-        labels:
-          name: blackbox-exporter
-          __metrics_path__: /blackbox-exporter/metrics
       - targets:
           - cogstack-observability-traefik-1:8080  # Collect Traefik metrics like number of success calls
         labels:
@@ -25,33 +20,4 @@ scrape_configs:
         - target_label: env
           replacement: observability-stack
         - target_label: host
-          replacement: prometheus-host
-  - job_name: "probe-observability-stack" # Probe if the observability services are running
-    metrics_path: /blackbox-exporter/probe
-    params:
-      module: [http_get_200]
-    static_configs:
-      - targets:
-          - cogstack-observability-traefik-1/grafana/api/health
-        labels:
-          name: grafana
-      - targets:
-          - cogstack-observability-traefik-1/prometheus/-/healthy
-        labels:
-          name: prometheus
-    relabel_configs:
-      - source_labels: [__address__]
-        regex: ([0-9.]+):[0-9]+
-        target_label: ip_address
-      - target_label: env
-        replacement: observability-stack
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__address__]
-        regex: ([^/]+)/(.*)
-        target_label: instance
-        replacement: prometheus-host/$2
-      - target_label: host
-        replacement: prometheus-host
-      - target_label: __address__
-        replacement: cogstack-observability-traefik-1 # The blackbox exporter's real host:port.
+          replacement: prometheus-host
\ No newline at end of file

From ebcc0131f633a45193ea8502d02e5556499c9b6d Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Mon, 9 Jun 2025 16:32:21 +0000
Subject: [PATCH 08/11] Update documentation and examples

---
 .../custom-prometheus-configs.md              | 31 +++++++++-
 docs/observability/setup/production-setup.md  |  6 +-
 docs/observability/setup/telemetry.md         | 27 ++-------
 .../examples/alloy/docker-compose.yml         | 56 -------------------
 .../grafana-alloy/probes/probe-external.yml   |  6 --
 .../scrape-configs/recording-rules/slo.yml    |  8 ---
 .../alloy}/elasticsearch.alloy                |  2 +-
 .../examples/full/docker-compose.yml          | 21 +++++--
 .../full/exporters.docker-compose.yml         | 36 ++++--------
 .../full/exporters.elastic.docker-compose.yml | 30 ++++++----
 .../probers/probe-external.yml                |  0
 .../examples/full/probers/probe-internal.yml  |  6 ++
 .../scrape-configs/exporters/exporters.yml    |  1 +
 .../scrape-configs/probers/probe-internal.yml |  6 --
 .../probers/probe-simple.yml                  |  0
 .../examples/simple/docker-compose.yml        | 19 +++----
 .../exporters/exporters-simple.yml            |  5 --
 observability/examples/simple/quickstart.sh   | 13 ++---
 .../grafana-alloy/defaults/config.alloy       |  6 +-
 .../defaults/probe-default-config.alloy       |  2 +-
 20 files changed, 106 insertions(+), 175 deletions(-)
 delete mode 100755 observability/examples/alloy/docker-compose.yml
 delete mode 100644 observability/examples/alloy/grafana-alloy/probes/probe-external.yml
 delete mode 100644 observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml
 rename observability/examples/{alloy/grafana-alloy/advanced => full/alloy}/elasticsearch.alloy (93%)
 rename observability/examples/full/{prometheus/scrape-configs => }/probers/probe-external.yml (100%)
 create mode 100644 observability/examples/full/probers/probe-internal.yml
 delete mode 100644 observability/examples/full/prometheus/scrape-configs/probers/probe-internal.yml
 rename observability/examples/simple/{prometheus/scrape-configs => alloy}/probers/probe-simple.yml (100%)
 delete mode 100644 observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml

diff --git a/docs/observability/customization/custom-prometheus-configs.md b/docs/observability/customization/custom-prometheus-configs.md
index 5ec75c2..6f2c34f 100644
--- a/docs/observability/customization/custom-prometheus-configs.md
+++ b/docs/observability/customization/custom-prometheus-configs.md
@@ -1,4 +1,33 @@
 # Custom Prometheus Configuration
+
+Prometheus can be fully customized, still using the provided defaults. To do this you can mount files in the right directories in docker. 
+
+## Custom Prometheus Exporters
+
+Grafana Alloy is used by default in the stack to get metrics for telemetry. If desired, it is also possible to get metrics by having prometheus scrape APIs
+
+To do this, add the host and IPs into a yaml file in `prometheus/scrape-configs/exporters/` to tell prometheus where to scrape.
+
+Mount these files under `site/prometheus/scrape-configs/exporters/*.yml` in docker
+
+
+### Add Exporters to Prometheus
+- `prometheus/scrape-configs/exporters/` -
+ Add yaml files into this folder. These file should contain all exporter prometheus metrics, for example from node_exporter or CAdvisor. Add any hosts and ip addresses you want to collect /metrics from will be retrieved
+
+```yaml 
+# Exporter example yml
+- targets:
+    - 123.0.0.1:9100 # Enter your IP address and port of a target
+  labels:
+    job: node_exporter # Mandatory  - Enter the type of metric being collected
+    host: my-host-name  # (Optional) A readable hostname
+    custom_label: a_custom_label # (Optional)
+    # __metrics_path__: /path/metrics  # Optionally override the metrics path, the default is just /metrics
+# ... add all targets
+```
+## Custom Prometheus Scrape Configs
+
 You can add compeltely custom prometheus scrape configs and recording rules by mounting in docker.
 
 - `site/prometheus/scrape-configs/*.yml`. This is for advanced configuration. 
@@ -14,4 +43,4 @@ scrape_configs:
           - my-custom-target:9114
         labels:
           custom_label: custom # (Optional)
-```
\ No newline at end of file
+```
diff --git a/docs/observability/setup/production-setup.md b/docs/observability/setup/production-setup.md
index 6448f3f..f0223f0 100644
--- a/docs/observability/setup/production-setup.md
+++ b/docs/observability/setup/production-setup.md
@@ -1,4 +1,5 @@
 # Production Setup Tutorial
+
 This tutorial guides you through setting up the **CogStack Observability Stack** for production use.
 
 If you're new, we recommend completing the [Quickstart Tutorial](../get-started/quickstart.md) first to get a simplified setup running.
@@ -14,12 +15,13 @@ Your project configuration should follow this structure:
 ```
 observability.docker-compose.yml
 exporters.docker-compose.yml
+alloy/
+    probers/           # HTTP endpoints to check availability
+        blackbox-exporter/     # (Optional) Custom Probe configuration
 prometheus/
     scrape-configs/
         exporters/         # Targets that expose metrics (e.g. Elasticsearch, Docker, VMs)
-        probers/           # HTTP endpoints to check availability
     recording-rules/       # Prometheus recording rules (e.g. for SLOs, summaries)
-    blackbox-exporter/     # (Optional) Custom Probe configuration
 grafana/                   # (Optional) Custom Grafana dashboards and config
 ```
 
diff --git a/docs/observability/setup/telemetry.md b/docs/observability/setup/telemetry.md
index e282c22..acbdabf 100644
--- a/docs/observability/setup/telemetry.md
+++ b/docs/observability/setup/telemetry.md
@@ -4,32 +4,15 @@ We can get telemetry from our services and VMs displayed in our dashboards. This
 
 Using telemetry lets us get feedback from the stack, diagnose problems, and predict issues before they occur.
 
-## Prometheus Exporters
-Prometheus gets metrics from "Exporters". These need to be run on each VM you want to get metrics from
-
-Run these exporters on each virtual machine.
+Grafana Alloy is used to get telemetry. These features are configured by default in this project inside the created image
 
 - Node Exporter: This gives host metrics eg disk usage, memory
 - Elastic Search Exporter: Get ES metrics like index size
 - CAdvisor: This gives docker metrics, eg what containers are running
 
- Then add the host and IPs into a yaml file in `scrape-configs/exporters/` to tell prometheus where to scrape.
-
-
-## How to run Exporters
 
-## Add Exporters to Prometheus
-- `scrape-configs/exporters/` -
- Add yaml files into this folder. These file should contain all exporter prometheus metrics, for example from node_exporter or CAdvisor. Add any hosts and ip addresses you want to collect /metrics from will be retrieved
+## How to get Telemetry
 
-```yaml 
-# Exporter example yml
-- targets:
-    - 123.0.0.1:9100 # Enter your IP address and port of a target
-  labels:
-    job: node_exporter # Mandatory  - Enter the type of metric being collected
-    host: my-host-name  # (Optional) A readable hostname
-    custom_label: a_custom_label # (Optional)
-    # __metrics_path__: /path/metrics  # Optionally override the metrics path, the default is just /metrics
-# ... add all targets
-```
\ No newline at end of file
+- Copy this docker compose file: (exporters.docker-compose.yml)[observability/examples/full/exporters.docker-compose.yml]
+- Edit the environment variables to point to your prometheus URL
+- Run `docker compose -f exporters.docker-compose.yml up -d ` on every VM you want metrics from
diff --git a/observability/examples/alloy/docker-compose.yml b/observability/examples/alloy/docker-compose.yml
deleted file mode 100755
index 5b4ca6b..0000000
--- a/observability/examples/alloy/docker-compose.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-# Observability main stack. Prometheus and Grafana. 
-# Depends on docker-compose.exporters.yml for the network
-name: "cogstack-observability"
-services:
-  prometheus:
-    image: cogstacksystems/cogstack-observability-prometheus:latest
-    restart: unless-stopped
-    volumes:
-      - ./prometheus:/etc/prometheus/cogstack/site/
-      - prometheus-data:/prometheus
-    networks:
-      - observability
-  grafana:
-    image: cogstacksystems/cogstack-observability-grafana:latest
-    restart: unless-stopped
-    volumes:
-      - grafana-data:/var/lib/grafana
-    networks:
-      - observability
-    environment:
-      - GF_AUTH_ANONYMOUS_ENABLED=true # Allows use of grafana without sign in 
-      - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
-  traefik:
-    image: cogstacksystems/cogstack-observability-traefik:latest
-    networks:
-     - observability
-    restart: unless-stopped
-    ports:
-      - "80:80"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
-  alloy:
-    image: cogstacksystems/cogstack-observability-alloy:latest
-    ports:
-      - "12345:12345"
-    volumes:
-      - ${BASE_DIR-.}/grafana-alloy/probes:/etc/alloy/probes
-      # - ${BASE_DIR-.}/grafana-alloy/advanced/elasticsearch.alloy:/etc/alloy/elasticsearch.alloy # Enable Elastic Exporter
-      # CAdvisor
-      - /:/rootfs:ro
-      - /var/run:/var/run:rw
-      - /sys:/sys:ro
-      - /var/lib/docker/:/var/lib/docker:ro
-    environment:
-      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
-      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics
-      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics
-    networks:
-      - observability
-networks:
-  observability:
-    driver: bridge
-
-volumes:
-  prometheus-data:
-  grafana-data:
\ No newline at end of file
diff --git a/observability/examples/alloy/grafana-alloy/probes/probe-external.yml b/observability/examples/alloy/grafana-alloy/probes/probe-external.yml
deleted file mode 100644
index cb61134..0000000
--- a/observability/examples/alloy/grafana-alloy/probes/probe-external.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-# Example of probe targets
-- targets:
-    - https://cogstack.org
-  labels:
-    name: homepage
-    job: probe-services
\ No newline at end of file
diff --git a/observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml b/observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml
deleted file mode 100644
index 440913c..0000000
--- a/observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml
+++ /dev/null
@@ -1,8 +0,0 @@
-groups:
-  - name:  slo-target-rules
-    rules:
-      # What SLO am I targeting
-      - record: slo_target_over_30_days
-        expr: 0.95 # We target 95% uptime over 30 days
-        labels:
-          job: "probe-external-demo-apps" #Job here must match the job in the probe targets
\ No newline at end of file
diff --git a/observability/examples/alloy/grafana-alloy/advanced/elasticsearch.alloy b/observability/examples/full/alloy/elasticsearch.alloy
similarity index 93%
rename from observability/examples/alloy/grafana-alloy/advanced/elasticsearch.alloy
rename to observability/examples/full/alloy/elasticsearch.alloy
index 9020fb7..0084479 100644
--- a/observability/examples/alloy/grafana-alloy/advanced/elasticsearch.alloy
+++ b/observability/examples/full/alloy/elasticsearch.alloy
@@ -1,6 +1,6 @@
 
 prometheus.exporter.elasticsearch "elasticsearch" {
-    address = "https://host.docker.internal:9200"
+    address = sys.env("ELASTICSEARCH_URL")
     basic_auth {
         username = sys.env("ELASTICSEARCH_USERNAME")
         password =  sys.env("ELASTICSEARCH_PASSWORD")
diff --git a/observability/examples/full/docker-compose.yml b/observability/examples/full/docker-compose.yml
index ab994fa..16c999f 100755
--- a/observability/examples/full/docker-compose.yml
+++ b/observability/examples/full/docker-compose.yml
@@ -2,6 +2,21 @@
 # Depends on docker-compose.exporters.yml for the network
 name: "cogstack-observability"
 services:
+  alloy:
+    image: cogstacksystems/cogstack-observability-alloy:latest
+    ports:
+      - "12345:12345"
+    volumes:
+      - ${BASE_DIR-.}/probers:/etc/alloy/probers
+      # CAdvisor
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    environment:
+      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
+      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics
+      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics
   prometheus:
     image: cogstacksystems/cogstack-observability-prometheus:latest
     restart: unless-stopped
@@ -29,11 +44,7 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
-  blackbox-exporter:
-    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
-    restart: unless-stopped
-    networks:
-      - observability
+
 networks:
   observability:
     driver: bridge
diff --git a/observability/examples/full/exporters.docker-compose.yml b/observability/examples/full/exporters.docker-compose.yml
index 0cae3bf..67c9aa3 100755
--- a/observability/examples/full/exporters.docker-compose.yml
+++ b/observability/examples/full/exporters.docker-compose.yml
@@ -1,39 +1,23 @@
-# Exporter deployment on each VM for exporting metrics.
+# Run Exporters on every VM you want metrics from. Grafana Alloy provides multiple components for this
 name: "cogstack-observability-exporters"
 services:
-  node-exporter:
-    image: prom/node-exporter
-    restart: unless-stopped
-    networks:
-      - observability-exporters
+  alloy:
+    image: cogstacksystems/cogstack-observability-alloy:latest
     ports:
-      - 9100:9100
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.node-exporter.rule=PathPrefix(`/node-exporter`)"
-      - "traefik.http.middlewares.node-exporter-stripprefix.stripprefix.prefixes=/node-exporter"
-      - "traefik.http.routers.node-exporter.middlewares=node-exporter-stripprefix@docker"
-  cadvisor:
-    image: gcr.io/cadvisor/cadvisor:latest
+      - "12345:12345"
     volumes:
+      - ${BASE_DIR-.}/probers:/etc/alloy/probers
+      # CAdvisor
       - /:/rootfs:ro
       - /var/run:/var/run:rw
       - /sys:/sys:ro
       - /var/lib/docker/:/var/lib/docker:ro
-    ports:
-      - 9116:8080
+    environment:
+      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
+      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics
+      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics
     networks:
       - observability-exporters
-    command:
-      - '-housekeeping_interval=10s'
-      - '-docker_only=true'
-    security_opt:
-      - no-new-privileges:true
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.cadvisor.rule=PathPrefix(`/cadvisor`)"
-      - "traefik.http.middlewares.cadvisor-stripprefix.stripprefix.prefixes=/cadvisor"
-      - "traefik.http.routers.cadvisor.middlewares=cadvisor-stripprefix@docker"
 
 networks:
   observability-exporters:
diff --git a/observability/examples/full/exporters.elastic.docker-compose.yml b/observability/examples/full/exporters.elastic.docker-compose.yml
index 3a02bd5..6b96d21 100644
--- a/observability/examples/full/exporters.elastic.docker-compose.yml
+++ b/observability/examples/full/exporters.elastic.docker-compose.yml
@@ -1,21 +1,29 @@
-# Exporter deployment on each VM for exporting metrics.
+# Run to get elasticsearch metrics into prometheus
 name: "cogstack-observability-exporters"
 services:
-  elasticsearch_exporter:
-    image: quay.io/prometheuscommunity/elasticsearch-exporter:latest
+  alloy:
+    image: cogstacksystems/cogstack-observability-alloy:latest
     restart: unless-stopped
-    command:
-     - '--es.uri=${ES_URL}'
-     - '--es.ssl-skip-verify'
     ports:
-    - "9114:9114"
+      - "12345:12345"
+    volumes:
+      - ${BASE_DIR-.}/probers:/etc/alloy/probers
+      - ${BASE_DIR-.}/alloy/elasticsearch.alloy:/etc/alloy/elasticsearch.alloy # Enable Elastic Exporter
+      # CAdvisor
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
     environment:
-      - ES_USERNAME=${ES_USERNAME}
-      - ES_PASSWORD=${ES_PASSWORD}
+      - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
+      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost} # Used to add a label to metrics
+      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost} # Used to add a label to metrics
+      - ELASTICSEARCH_URL=${ELASTICSEARCH_URL-https://elassticsearch-1:9200}
+      - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME-user} # Used to get metrics from Elasticsearch
+      - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD-pass} # Used to get metrics from Elasticsearch
     networks:
       - observability-exporters
 
 networks:
   observability-exporters:
-    name: cogstack-observability-exporters_observability-exporters
-    external: true
\ No newline at end of file
+    driver: bridge
diff --git a/observability/examples/full/prometheus/scrape-configs/probers/probe-external.yml b/observability/examples/full/probers/probe-external.yml
similarity index 100%
rename from observability/examples/full/prometheus/scrape-configs/probers/probe-external.yml
rename to observability/examples/full/probers/probe-external.yml
diff --git a/observability/examples/full/probers/probe-internal.yml b/observability/examples/full/probers/probe-internal.yml
new file mode 100644
index 0000000..fa16233
--- /dev/null
+++ b/observability/examples/full/probers/probe-internal.yml
@@ -0,0 +1,6 @@
+# Example of probe targets in a different file. 
+- targets:
+    - https://cogstack.org
+  labels:
+    name: cogstack-homepage
+    job: probe-internal-services
\ No newline at end of file
diff --git a/observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml b/observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml
index 8036d08..004fc0e 100644
--- a/observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml
+++ b/observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml
@@ -1,3 +1,4 @@
+# Scrape metrics from any targets that are exposing Prometheus formatted data. Default is to call the /metrics API. 
 - targets:
     - cogstack-observability-node-exporter-1:9100
   labels:
diff --git a/observability/examples/full/prometheus/scrape-configs/probers/probe-internal.yml b/observability/examples/full/prometheus/scrape-configs/probers/probe-internal.yml
deleted file mode 100644
index b3d7353..0000000
--- a/observability/examples/full/prometheus/scrape-configs/probers/probe-internal.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-# Example of probe targets
-- targets:
-    - https://cogstack.org
-  labels:
-    name: cogstack-homepage
-    job: probe-services
\ No newline at end of file
diff --git a/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml b/observability/examples/simple/alloy/probers/probe-simple.yml
similarity index 100%
rename from observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
rename to observability/examples/simple/alloy/probers/probe-simple.yml
diff --git a/observability/examples/simple/docker-compose.yml b/observability/examples/simple/docker-compose.yml
index 7f2f8d1..3f22573 100755
--- a/observability/examples/simple/docker-compose.yml
+++ b/observability/examples/simple/docker-compose.yml
@@ -2,11 +2,17 @@
 # Depends on docker-compose.exporters.yml for the network
 name: "cogstack-observability"
 services:
+  alloy:
+    image: cogstacksystems/cogstack-observability-alloy:latest
+    restart: unless-stopped
+    volumes:
+      - ${BASE_DIR-.}/alloy/probers:/etc/alloy/probers
+    networks:
+      - observability
   prometheus:
     image: cogstacksystems/cogstack-observability-prometheus:latest
     restart: unless-stopped
     volumes:
-      - ./prometheus:/etc/prometheus/cogstack/site/
       - prometheus-data:/prometheus
     networks:
       - observability
@@ -29,16 +35,7 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
-  blackbox-exporter:
-    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
-    restart: unless-stopped
-    networks:
-      - observability
-  node-exporter:
-    image: prom/node-exporter
-    restart: unless-stopped
-    networks:
-      - observability
+
 networks:
   observability:
     driver: bridge
diff --git a/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml b/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml
deleted file mode 100644
index 8036d08..0000000
--- a/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-- targets:
-    - cogstack-observability-node-exporter-1:9100
-  labels:
-    job: node_exporter
-    host: localhost
\ No newline at end of file
diff --git a/observability/examples/simple/quickstart.sh b/observability/examples/simple/quickstart.sh
index a252b82..ecc7569 100644
--- a/observability/examples/simple/quickstart.sh
+++ b/observability/examples/simple/quickstart.sh
@@ -1,21 +1,16 @@
 #!/bin/bash
 set -e
 
-mkdir -p observability-simple/prometheus/scrape-configs/probers
-mkdir -p observability-simple/prometheus/scrape-configs/exporters
+mkdir -p observability-simple/alloy/probers
 cd observability-simple
 
 echo "Downloading docker-compose.yml..."
 curl -fsSL -o docker-compose.yml \
   https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/docker-compose.yml
 
-echo "Downloading probe-simple.yml into prometheus/scrape-configs/probers/..."
-curl -fsSL -o prometheus/scrape-configs/probers/probe-simple.yml \
-  https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
-
-echo "Downloading exporters-simple.yml into prometheus/scrape-configs/exporters/..."
-curl -fsSL -o prometheus/scrape-configs/exporters/exporters-simple.yml \
-  https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/refs/heads/main/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml
+echo "Downloading probe-simple.yml into alloy/probers/..."
+curl -fsSL -o probers/probe-simple.yml \
+  https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/probers/probe-simple.yml
 
 echo "Setup complete in observability-simple/"
 
diff --git a/observability/grafana-alloy/defaults/config.alloy b/observability/grafana-alloy/defaults/config.alloy
index 41482ff..34f8c7a 100644
--- a/observability/grafana-alloy/defaults/config.alloy
+++ b/observability/grafana-alloy/defaults/config.alloy
@@ -1,12 +1,8 @@
 logging {
-  level  = "debug"
+  level  = "info"
   format = "logfmt"
 }
 
-livedebugging {
-  enabled = true
-}
-
 prometheus.remote_write "default" {
   endpoint {
     url = sys.env("PROMETHEUS_URL") + "/api/v1/write"
diff --git a/observability/grafana-alloy/defaults/probe-default-config.alloy b/observability/grafana-alloy/defaults/probe-default-config.alloy
index 9b75043..8c55d47 100644
--- a/observability/grafana-alloy/defaults/probe-default-config.alloy
+++ b/observability/grafana-alloy/defaults/probe-default-config.alloy
@@ -44,7 +44,7 @@ discovery.relabel "probe_observability_stack_results" {
 
 // Blackbox Exporter - Probe External Services
 discovery.file "probe_target_files" {
-  files = ["/etc/alloy/probes/*.yml"]
+  files = ["/etc/alloy/probers/*.yml"]
   refresh_interval = "5m"
 }
 

From dd4929d76b68a52d9f3cfa64876f6b3b3ee765d7 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Mon, 9 Jun 2025 16:36:02 +0000
Subject: [PATCH 09/11] Update documentation and examples

---
 docs/observability/setup/production-setup.md               | 7 ++++---
 observability/examples/full/docker-compose.yml             | 2 +-
 observability/examples/full/exporters.docker-compose.yml   | 3 ---
 .../examples/full/exporters.elastic.docker-compose.yml     | 3 ---
 observability/examples/full/full-quickstart.sh             | 4 ++--
 observability/examples/full/probers/probe-external.yml     | 6 ------
 observability/examples/full/probers/probe-internal.yml     | 6 ------
 observability/grafana-alloy/Dockerfile                     | 1 +
 8 files changed, 8 insertions(+), 24 deletions(-)
 delete mode 100644 observability/examples/full/probers/probe-external.yml
 delete mode 100644 observability/examples/full/probers/probe-internal.yml

diff --git a/docs/observability/setup/production-setup.md b/docs/observability/setup/production-setup.md
index f0223f0..2c01ac2 100644
--- a/docs/observability/setup/production-setup.md
+++ b/docs/observability/setup/production-setup.md
@@ -42,13 +42,14 @@ Downloads the example docker compose files:
 - [exporters.docker-compose.yml](../../../observability/examples/full/exporters.docker-compose.yml)
 - [exporters.elastic.docker-compose.yml](../../../observability/examples/full/exporters.elastic.docker-compose.yml)
 
-Downloads the prometheus configurations:
+Downloads the configurations:
+- [alloy/probers/probe-external.yml](../../../observability/examples/full/alloy/probers/probe-external.yml)
+- [alloy/probers/probe-internal.yml ](../../../observability/examples/full/alloy/probers/probe-internal.yml)
 - [prometheus/scrape-configs/exporters/exporters.yml](../../../observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml)
-- [prometheus/scrape-configs/probers/probe-external.ymll](../../../observability/examples/full/prometheus/scrape-configs/probers/probe-external.yml)
-- [prometheus/scrape-configs/probers/probe-internal.yml ](../../../observability/examples/full/prometheus/scrape-configs/probers/probe-internal.yml)
 - [prometheus/scrape-configs/recording-rules/slo.yml](../../../observability/examples/full/prometheus/scrape-configs/recording-rules/slo.yml)
 
 
+
 Inspect the results in your local directory, and see that it matches the folder layout defined in step 1. 
 
 ## Step 3: Run the Stack
diff --git a/observability/examples/full/docker-compose.yml b/observability/examples/full/docker-compose.yml
index 16c999f..87b6a8a 100755
--- a/observability/examples/full/docker-compose.yml
+++ b/observability/examples/full/docker-compose.yml
@@ -7,7 +7,7 @@ services:
     ports:
       - "12345:12345"
     volumes:
-      - ${BASE_DIR-.}/probers:/etc/alloy/probers
+      - ${BASE_DIR-.}/alloy/probers:/etc/alloy/probers
       # CAdvisor
       - /:/rootfs:ro
       - /var/run:/var/run:rw
diff --git a/observability/examples/full/exporters.docker-compose.yml b/observability/examples/full/exporters.docker-compose.yml
index 67c9aa3..a16f194 100755
--- a/observability/examples/full/exporters.docker-compose.yml
+++ b/observability/examples/full/exporters.docker-compose.yml
@@ -3,10 +3,7 @@ name: "cogstack-observability-exporters"
 services:
   alloy:
     image: cogstacksystems/cogstack-observability-alloy:latest
-    ports:
-      - "12345:12345"
     volumes:
-      - ${BASE_DIR-.}/probers:/etc/alloy/probers
       # CAdvisor
       - /:/rootfs:ro
       - /var/run:/var/run:rw
diff --git a/observability/examples/full/exporters.elastic.docker-compose.yml b/observability/examples/full/exporters.elastic.docker-compose.yml
index 6b96d21..0b5c69f 100644
--- a/observability/examples/full/exporters.elastic.docker-compose.yml
+++ b/observability/examples/full/exporters.elastic.docker-compose.yml
@@ -4,10 +4,7 @@ services:
   alloy:
     image: cogstacksystems/cogstack-observability-alloy:latest
     restart: unless-stopped
-    ports:
-      - "12345:12345"
     volumes:
-      - ${BASE_DIR-.}/probers:/etc/alloy/probers
       - ${BASE_DIR-.}/alloy/elasticsearch.alloy:/etc/alloy/elasticsearch.alloy # Enable Elastic Exporter
       # CAdvisor
       - /:/rootfs:ro
diff --git a/observability/examples/full/full-quickstart.sh b/observability/examples/full/full-quickstart.sh
index 3f77c9a..c811f3d 100644
--- a/observability/examples/full/full-quickstart.sh
+++ b/observability/examples/full/full-quickstart.sh
@@ -19,8 +19,8 @@ download_to docker-compose.yml
 download_to exporters.docker-compose.yml
 download_to exporters.elastic.docker-compose.yml
 
-download_to prometheus/scrape-configs/probers/probe-internal.yml
-download_to prometheus/scrape-configs/probers/probe-external.yml
+download_to alloy/probers/probe-internal.yml
+download_to alloy/probers/probe-external.yml
 download_to prometheus/scrape-configs/exporters/exporters.yml
 download_to prometheus/scrape-configs/recording-rules/slo.yml
 
diff --git a/observability/examples/full/probers/probe-external.yml b/observability/examples/full/probers/probe-external.yml
deleted file mode 100644
index b3d7353..0000000
--- a/observability/examples/full/probers/probe-external.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-# Example of probe targets
-- targets:
-    - https://cogstack.org
-  labels:
-    name: cogstack-homepage
-    job: probe-services
\ No newline at end of file
diff --git a/observability/examples/full/probers/probe-internal.yml b/observability/examples/full/probers/probe-internal.yml
deleted file mode 100644
index fa16233..0000000
--- a/observability/examples/full/probers/probe-internal.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-# Example of probe targets in a different file. 
-- targets:
-    - https://cogstack.org
-  labels:
-    name: cogstack-homepage
-    job: probe-internal-services
\ No newline at end of file
diff --git a/observability/grafana-alloy/Dockerfile b/observability/grafana-alloy/Dockerfile
index d431ed1..f8895c7 100644
--- a/observability/grafana-alloy/Dockerfile
+++ b/observability/grafana-alloy/Dockerfile
@@ -14,3 +14,4 @@ CMD [ \
     ] 
 
 ENV PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
+full/prometheus/scrape-configs/probers/probe-external.ym
\ No newline at end of file

From 1312622f54572a4b8563148414d3ebb2fb9d5a44 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Mon, 9 Jun 2025 16:36:11 +0000
Subject: [PATCH 10/11] Update documentation and examples

---
 .../examples/full/alloy/probers/probe-external.yml          | 6 ++++++
 .../examples/full/alloy/probers/probe-internal.yml          | 6 ++++++
 2 files changed, 12 insertions(+)
 create mode 100644 observability/examples/full/alloy/probers/probe-external.yml
 create mode 100644 observability/examples/full/alloy/probers/probe-internal.yml

diff --git a/observability/examples/full/alloy/probers/probe-external.yml b/observability/examples/full/alloy/probers/probe-external.yml
new file mode 100644
index 0000000..b3d7353
--- /dev/null
+++ b/observability/examples/full/alloy/probers/probe-external.yml
@@ -0,0 +1,6 @@
+# Example of probe targets
+- targets:
+    - https://cogstack.org
+  labels:
+    name: cogstack-homepage
+    job: probe-services
\ No newline at end of file
diff --git a/observability/examples/full/alloy/probers/probe-internal.yml b/observability/examples/full/alloy/probers/probe-internal.yml
new file mode 100644
index 0000000..fa16233
--- /dev/null
+++ b/observability/examples/full/alloy/probers/probe-internal.yml
@@ -0,0 +1,6 @@
+# Example of probe targets in a different file. 
+- targets:
+    - https://cogstack.org
+  labels:
+    name: cogstack-homepage
+    job: probe-internal-services
\ No newline at end of file

From db986c62112e547fc39a77e03c65bba3caea61ec Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Mon, 9 Jun 2025 16:38:21 +0000
Subject: [PATCH 11/11] Update documentation and examples

---
 observability/grafana-alloy/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/observability/grafana-alloy/Dockerfile b/observability/grafana-alloy/Dockerfile
index f8895c7..affb334 100644
--- a/observability/grafana-alloy/Dockerfile
+++ b/observability/grafana-alloy/Dockerfile
@@ -14,4 +14,4 @@ CMD [ \
     ] 
 
 ENV PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus
-full/prometheus/scrape-configs/probers/probe-external.ym
\ No newline at end of file
+