Add testing and update documentation

.devcontainer/devcontainer.json

@@ -5,7 +5,8 @@
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
 	"image": "mcr.microsoft.com/devcontainers/base:jammy",
 	"features": {
-		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
+		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
+		"ghcr.io/devcontainers/features/python:1": {}
 	},
 
 	// Features to add to the dev container. More info: https://containers.dev/features.

.github/workflows/observability-docker.yml

@@ -66,3 +66,37 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+
+
+
+  test-observability-quickstart-scripts:
+    runs-on: ubuntu-latest
+    name: Test observability Quickstart scripts
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+
+    - name: Install Python dependencies
+      run: pip install -r observability/test/requirements.txt
+
+    - name: Run Simple quickstart test
+      run: |
+        echo "🧪 Running Simple quickstart test..."
+        bash observability/test/test-quickstart.sh
+
+    - name: Report test result
+      if: always()
+      run: |
+        if [ $? -eq 0 ]; then
+          echo "✅ quickstart test PASSED"
+        else
+          echo "❌ quickstart test FAILED"
+          exit 1
+        fi

docs/observability/customization/blackbox-exporter-config.md

@@ -11,12 +11,12 @@ You will need to create a new file, and then mount it over the existing provided
 1. Create a new file:
 
    ```
-    prometheus/blackbox-exporter/custom-blackbox-config.yml
+    alloy/blackbox-exporter.yml
    ```
 
 2. Add the existing defaults
 
-```  
+```yaml
 modules:
     http_get_200:
         prober: http
@@ -31,7 +31,7 @@ modules:
 ```
 
 3. Add your own module to the modules in that file
-```
+```yaml
   http_2xx_custom:
     prober: http
     timeout: 5s
@@ -55,7 +55,7 @@ This example adds a module named `http_2xx_custom` that adds some basic auth cre
 
 In your probe YAML file, reference the module in the `module` field of the `labels` section:
 
-```
+```yaml
     - targets:
         - https://myservice.example.com/health
       labels:
@@ -66,14 +66,11 @@ In your probe YAML file, reference the module in the `module` field of the `labe
 ### 3. Mount the config file
 You lastly need to mount the new config file and refer to it in docker compose
 
-```
-  blackbox-exporter:
-    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
-    restart: unless-stopped
-    networks:
-      - observability
+```yaml
+ alloy:
+    image: cogstacksystems/cogstack-observability-alloy:latest
+...
     volumes:
-      - ./prometheus/blackbox-exporter:/config
-    command:
-      - "--config.file=/config/custom-blackbox-config.yml" 
+      - ${BASE_DIR-.}/alloy/blackbox-exporter.yml:/etc/alloy/blackbox-exporter.yml
+...
 ```

docs/observability/reference/quickstart-manual.md

@@ -1,22 +1,18 @@
 # Manual Quickstart
-//TODO
 The quickstart page uses a script to setup the folders for you.
 
 This page instead details how to do it manually, to provide clarity.
 
 ## Step 1: Setup directory
 Create the necessary directory structure
 ```
-mkdir -p observability-simple/prometheus/scrape-configs/probers
-mkdir -p observability-simple/prometheus/scrape-configs/exporters
+mkdir -p observability-simple/alloy/probers
 ```
- Something
-
 
-Download these two files from github, and place in the right folder
+Download these two files, and place in the right folder
 
-- [docker-compose.yml](https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/docker-compose.yml) in observability-simple/
-- [prometheus/scrape-configs/probers/probe-simple.yml](https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml)
+- [docker-compose.yml](../../../observability/examples/simple/docker-compose.yml) in observability-simple/
+- [probe-observability.yml](../../../observability/examples/simple/alloy/probers/probe-observability.yml) into observability-simple/alloy/probers
 
 ### Step 2: Start the stack
 
@@ -29,7 +25,7 @@ docker compose up -d
 ### Step 3: Access the dashboards
 Open your web browser and go to:
 
-`localhost/grafana`
+`http://localhost/grafana`
 
 You should see the Grafana dashboard displaying the availability of the sample web page.
 

docs/observability/setup/probing.md

@@ -1,6 +1,6 @@
 # Availability
 
-This guide explains how to configure HTTP probers using Blackbox Exporter to monitor the availability of your services. These probers generate uptime and latency metrics, which can then be visualized in Grafana.
+This guide explains how to configure HTTP probers using Blackbox Exporter in Grafana Alloy to monitor the availability of your services. These probers generate uptime and latency metrics, which can then be visualized in Grafana.
 
 See the [Reference](../reference/understanding-metrics.md) for an explanation of the metrics this generates.
 
@@ -13,7 +13,7 @@ To add a new prober target:
 1. Navigate to the folder:
 
    ```
-    prometheus/scrape-configs/probers/
+    alloy/probers/
    ```
 
 2. Create a new YAML file (e.g., `probe.my-services.yml`) with the following structure:
@@ -27,10 +27,10 @@ To add a new prober target:
         job: my-services             # Mandatory - used to group probes in dashboards
         ip_address: "10.0.0.12"      # Optional - IP of the host being probed
         host: service-hostname       # Optional - Human-readable hostname
-        region: eu-west              # Optional - Any additional metadata label
+        any_custom_field: anything   # Optional - Add as many labels as desired here
    ```
 
-3. Ensure the folder is mounted in docker under `/etc/prometheus/cogstack/site/prometheus/scrape-configs/probers`, which it should be by default if you've followed the setup guids. Any valid `.yml` files in this folder will be automatically picked up and used as Blackbox targets.
+3. Ensure the folder is mounted in docker under `/etc/alloy/probers`, which it should be by default if you've followed the setup guids. Any valid `.yml` files in this folder will be automatically picked up and used as Blackbox targets.
 
 ---
 

docs/observability/setup/production-setup.md

@@ -7,31 +7,13 @@ If you're new, we recommend completing the [Quickstart Tutorial](../get-started/
 By the end of the tutorial, you will have a complete stack offering all the observability features, customized to your usage. 
 
 We will run the stack and then:
-- Configure *Telemetry* like VM memory usage, and Elasticsearch index size, by running Exporters
-- Enable *Alerting* based on our availability and a defined Service Level Objective (SLO)
 - Setup further *Probing* of our running services to get availability metrics
-
+- Configure *Telemetry* like VM memory usage, and Elasticsearch index size, by running Grafana Alloy
+- Enable *Alerting* based on our availability and a defined Service Level Objective (SLO)
 
 ---
 
-## Step 1: Understand the Folder Structure
-
-Your project configuration should follow this structure:
-
-```
-observability.docker-compose.yml
-exporters.docker-compose.yml
-alloy/
-    probers/           # HTTP endpoints to check availability
-        blackbox-exporter/     # (Optional) Custom Probe configuration
-prometheus/
-    scrape-configs/
-        exporters/         # Targets that expose metrics (e.g. Elasticsearch, Docker, VMs)
-    recording-rules/       # Prometheus recording rules (e.g. for SLOs, summaries)
-grafana/                   # (Optional) Custom Grafana dashboards and config
-```
-
-## Step 2: Initialise the project
+## Step 1: Initialise the project
 
 Run:
 ```bash
@@ -56,21 +38,39 @@ Downloads the configurations:
 
 
 
-Inspect the results in your local directory, and see that it matches the folder layout defined in step 1. 
+## Step 2: Understand the Folder structure
+
+Your project configuration will be created with follow this structure:
+
+```
+observability.docker-compose.yml
+exporters.docker-compose.yml
+alloy/
+    probers/           # HTTP endpoints to check availability
+        blackbox-exporter/     # (Optional) Custom Probe configurations like auth details
+prometheus/
+    scrape-configs/
+        exporters/         # Targets that expose metrics (e.g. Elasticsearch, Docker, VMs)
+    recording-rules/       # Prometheus recording rules (e.g. for SLOs, summaries)
+grafana/                   # (Optional) Custom Grafana dashboards and config
+```
+
+Inspect the results of the script and see that it matches this layout
+
 
 ## Step 3: Run the Stack
 The files come with basic defaults, so we can now run the stack
 
-
    ```
    docker compose up -d
    ```
 
 This will launch Prometheus, Grafana, and Alloy
 
+Navigate to the dashboard urls on `http://localhost/grafana` to view the dashboards.
 
 
-## Step 4: Create Site-Specific Config Files
+## Step 4: Create Site-Specific Probing files
 You must provide your own scrape and recording rules to tell Prometheus what to monitor.
 
 This is probably the hardest step: You will actually need to know what is running, and where it is! Building out these config files will give you that inventory, and give a real definition of what is running where.
@@ -79,9 +79,6 @@ This is probably the hardest step: You will actually need to know what is runnin
   - Add files in `alloy/probers/*.yml`
   - [Configure Probers](./probing.md)
 
-- Telemetry: Run Grafana Alloy on every VM you want telemetry from
-    - [Configure Telemetry](./probing.md)
-
 - Recording Rules: Define uptime goals or custom aggregations
   - Add files in `recording-rules/*.yml`
   - [Enable Alerting](./alerting.md)
@@ -94,6 +91,9 @@ Use the example docker compose file in [exporters.docker-compose.yml](../../../o
    ```
    docker compose -f exporters.docker-compose.yml up -d
    ```
+
+See [Configure Telemetry](./telemetry.md) for the full details
+
 ---
 
 ## What’s Next?

docs/observability/setup/telemetry.md

@@ -12,6 +12,46 @@ Grafana Alloy is used to get telemetry. These features are configured by default
 
 ## How to get Telemetry
 
-- Copy this docker compose file: (exporters.docker-compose.yml)[observability/examples/full/exporters.docker-compose.yml]
-- Edit the environment variables to point to your prometheus URL
-- Run `docker compose -f exporters.docker-compose.yml up -d ` on every VM you want metrics from
+We have to run Grafana Alloy on every single VM to get telemetry.
+
+Alloy is setup to push metrics to a central prometheus instance.
+
+- Copy this docker compose file: [exporters.docker-compose.yml](../../../observability/examples/full/exporters.docker-compose.yml)
+- Edit the environment variables to point to your prometheus URL:
+
+```yaml
+    environment:
+      - PROMETHEUS_URL=http://some-host:9090/prometheus # The URL that is running prometheus.
+      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME-localhost}      # Used to add a label to metrics
+      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS-localhost}  # Used to add a label to metrics
+```
+
+Now you have the setup, you will have to run this on every VM you want metrics from. This is a good opportunity to look into orchestrating deployments from a central place.
+
+- On each specific VM, run `docker compose -f exporters.docker-compose.yml up -d ` 
+
+
+### Elastic Search Metrics
+To get elasticsearch metrics we have to mount an alloy config file into the image.
+
+- Copy this docker compose file: [exporters.elastic.docker-compose.yml](../../../observability/examples/full/exporters.elastic.docker-compose.yml)
+- Copy this configuration file [elasticsearch.alloy](../../../observability/examples/full/alloy/elasticsearch.alloy) into `alloy/elasticsearch.alloy`
+
+In the docker compose file, we can see there are two changes to the usual exporter:
+
+```yaml
+    volumes:
+      - ${BASE_DIR-.}/alloy/elasticsearch.alloy:/etc/alloy/elasticsearch.alloy # Enable Elastic Exporter
+      ...
+    environment:    
+      - ELASTICSEARCH_URL=${ELASTICSEARCH_URL-https://elasticsearch-1:9200}
+      - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME-user} # Used to get metrics from Elasticsearch
+      - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD-pass} # Used to get metrics from Elasticsearch
+```
+
+By adding the alloy config file, and the elasticsearch environment details, Alloy will run and get metrics from elasticsearch.
+
+This pattern can be used to customize grafana alloy fully: mount any config files you want to in `/etc/alloy` and it will read them.
+
+## Next Steps
+- You can fully customise Grafana Alloy by mounting your own alloy config files

observability/examples/full/full-quickstart.sh

@@ -20,10 +20,15 @@ download_to docker-compose.yml
 download_to exporters.docker-compose.yml
 download_to exporters.elastic.docker-compose.yml
 
-download_to alloy/probers/probe-internal.yml
+download_to alloy/probers/probe-observability.yml
 download_to alloy/probers/probe-external.yml
 download_to prometheus/scrape-configs/exporters/exporters.yml
 download_to prometheus/scrape-configs/recording-rules/slo.yml
 
-
 echo "Setup complete in cogstack-observability/"
+
+echo "Starting the observability stack"
+
+docker compose up -d
+
+echo "Please open http://localhost/grafana in your browser"

observability/examples/full/prometheus/scrape-configs/exporters/exporters.yml

@@ -1,6 +1,6 @@
 # Scrape metrics from any targets that are exposing Prometheus formatted data. Default is to call the /metrics API. 
-- targets:
-    - cogstack-observability-node-exporter-1:9100
-  labels:
-    job: node_exporter
-    host: localhost
+# - targets:
+#     - cogstack-observability-node-exporter-1:9100
+#   labels:
+#     job: node_exporter
+#     host: localhost

observability/examples/simple/quickstart.sh

@@ -9,8 +9,8 @@ curl -fsSL -o docker-compose.yml \
   https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/docker-compose.yml
 
 echo "Downloading probe-simple.yml into alloy/probers/..."
-curl -fsSL -o probers/probe-observability.yml \
-  https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/probers/probe-observability.yml
+curl -fsSL -o alloy/probers/probe-observability.yml \
+  https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/alloy/probers/probe-observability.yml
 
 echo "Setup complete in observability-simple/"
 

observability/grafana-alloy/Dockerfile

@@ -3,6 +3,8 @@ FROM grafana/alloy:latest
 LABEL traefik.enable="true" \
     traefik.http.routers.alloy.rule="PathPrefix(`/alloy`)" 
 
+EXPOSE 12345
+
 RUN mkdir -p /etc/alloy/probers
 
 COPY ./defaults /etc/alloy

observability/prometheus/Dockerfile.prometheus

@@ -3,6 +3,9 @@ FROM prom/prometheus
 LABEL traefik.enable="true" \
     traefik.http.routers.prometheus-path-router.rule="PathPrefix(`/prometheus`)" 
 
+RUN mkdir -p /etc/prometheus/cogstack/site/scrape-targets/probers
+RUN mkdir -p /etc/prometheus/cogstack/site/scrape-targets/exporters
+
 COPY ./defaults /etc/prometheus/cogstack/defaults
 
 CMD [ \