From 0c49071c4daa7239bbb4bf94819118d1af5d5bca Mon Sep 17 00:00:00 2001
From: nikolamilosa <nikola.milosavljevic@dfinity.org>
Date: Tue, 4 Nov 2025 20:43:13 +0100
Subject: [PATCH 1/6] updating defitions

---
 TODO.md                                     |  6 +++++
 definitions/multiservice-discovery-crds.yml |  2 +-
 tasks/deploy-k3s-cluster.yml                |  6 +++++
 tasks/install-vagrant.yml                   | 30 +++++++++++++++------
 tasks/install-virtualbox.yml                | 27 +++++++++++++++----
 vagrant/Vagrantfile                         |  3 ++-
 6 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/TODO.md b/TODO.md
index 2570450..13af220 100644
--- a/TODO.md
+++ b/TODO.md
@@ -19,6 +19,12 @@
     services (such as VictoriaMetrics and Grafana) for this stack.
     * This may not end up being that necessary.
 
+## Quality of life improvements:
+* On some Linux distros `host` isn't present outside of the box. We should
+  ensure that all of the commands are present before proceeding.
+* We should allow people to install vagrant and vbox on their own but
+  specify the requirements (like vagrant disksize plugin)
+
 ## Long shots
 
 * Persistent storage for Grafana would be nice to have.
diff --git a/definitions/multiservice-discovery-crds.yml b/definitions/multiservice-discovery-crds.yml
index 6595a88..fed37a8 100644
--- a/definitions/multiservice-discovery-crds.yml
+++ b/definitions/multiservice-discovery-crds.yml
@@ -31,7 +31,7 @@ spec:
         app.kubernetes.io/name: service-discovery
     spec:
       containers:
-        - image: ghcr.io/dfinity/dre/multiservice-discovery:e00f95d876050cbb08f0f320ad6ba1ca0cf6442b
+        - image: ghcr.io/dfinity/dre/multiservice-discovery:da774e3d01d500e70f2ff862887e53a960693420
           imagePullPolicy: IfNotPresent
           name: service-discovery
           ports:
diff --git a/tasks/deploy-k3s-cluster.yml b/tasks/deploy-k3s-cluster.yml
index 41ac64e..1b63c13 100644
--- a/tasks/deploy-k3s-cluster.yml
+++ b/tasks/deploy-k3s-cluster.yml
@@ -24,6 +24,12 @@
   check_mode: false
   changed_when: false
   register: k3s_check
+- name: Make sure that anyone can read /etc/rancher/k3s/k3s.yaml
+  ansible.builtin.shell:
+    cmd: chmod 644 /etc/rancher/k3s/k3s.yaml
+  check_mode: false
+  changed_when: false
+  become: true
 - name: Wait until K3s is ready
   ansible.builtin.shell:
     cmd: |
diff --git a/tasks/install-vagrant.yml b/tasks/install-vagrant.yml
index f810702..fdf314f 100644
--- a/tasks/install-vagrant.yml
+++ b/tasks/install-vagrant.yml
@@ -1,15 +1,27 @@
+- name: Check if Vagrant is already installed
+  ansible.builtin.command: which vagrant
+  register: vagrant_check
+  ignore_errors: true
+
+- name: Skip installation if Vagrant is already installed
+  ansible.builtin.debug:
+    msg: "Vagrant is already installed at {{ vagrant_check.stdout }}"
+  when: vagrant_check.rc == 0
+
 - name: Fail on unsupported distribution
   ansible.builtin.fail:
     msg: "Your operating system is not supported.  Please install Vagrant on your system using the method documented by the Vagrant project."
-  when: >-
-    not (ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin")
+  when:
+    - vagrant_check.rc != 0
+    - not (ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin")
 
 - name: Install Vagrant
   ansible.builtin.package:
     name: vagrant
     state: present
-  when: >-
-    ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora"
+  when: 
+    - vagrant_check.rc != 0
+    - ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora"
   become: true
   become_user: root
 
@@ -17,8 +29,9 @@
   community.general.homebrew_cask:
     name: vagrant
     state: present
-  when: >-
-    ansible_facts["os_family"] == "Darwin"
+  when: 
+    - vagrant_check.rc != 0
+    - ansible_facts["os_family"] == "Darwin"
   become: false
 
 - name: Install Vagrant disk size plugin
@@ -28,8 +41,9 @@
         vagrant plugin install vagrant-disksize
         >&2 echo Plugin installed
       }
-  when: >-
-    ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin"
+  when:
+    - vagrant_check.rc != 0
+    - ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin"
   register: vagrant_disksize
   changed_when: >-
     "Plugin installed" in vagrant_disksize.stderr
diff --git a/tasks/install-virtualbox.yml b/tasks/install-virtualbox.yml
index 200723f..cd33be1 100644
--- a/tasks/install-virtualbox.yml
+++ b/tasks/install-virtualbox.yml
@@ -1,11 +1,24 @@
+- name: Check if Virtualbox is already installed
+  ansible.builtin.command: which virtualbox
+  register: virtualbox_check
+  ignore_errors: true
+
+- name: Skip installation if Virtualbox is already installed
+  ansible.builtin.debug:
+    msg: "Virtualbox is already installed at {{ virtualbox_check.stdout }}"
+  when: virtualbox_check.rc == 0
+
 - name: Fail on unsupported distribution
   ansible.builtin.fail:
     msg: "Your operating system is not supported.  Please install Vagrant on your system using the method documented by the Vagrant project."
-  when: >-
-    not (ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin")
+  when: 
+    - virtualbox_check.rc != 0 
+    - not (ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin")
 
 - name: Install VirtualBox on Debian-based systems
-  when: ansible_facts["os_family"] == "Debian"
+  when: 
+    - virtualbox_check.rc != 0
+    - ansible_facts["os_family"] == "Debian"
   block:
     - name: Install VirtualBox
       ansible.builtin.package:
@@ -43,7 +56,9 @@
       become_user: root
 
 - name: Install VirtualBox on Fedora systems
-  when: ansible_facts["distribution"] == "Fedora"
+  when: 
+    - virtualbox_check.rc != 0
+    - ansible_facts["distribution"] == "Fedora"
   block:
     - name: Import VirtualBox RPM key
       ansible.builtin.rpm_key:
@@ -85,7 +100,9 @@
       become_user: root
 
 - name: Install VirtualBox on Mac OS X
-  when: ansible_facts["os_family"] == "Darwin"
+  when: 
+    - virtualbox_check.rc != 0
+    - ansible_facts["os_family"] == "Darwin"
   block:
     - community.general.homebrew_cask:
         name: virtualbox
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index 8b92790..0430966 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -20,6 +20,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.network :private_network, ip: "192.168.56.42"
   config.vm.network "forwarded_port", guest: 32090, host: 32090
   config.vm.network "forwarded_port", guest: 32091, host: 32091
+  config.vm.network "forwarded_port", guest: 32080, host: 32080
 
   # Set the name of the VM. See: http://stackoverflow.com/a/17864388/100134
   config.vm.define :observability do |observability|
@@ -30,4 +31,4 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     ansible.become = true
   end
 
-end
\ No newline at end of file
+end

From 2997f6242a3392b644881dd98c2912f6de6a4730 Mon Sep 17 00:00:00 2001
From: nikolamilosa <nikola.milosavljevic@dfinity.org>
Date: Wed, 5 Nov 2025 00:29:38 +0100
Subject: [PATCH 2/6] Not using system pip

---
 tasks/setup-observability-stack-host-reqs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tasks/setup-observability-stack-host-reqs.yml b/tasks/setup-observability-stack-host-reqs.yml
index 3272f28..2937c6c 100644
--- a/tasks/setup-observability-stack-host-reqs.yml
+++ b/tasks/setup-observability-stack-host-reqs.yml
@@ -4,12 +4,12 @@
     pkg: python3-pip
     state: present
 - name: Install OpenShift pip module
-  ansible.builtin.pip:
-    name: openshift
+  ansible.builtin.apt:
+    name: python3-openshift
     state: present
 - name: Install Kubernetes pip module
-  ansible.builtin.pip:
-    name: kubernetes
+  ansible.builtin.apt:
+    name: python3-kubernetes
     state: present
 - block:
     - import_tasks: ../../tasks/install-helm.yml

From 39384a88f8bd9e940e446c0ac06cf5013dc60d3c Mon Sep 17 00:00:00 2001
From: nikolamilosa <nikola.milosavljevic@dfinity.org>
Date: Wed, 5 Nov 2025 01:41:05 +0100
Subject: [PATCH 3/6] Updating todo

---
 TODO.md | 24 +++++-------------------
 1 file changed, 5 insertions(+), 19 deletions(-)

diff --git a/TODO.md b/TODO.md
index 13af220..09893bb 100644
--- a/TODO.md
+++ b/TODO.md
@@ -2,28 +2,14 @@
 
 ## Functionality
 
-* Provide Prometheus + Grafana interfaces using Let's Encrypt TLS
-  certificates and authenticated access, when setting up a stack on
-  a SSH remote target.
-* Enable auto-discovery for machines of particular node providers.
-  This skips having to specify addresses manually.
-* Allow the user to specify “I’m a node provider” mode during auto-
-  discovery, which will cause the stack to scrape host/guest node
-  exporter, orchestrator and replica directly.  This is already
-  possible to do by manually saying `mode: direct` but not yet
-  working in practice due to IC firewall rules.
 * Reduce maintenance cost of the stack:
   * Re-use the DFINITY observability stack code to provide dashboards
     for this stack.
-  * Re-use the DFINITY observability stack code to provide infrastructure
-    services (such as VictoriaMetrics and Grafana) for this stack.
-    * This may not end up being that necessary.
-
-## Quality of life improvements:
-* On some Linux distros `host` isn't present outside of the box. We should
-  ensure that all of the commands are present before proceeding.
-* We should allow people to install vagrant and vbox on their own but
-  specify the requirements (like vagrant disksize plugin)
+* Allow specifying the setup local-path where k3s will store data
+* Set sane defaults for prometheus resources
+* Add alerting configuration
+* Rework the sample dashboard to use principal ids instead of hosts
+* Document service discovery endpoint
 
 ## Long shots
 

From 032a9a5a6e434f8601e9e22ca3e73bb1be073d14 Mon Sep 17 00:00:00 2001
From: nikolamilosa <nikola.milosavljevic@dfinity.org>
Date: Wed, 5 Nov 2025 16:39:29 +0100
Subject: [PATCH 4/6] sane defaults for prometheus and default path for
 provisioner for k3s

---
 TODO.md                                                    | 2 --
 .../{prometheus-crds.yml => prometheus-crds.yml.j2}        | 4 ++--
 hosts                                                      | 2 +-
 .../deploy-observability-stack-onto-kubernetes.yml         | 1 +
 tasks/deploy-k3s-cluster.yml                               | 4 ++++
 tasks/deploy-prometheus.yml                                | 2 +-
 vars/k3s.yml                                               | 6 +++++-
 vars/prometheus.yml                                        | 5 +++++
 vars/scrape_configs.yml                                    | 7 +++++++
 9 files changed, 26 insertions(+), 7 deletions(-)
 rename definitions/{prometheus-crds.yml => prometheus-crds.yml.j2} (96%)
 create mode 100644 vars/prometheus.yml

diff --git a/TODO.md b/TODO.md
index 09893bb..f6e9490 100644
--- a/TODO.md
+++ b/TODO.md
@@ -5,8 +5,6 @@
 * Reduce maintenance cost of the stack:
   * Re-use the DFINITY observability stack code to provide dashboards
     for this stack.
-* Allow specifying the setup local-path where k3s will store data
-* Set sane defaults for prometheus resources
 * Add alerting configuration
 * Rework the sample dashboard to use principal ids instead of hosts
 * Document service discovery endpoint
diff --git a/definitions/prometheus-crds.yml b/definitions/prometheus-crds.yml.j2
similarity index 96%
rename from definitions/prometheus-crds.yml
rename to definitions/prometheus-crds.yml.j2
index 9504e05..f17ed0b 100644
--- a/definitions/prometheus-crds.yml
+++ b/definitions/prometheus-crds.yml.j2
@@ -72,7 +72,7 @@ spec:
   replicas: 1
   resources:
     requests:
-      memory: 400Mi
+      memory: {{ prometheus.ram_request }}
   securityContext:
     fsGroup: 2000
     runAsNonRoot: true
@@ -86,7 +86,7 @@ spec:
         storageClassName: local-path
         resources:
           requests:
-            storage: 20Gi
+            storage: {{ prometheus.storage_request }}
   scrapeConfigSelector:
     matchLabels:
       app: prometheus
diff --git a/hosts b/hosts
index f145e59..805d3bb 100644
--- a/hosts
+++ b/hosts
@@ -1 +1 @@
-observability ansible_connection=vagrant ansible_vagrantfile=vagrant/Vagrantfile ansible_remote_user=vagrant
+observability ansible_host=devenv
diff --git a/playbooks/prepare-node/deploy-observability-stack-onto-kubernetes.yml b/playbooks/prepare-node/deploy-observability-stack-onto-kubernetes.yml
index 8017838..d354c7f 100644
--- a/playbooks/prepare-node/deploy-observability-stack-onto-kubernetes.yml
+++ b/playbooks/prepare-node/deploy-observability-stack-onto-kubernetes.yml
@@ -4,6 +4,7 @@
     - kubernetes.core
   vars_files:
     - ../../vars/k3s.yml
+    - ../../vars/prometheus.yml
     - ../../vars/grafana.yml
   become: true
   tasks:
diff --git a/tasks/deploy-k3s-cluster.yml b/tasks/deploy-k3s-cluster.yml
index 1b63c13..b49f384 100644
--- a/tasks/deploy-k3s-cluster.yml
+++ b/tasks/deploy-k3s-cluster.yml
@@ -5,6 +5,10 @@
   failed_when: false
   register: k3s_status
   changed_when: false
+- name: Ensure k3s local provisioner path exists
+  ansible.builtin.file:
+    path: "{{ persistence_volume_path | expanduser }}"
+    state: directory
 - name: Retrieve K3s installer
   ansible.builtin.get_url:
     url: https://get.k3s.io
diff --git a/tasks/deploy-prometheus.yml b/tasks/deploy-prometheus.yml
index 8ba6ac5..f4e9098 100644
--- a/tasks/deploy-prometheus.yml
+++ b/tasks/deploy-prometheus.yml
@@ -13,7 +13,7 @@
         context: default
         state: present
         definition: >-
-          {{ lookup('file', '../definitions/prometheus-crds.yml') | from_yaml_all }}
+          {{ lookup('template', '../definitions/prometheus-crds.yml.j2') | from_yaml_all }}
     - name: Deploy self-monitoring for Prometheus and its operator
       k8s:
         kubeconfig: "{{ k3s_config_file }}"
diff --git a/vars/k3s.yml b/vars/k3s.yml
index 97a858b..836699c 100644
--- a/vars/k3s.yml
+++ b/vars/k3s.yml
@@ -1,3 +1,7 @@
+# Path on the host where the local provisioner will persist data for the pods
+# of the observability stack.
+persistence_volume_path: ~/k3s/volumes
+
 k8s_version: "v1.29"
 k3s_cluster_cidrs:
   - 10.42.0.0/16
@@ -5,5 +9,5 @@ k3s_cluster_cidrs:
 k3s_service_cidrs:
   - 10.43.0.0/16 # services
   - 2001:cafe:43::/112
-k3s_INSTALL_K3S_EXEC: "--flannel-ipv6-masq --cluster-cidr={{ k3s_cluster_cidrs | join(',') }} --service-cidr={{ k3s_service_cidrs | join(',') }}"
+k3s_INSTALL_K3S_EXEC: "--flannel-ipv6-masq --cluster-cidr={{ k3s_cluster_cidrs | join(',') }} --service-cidr={{ k3s_service_cidrs | join(',') }} --default-local-storage-path={{ persistence_volume_path }}"
 k3s_config_file: /etc/rancher/k3s/k3s.yaml
diff --git a/vars/prometheus.yml b/vars/prometheus.yml
new file mode 100644
index 0000000..b986882
--- /dev/null
+++ b/vars/prometheus.yml
@@ -0,0 +1,5 @@
+# Prometheus configuration for storage and ram.
+# To learn more about the notation, read: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes
+prometheus:
+  ram_request: 8Gi
+  storage_request: 80Gi
diff --git a/vars/scrape_configs.yml b/vars/scrape_configs.yml
index 1477e81..237f813 100644
--- a/vars/scrape_configs.yml
+++ b/vars/scrape_configs.yml
@@ -1,3 +1,10 @@
 # You can add your scrape configuration here, if you plan to check this
 # data into a Git repository of your own.  The scrape_configs.yml in
 # the root folder of this project is not version-controlled.
+# Sample scrape configuration
+scrape_configs:
+  ic_nodes:
+    mode: direct
+    criteria:
+      # Only monitor nodes from this node provider:
+      node_provider_id: bvcsg-3od6r-jnydw-eysln-aql7w-td5zn-ay5m6-sibd2-jzojt-anwag-mqe

From 9a093fce5ef66d9f41279b370d4f49417a9483ca Mon Sep 17 00:00:00 2001
From: nikolamilosa <nikola.milosavljevic@dfinity.org>
Date: Wed, 5 Nov 2025 22:58:58 +0100
Subject: [PATCH 5/6] adding docs and upgrading vagrant

---
 README.md           | 11 +++++++++++
 vagrant/Vagrantfile |  4 ++--
 vars/k3s.yml        |  2 +-
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 59fb61e..9508c28 100644
--- a/README.md
+++ b/README.md
@@ -58,6 +58,8 @@ to your instance of the stack.
     you log onto this machine with must have root-equivalent access via
     `sudoers`, and should be running an Ubuntu LTS machine or a recent
     Fedora release.
+  * The recommended spec for the machine require 16GB of RAM and around
+    100GB of storage.
 
 ## Preparation
 
@@ -96,6 +98,15 @@ to let the observability stack know which targets to obtain telemetry from.
 See the documentation [on scrape configuration](doc/scrape-configs.md) for
 more information on how to describe these targets.
 
+After that, ensure to setup the correct path for persisting the data. By
+default, the configuration will create a directory `$HOME/k3s/volumes` and will
+store volumes there. To change this, modify variable `persistence_volume_path` 
+found in `./vars/k3s.yml`.
+
+If you would want to (at your own risk) run a setup that is weaker than
+recommended you can modify `./vars/prometheus.yml` and set your prefered
+ram and storage requests.
+
 With the scrape configuration in place, set up the stack by running:
 
 ```sh
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index 0430966..dd9453c 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -3,11 +3,11 @@ VAGRANTFILE_API_VERSION = "2"
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box = "ubuntu/jammy64"
   # Use config.vm.box = "alvistack/fedora-39" for Fedora.
-  config.disksize.size = '50GB'
+  config.disksize.size = '100GB'
 
   config.vm.provider :virtualbox do |v|
     v.name = "observability"
-    v.memory = 4096
+    v.memory = 16384
     v.cpus = 2
     v.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
     v.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
diff --git a/vars/k3s.yml b/vars/k3s.yml
index 836699c..734e154 100644
--- a/vars/k3s.yml
+++ b/vars/k3s.yml
@@ -9,5 +9,5 @@ k3s_cluster_cidrs:
 k3s_service_cidrs:
   - 10.43.0.0/16 # services
   - 2001:cafe:43::/112
-k3s_INSTALL_K3S_EXEC: "--flannel-ipv6-masq --cluster-cidr={{ k3s_cluster_cidrs | join(',') }} --service-cidr={{ k3s_service_cidrs | join(',') }} --default-local-storage-path={{ persistence_volume_path }}"
+k3s_INSTALL_K3S_EXEC: "--flannel-ipv6-masq --cluster-cidr={{ k3s_cluster_cidrs | join(',') }} --service-cidr={{ k3s_service_cidrs | join(',') }} --default-local-storage-path={{ persistence_volume_path | expanduser }}"
 k3s_config_file: /etc/rancher/k3s/k3s.yaml

From 9988f34830938a7c25f7b9dc5edd6c14bf1ef0b4 Mon Sep 17 00:00:00 2001
From: nikolamilosa <nikola.milosavljevic@dfinity.org>
Date: Wed, 5 Nov 2025 23:11:07 +0100
Subject: [PATCH 6/6] reverting DFinity setup for scraping

---
 hosts                   | 2 +-
 vars/scrape_configs.yml | 7 -------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/hosts b/hosts
index 805d3bb..f145e59 100644
--- a/hosts
+++ b/hosts
@@ -1 +1 @@
-observability ansible_host=devenv
+observability ansible_connection=vagrant ansible_vagrantfile=vagrant/Vagrantfile ansible_remote_user=vagrant
diff --git a/vars/scrape_configs.yml b/vars/scrape_configs.yml
index 237f813..1477e81 100644
--- a/vars/scrape_configs.yml
+++ b/vars/scrape_configs.yml
@@ -1,10 +1,3 @@
 # You can add your scrape configuration here, if you plan to check this
 # data into a Git repository of your own.  The scrape_configs.yml in
 # the root folder of this project is not version-controlled.
-# Sample scrape configuration
-scrape_configs:
-  ic_nodes:
-    mode: direct
-    criteria:
-      # Only monitor nodes from this node provider:
-      node_provider_id: bvcsg-3od6r-jnydw-eysln-aql7w-td5zn-ay5m6-sibd2-jzojt-anwag-mqe