chore: QOL improvements

README.md

@@ -58,6 +58,8 @@ to your instance of the stack.
     you log onto this machine with must have root-equivalent access via
     `sudoers`, and should be running an Ubuntu LTS machine or a recent
     Fedora release.
+  * The recommended spec for the machine require 16GB of RAM and around
+    100GB of storage.
 
 ## Preparation
 
@@ -96,6 +98,15 @@ to let the observability stack know which targets to obtain telemetry from.
 See the documentation [on scrape configuration](doc/scrape-configs.md) for
 more information on how to describe these targets.
 
+After that, ensure to setup the correct path for persisting the data. By
+default, the configuration will create a directory `$HOME/k3s/volumes` and will
+store volumes there. To change this, modify variable `persistence_volume_path` 
+found in `./vars/k3s.yml`.
+
+If you would want to (at your own risk) run a setup that is weaker than
+recommended you can modify `./vars/prometheus.yml` and set your prefered
+ram and storage requests.
+
 With the scrape configuration in place, set up the stack by running:
 
 ```sh

TODO.md

@@ -2,22 +2,12 @@
 
 ## Functionality
 
-* Provide Prometheus + Grafana interfaces using Let's Encrypt TLS
-  certificates and authenticated access, when setting up a stack on
-  a SSH remote target.
-* Enable auto-discovery for machines of particular node providers.
-  This skips having to specify addresses manually.
-* Allow the user to specify “I’m a node provider” mode during auto-
-  discovery, which will cause the stack to scrape host/guest node
-  exporter, orchestrator and replica directly.  This is already
-  possible to do by manually saying `mode: direct` but not yet
-  working in practice due to IC firewall rules.
 * Reduce maintenance cost of the stack:
   * Re-use the DFINITY observability stack code to provide dashboards
     for this stack.
-  * Re-use the DFINITY observability stack code to provide infrastructure
-    services (such as VictoriaMetrics and Grafana) for this stack.
-    * This may not end up being that necessary.
+* Add alerting configuration
+* Rework the sample dashboard to use principal ids instead of hosts
+* Document service discovery endpoint
 
 ## Long shots
 

definitions/multiservice-discovery-crds.yml

@@ -31,7 +31,7 @@ spec:
         app.kubernetes.io/name: service-discovery
     spec:
       containers:
-        - image: ghcr.io/dfinity/dre/multiservice-discovery:e00f95d876050cbb08f0f320ad6ba1ca0cf6442b
+        - image: ghcr.io/dfinity/dre/multiservice-discovery:da774e3d01d500e70f2ff862887e53a960693420
           imagePullPolicy: IfNotPresent
           name: service-discovery
           ports:

definitions/prometheus-crds.yml → definitions/prometheus-crds.yml.j2

@@ -72,7 +72,7 @@ spec:
   replicas: 1
   resources:
     requests:
-      memory: 400Mi
+      memory: {{ prometheus.ram_request }}
   securityContext:
     fsGroup: 2000
     runAsNonRoot: true
@@ -86,7 +86,7 @@ spec:
         storageClassName: local-path
         resources:
           requests:
-            storage: 20Gi
+            storage: {{ prometheus.storage_request }}
   scrapeConfigSelector:
     matchLabels:
       app: prometheus

playbooks/prepare-node/deploy-observability-stack-onto-kubernetes.yml

@@ -4,6 +4,7 @@
     - kubernetes.core
   vars_files:
     - ../../vars/k3s.yml
+    - ../../vars/prometheus.yml
     - ../../vars/grafana.yml
   become: true
   tasks:

tasks/deploy-k3s-cluster.yml

@@ -5,6 +5,10 @@
   failed_when: false
   register: k3s_status
   changed_when: false
+- name: Ensure k3s local provisioner path exists
+  ansible.builtin.file:
+    path: "{{ persistence_volume_path | expanduser }}"
+    state: directory
 - name: Retrieve K3s installer
   ansible.builtin.get_url:
     url: https://get.k3s.io
@@ -24,6 +28,12 @@
   check_mode: false
   changed_when: false
   register: k3s_check
+- name: Make sure that anyone can read /etc/rancher/k3s/k3s.yaml
+  ansible.builtin.shell:
+    cmd: chmod 644 /etc/rancher/k3s/k3s.yaml
+  check_mode: false
+  changed_when: false
+  become: true
 - name: Wait until K3s is ready
   ansible.builtin.shell:
     cmd: |

tasks/deploy-prometheus.yml

@@ -13,7 +13,7 @@
         context: default
         state: present
         definition: >-
-          {{ lookup('file', '../definitions/prometheus-crds.yml') | from_yaml_all }}
+          {{ lookup('template', '../definitions/prometheus-crds.yml.j2') | from_yaml_all }}
     - name: Deploy self-monitoring for Prometheus and its operator
       k8s:
         kubeconfig: "{{ k3s_config_file }}"

tasks/install-vagrant.yml

@@ -1,8 +1,19 @@
+- name: Check if Vagrant is already installed
+  ansible.builtin.command: which vagrant
+  register: vagrant_check
+  ignore_errors: true
+
+- name: Skip installation if Vagrant is already installed
+  ansible.builtin.debug:
+    msg: "Vagrant is already installed at {{ vagrant_check.stdout }}"
+  when: vagrant_check.rc == 0
+
 - name: Fail on unsupported distribution
   ansible.builtin.fail:
     msg: "Your operating system is not supported.  Please install Vagrant on your system using the method documented by the Vagrant project."
-  when: >-
-    not (ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin")
+  when:
+    - vagrant_check.rc != 0
+    - not (ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin")
 
 - name: Add Vagrant repository
   ansible.builtin.deb822_repository:
@@ -41,17 +52,19 @@
   ansible.builtin.package:
     name: vagrant
     state: present
-  when: >-
-    ansible_facts["distribution"] == "Fedora"
+  when: 
+    - vagrant_check.rc != 0
+    - ansible_facts["distribution"] == "Fedora"
   become: true
   become_user: root
 
 - name: Install Vagrant on Mac OS X
   community.general.homebrew_cask:
     name: vagrant
     state: present
-  when: >-
-    ansible_facts["os_family"] == "Darwin"
+  when: 
+    - vagrant_check.rc != 0
+    - ansible_facts["os_family"] == "Darwin"
   become: false
 
 - name: Install Vagrant disk size plugin
@@ -61,8 +74,9 @@
         vagrant plugin install vagrant-disksize
         >&2 echo Plugin installed
       }
-  when: >-
-    ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin"
+  when:
+    - vagrant_check.rc != 0
+    - ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin"
   register: vagrant_disksize
   changed_when: >-
     "Plugin installed" in vagrant_disksize.stderr

tasks/install-virtualbox.yml

@@ -1,11 +1,24 @@
+- name: Check if Virtualbox is already installed
+  ansible.builtin.command: which virtualbox
+  register: virtualbox_check
+  ignore_errors: true
+
+- name: Skip installation if Virtualbox is already installed
+  ansible.builtin.debug:
+    msg: "Virtualbox is already installed at {{ virtualbox_check.stdout }}"
+  when: virtualbox_check.rc == 0
+
 - name: Fail on unsupported distribution
   ansible.builtin.fail:
     msg: "Your operating system is not supported.  Please install Vagrant on your system using the method documented by the Vagrant project."
-  when: >-
-    not (ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin")
+  when: 
+    - virtualbox_check.rc != 0 
+    - not (ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin")
 
 - name: Install VirtualBox on Debian-based systems
-  when: ansible_facts["os_family"] == "Debian"
+  when: 
+    - virtualbox_check.rc != 0
+    - ansible_facts["os_family"] == "Debian"
   block:
     - name: Prevent System76 packages from screwing with VirtualBox
       ansible.builtin.copy:
@@ -78,7 +91,9 @@
         "VirtualBox is now activated properly" in vbox.stderr
 
 - name: Install VirtualBox on Fedora systems
-  when: ansible_facts["distribution"] == "Fedora"
+  when: 
+    - virtualbox_check.rc != 0
+    - ansible_facts["distribution"] == "Fedora"
   block:
     - name: Import VirtualBox RPM key
       ansible.builtin.rpm_key:
@@ -127,7 +142,9 @@
         "VirtualBox is now activated properly" in vbox.stderr
 
 - name: Install VirtualBox on Mac OS X
-  when: ansible_facts["os_family"] == "Darwin"
+  when: 
+    - virtualbox_check.rc != 0
+    - ansible_facts["os_family"] == "Darwin"
   block:
     - community.general.homebrew_cask:
         name: virtualbox

tasks/setup-observability-stack-host-reqs.yml

@@ -4,12 +4,12 @@
     pkg: python3-pip
     state: present
 - name: Install OpenShift pip module
-  ansible.builtin.pip:
-    name: openshift
+  ansible.builtin.apt:
+    name: python3-openshift
     state: present
 - name: Install Kubernetes pip module
-  ansible.builtin.pip:
-    name: kubernetes
+  ansible.builtin.apt:
+    name: python3-kubernetes
     state: present
 - block:
     - import_tasks: ../../tasks/install-helm.yml

vagrant/Vagrantfile

@@ -3,11 +3,11 @@ VAGRANTFILE_API_VERSION = "2"
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box = "ubuntu/jammy64"
   # Use config.vm.box = "alvistack/fedora-39" for Fedora.
-  config.disksize.size = '50GB'
+  config.disksize.size = '100GB'
 
   config.vm.provider :virtualbox do |v|
     v.name = "observability"
-    v.memory = 4096
+    v.memory = 16384
     v.cpus = 2
     v.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
     v.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
@@ -20,6 +20,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.network :private_network, ip: "192.168.56.42"
   config.vm.network "forwarded_port", guest: 32090, host: 32090
   config.vm.network "forwarded_port", guest: 32091, host: 32091
+  config.vm.network "forwarded_port", guest: 32080, host: 32080
 
   # Set the name of the VM. See: http://stackoverflow.com/a/17864388/100134
   config.vm.define :observability do |observability|
@@ -30,4 +31,4 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     ansible.become = true
   end
 
-end
+end

vars/k3s.yml

@@ -1,9 +1,13 @@
+# Path on the host where the local provisioner will persist data for the pods
+# of the observability stack.
+persistence_volume_path: ~/k3s/volumes
+
 k8s_version: "v1.29"
 k3s_cluster_cidrs:
   - 10.42.0.0/16
   - 2001:cafe:42::/56
 k3s_service_cidrs:
   - 10.43.0.0/16 # services
   - 2001:cafe:43::/112
-k3s_INSTALL_K3S_EXEC: "--flannel-ipv6-masq --cluster-cidr={{ k3s_cluster_cidrs | join(',') }} --service-cidr={{ k3s_service_cidrs | join(',') }}"
+k3s_INSTALL_K3S_EXEC: "--flannel-ipv6-masq --cluster-cidr={{ k3s_cluster_cidrs | join(',') }} --service-cidr={{ k3s_service_cidrs | join(',') }} --default-local-storage-path={{ persistence_volume_path | expanduser }}"
 k3s_config_file: /etc/rancher/k3s/k3s.yaml

vars/prometheus.yml

@@ -0,0 +1,5 @@
+# Prometheus configuration for storage and ram.
+# To learn more about the notation, read: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes
+prometheus:
+  ram_request: 8Gi
+  storage_request: 80Gi