Fix IC observability stack on Ubuntu 24.04 workstations.

.gitignore

@@ -3,3 +3,4 @@ __pycache__
 .mypy_cache
 scrape_configs.yml
 !vars/scrape_configs.yml
+.venv

README.md

@@ -102,8 +102,12 @@ With the scrape configuration in place, set up the stack by running:
 command -v ansible-playbook || {
   >&2 echo Please set up and export your PATH environment variable so the
   >&2 echo installed ansible-playbook can be found by your shell.  The
-  >&2 echo command might have been deployed to "$HOME/.local/bin".
+  >&2 echo command might have been deployed to "$HOME/.local/bin" or
+  >&2 echo to "$PWD"/.venv/bin or to /usr/local/bin.
   >&2 echo The next command will fail until PATH is set up properly.
+  >&2
+  >&2 echo PATH is typically set by running a command like:
+  >&2 echo '  export PATH="$PWD/.venv.bin:$PATH"'
 }
 ansible-playbook -v playbooks/prepare-node.yml
 ```
@@ -264,6 +268,45 @@ manually-edited dashboard after you are done persisting it.
 
 ## Troubleshooting
 
+### VirtualBox won't install on Ubuntu 24.04 during bootstrapping
+
+This is a known issue.  It stems from the kernel version shipping in
+Ubuntu as of October 2025.  There is a ticket tracking it:
+
+https://bugs.launchpad.net/ubuntu/+source/virtualbox/+bug/2089861
+
+The workaround to use VirtualBox again is to downgrade your kernel to
+6.12 or older, delete the more up to date kernels from your system,
+and reboot.
+
+Example of how to downgrade to the generic, older image:
+
+```sh
+apt-get install linux-image-generic=6.8.0-86.87 \
+  linux-headers-generic=6.8.0-86.87 \
+  linux-tools-generic=6.8.0-86.87 \
+  linux-tools-common=6.8.0-86.87
+# The command above might finish with an error.
+
+# The following command deletes kernel 6.16.3, which
+# may be causing you issue.  Discover kernels versions on
+# your system by running dpkg -l | grep linux-image | grep ^ii
+# then discover all packages to remove (that match the kernel)
+# with dpkg -l | grep linux | grep ^ii | grep <version>
+apt-get purge linux-headers-6.16.3-76061603-generic \
+  linux-image-6.16.3-76061603-generic \
+  linux-modules-6.16.3-76061603-generic \
+  linux-tools-6.16.3-76061603-generic
+```
+
+After removing the offending packages, APT will properly configure
+the DKMS modules for VirtualBox, and your system will be ready to
+continue the bootstrap.  Do note that you will have to use some
+mechanism in Ubuntu (like APT pinning) to prevent the
+`linux-image-generic`, `linux-headers-generic`,
+`linux-tools-common` and `linux-tools-generic` from being upgraded
+in the future, which would break VirtualBox and APT again.
+
 ### Debugging Vagrant-provisioned VMs
 
 You can SSH into the Vagrant-provisioned VM by running the following

bootstrap.sh

@@ -1,5 +1,8 @@
 #!/bin/bash -e
 
+MYDIR=$(cd `dirname "$0"` && pwd)
+echo "$MYDIR"
+
 UNSUPPORTED=32
 FAILED=16
 MANUAL_INTERVENTION=8
@@ -20,38 +23,47 @@ got() {
     which "$1" >/dev/null 2>&1
 }
 
+mod() {
+    python3 -c 'import '"$1" >/dev/null 2>&1
+}
+
 case "$(uname -s)" in
     Linux*) {
-        if got pip3
+        if mod venv >/dev/null 2>&1
         then
-            >&2 echo '* pip found and installed.'
+            >&2 echo '* Python 3 venv found and installed.'
         else
-            if got apt-get
-            then
-                sudo apt-get install -y python3-pip || {
-                    >&2 echo "* pip installation failed."
-                    exit $FAILED
-                }
-            elif got dnf
-            then
-                sudo dnf install -y python3-pip || {
-                    >&2 echo "* pip installation failed."
-                    exit $FAILED
-                }
-            else
-                >&2 echo "Your operating system is not currently supported by this script.  Please install the Python pip package using your operating system's package manager or the documented method for your operating system."
-                exit $UNSUPPORTED
-            fi
+            >&2 echo "* Your /usr/bin/python3 does not have support for venv virtual environments.  Please update to a newer Linux distribution."
+            exit $FAILED
         fi
+        if test -x "$MYDIR/.venv/python3"
+        then
+            >&2 echo '* Python 3 virtual environment found.'
+        else
+            /usr/bin/python3 -m venv "$MYDIR/.venv" || {
+                >&2 echo "* Failed to create Python 3 virtual environment."
+                exit $FAILED
+            }
+        fi
+        export PATH="$MYDIR/.venv/bin":"$PATH"
         if got ansible
         then
             check_ansible_version
         else
-            pip3 install --user ansible-core==2.17.1 ansible || {
+            pip3 install ansible-core==2.17.1 ansible || {
                 >&2 echo "* Ansible installation failed."
                 exit $FAILED
             }
         fi
+        if mod debian
+        then
+            >&2 echo '* Python 3 Debian library found.'
+        else
+            pip3 install python-debian || {
+                >&2 echo "* Debian library for Python installation failed."
+                exit $FAILED
+            }
+        fi
     };;
     Darwin*) {
         if got brew
@@ -77,6 +89,7 @@ case "$(uname -s)" in
                 exit $FAILED
             }
         fi
+        export PATH="$HOME/.local/bin:$PATH"
     };;
     *) {
        >&2 echo "Your operating system is not currently supported by this script.  Please install the Ansible package using your operating system's package manager or the documented method for your operating system."
@@ -85,5 +98,4 @@ case "$(uname -s)" in
 esac
 
 >&2 echo "Ansible is going to run on your machine to install some software, and you will now be prompted for your user account password (what Ansible calls 'BECOME password').  This will be used to become administrator and deploy various packages needed locally; if 'sudo' does not require a password on this machine, simply hit ENTER.  Follow onscreen instructions as the playbook runs.  If the next step hangs for a long period of time, interrupt it and check that you typed your root password correctly."
-export PATH="$HOME/.local/bin:$PATH"
 ansible-playbook -v -K playbooks/prepare-local-system.yml

definitions/prometheus-scrape-targets.yml.j2

@@ -85,11 +85,7 @@ spec:
           if criteria.get("dc_id")
         %}&dc_id={{ criteria.dc_id | urlencode() }}{%
           endif
-        %}{%
-          if criteria.get("subnet_id")
-        %}&subnet_id={{ criteria.subnet_id | urlencode() }}{%
-          endif
-        %}
+        %}&subnet_id={{ subnet_id | urlencode() }}
 
       refreshInterval: 15s
   honorLabels: false

tasks/install-vagrant.yml

@@ -4,12 +4,45 @@
   when: >-
     not (ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora" or ansible_facts["os_family"] == "Darwin")
 
+- name: Add Vagrant repository
+  ansible.builtin.deb822_repository:
+    name: vagrant
+    types: deb
+    uris: https://apt.releases.hashicorp.com
+    suites: "{{ ansible_distribution_release }}"
+    components:
+      - main
+    signed_by: https://apt.releases.hashicorp.com/gpg
+  when: >-
+    ansible_facts["os_family"] == "Debian"
+  become: true
+  become_user: root
+  register: debian_repo
+- name: Install Vagrant
+  ansible.builtin.package:
+    name: vagrant
+    state: present
+    update_cache: "{{ debian_repo.changed }}"
+  when: >-
+    ansible_facts["os_family"] == "Debian"
+  become: true
+  become_user: root
+
+- name: Add Vagrant repository
+  ansible.builtin.get_url:
+    url: https://rpm.releases.hashicorp.com/fedora/hashicorp.repo
+    dest: /etc/yum.repos.d/hashicorp.repo
+    checksum: sha256:566af9d3935185bbcb05bba8a9fff8959f0cc8e95153f15378e6fc157cb57ae1
+  when: >-
+    ansible_facts["distribution"] == "Fedora"
+  become: true
+  become_user: root
 - name: Install Vagrant
   ansible.builtin.package:
     name: vagrant
     state: present
   when: >-
-    ansible_facts["os_family"] == "Debian" or ansible_facts["distribution"] == "Fedora"
+    ansible_facts["distribution"] == "Fedora"
   become: true
   become_user: root
 

tasks/install-virtualbox.yml

@@ -7,31 +7,63 @@
 - name: Install VirtualBox on Debian-based systems
   when: ansible_facts["os_family"] == "Debian"
   block:
-    - name: Install VirtualBox
-      ansible.builtin.package:
-        name: virtualbox
-        state: present
+    - name: Prevent System76 packages from screwing with VirtualBox
+      ansible.builtin.copy:
+        dest: "{{ item }}-no-vbox"
+        content: |
+          Package: virtualbox*
+          Pin: release o=LP-PPA-system76-dev-stable
+          Pin-Priority: -1
+
+          Package: virtualbox*
+          Pin: release o=LP-PPA-system76-dev-pre-stable
+          Pin-Priority: -1
+
+          Package: virtualbox
+          Pin: release o=LP-PPA-system76-dev-stable
+          Pin-Priority: -1
+
+          Package: virtualbox
+          Pin: release o=LP-PPA-system76-dev-pre-stable
+          Pin-Priority: -1
+      with_first_found:
+        - files:
+            - /etc/apt/preferences.d/system76-apt-preferences
+          skip: true
       become: true
       become_user: root
-    - name: Install VirtualBox DKMS
+    - name: Install VirtualBox
       ansible.builtin.package:
-        name: virtualbox-dkms
+        name:
+          - virtualbox
+          - virtualbox-dkms
         state: present
       become: true
       become_user: root
     - name: Enable VirtualBox DKMS
       shell: |
         if modprobe vboxdrv && modprobe vboxnetadp && modprobe vboxnetflt
         then
-          >&2 echo VirtualBox is now activated properly
+          >&2 echo VirtualBox is activated properly
           exit
         else
-          output=$(dkms autoinstall 2>&1)
-          if echo "$output" | grep -q "Please.install.the"
+          output=$(dkms autoinstall 2>&1) ; ret=$? ; >&2 echo "$output"
+          if [ "$ret" != "0" ] && echo "$output" | grep -q "Please.install.the"
           then
             kernel=$(echo "$output" | grep "Please.install.the" | awk ' { print $4 } ')
             apt-get install -y "$kernel"
-            dkms autoinstall
+            dkms autoinstall >&2 ; ret=$?
+            if [ "$ret" != "0" ]
+            then
+              log=$(find /var/lib/dkms/virtualbox -mmin -5 -name make.log)
+              >&2 cat "$log"
+              exit $ret
+            fi
+          elif [ "$ret" != "0" ]
+          then
+            log=$(find /var/lib/dkms/virtualbox -mmin -5 -name make.log)
+            >&2 cat "$log"
+            exit $ret
           fi
           modprobe vboxdrv && modprobe vboxnetadp && modprobe vboxnetflt || {
             exit 16
@@ -41,6 +73,9 @@
         fi
       become: true
       become_user: root
+      register: vbox
+      changed_when: >-
+        "VirtualBox is now activated properly" in vbox.stderr
 
 - name: Install VirtualBox on Fedora systems
   when: ansible_facts["distribution"] == "Fedora"
@@ -74,15 +109,22 @@
       shell: |
         if modprobe vboxdrv && modprobe vboxnetadp && modprobe vboxnetflt
         then
-          >&2 echo VirtualBox is now activated properly
+          >&2 echo VirtualBox is activated properly
           exit
         else
-          >&2 echo Currently this playbook does not know how to compile the VirtualBox kernel modules for {{ ansible_facts["distribution"] }}.
-          >&2 echo Please use DKMS to install the kernel modules, then load modules using modprobe vboxdrv vboxnetadp vboxnetflt
-          exit 32
+          dkms autoinstall || {
+            ret=$?
+            log=$(find /var/lib/dkms/virtualbox -mmin -5 -name make.log)
+            >&2 cat "$log"
+            exit $ret
+          }
+          >&2 echo VirtualBox is now activated properly
         fi
       become: true
       become_user: root
+      register: vbox
+      changed_when: >-
+        "VirtualBox is now activated properly" in vbox.stderr
 
 - name: Install VirtualBox on Mac OS X
   when: ansible_facts["os_family"] == "Darwin"