From d53f6a28e6785364704e53504e549ea74e00c724 Mon Sep 17 00:00:00 2001
From: Caila Finn <caila.finn@stfc.ac.uk>
Date: Tue, 16 Sep 2025 12:24:11 +0100
Subject: [PATCH 1/3] Mirror data from the ISIS server

---
 .../ansible/jenkins-agent-production.yml      |  6 ++-
 .../roles/mirror-data/tasks/exchange-keys.yml | 44 +++++++++++++++++++
 .../ansible/roles/mirror-data/tasks/main.yml  | 29 ++++++++++++
 .../mirror-data/tasks/update-external-data.sh | 14 ++++++
 4 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml
 create mode 100644 Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml
 create mode 100644 Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh

diff --git a/Linux/jenkins-node/ansible/jenkins-agent-production.yml b/Linux/jenkins-node/ansible/jenkins-agent-production.yml
index d3c6e2d..a93e1ef 100644
--- a/Linux/jenkins-node/ansible/jenkins-agent-production.yml
+++ b/Linux/jenkins-node/ansible/jenkins-agent-production.yml
@@ -3,6 +3,7 @@
   vars:
     deploy_type: production
     jenkins_url: https://builds.mantidproject.org
+    data_server_hostname: 130.246.80.136
     pip_install_packages:
       - name: docker
 
@@ -16,7 +17,10 @@
       tags: "initial-setup"      
     - role: geerlingguy.docker
       become: yes
-      tags: "initial-setup"      
+      tags: "initial-setup"     
+    - role: mirror-data  # ONLY WORKS FOR ISIS NODES
+      become: yes
+      tags: ["mirror", never]
     - role: agent
       become: yes
       tags: "agent"
diff --git a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml
new file mode 100644
index 0000000..27ce3d6
--- /dev/null
+++ b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml
@@ -0,0 +1,44 @@
+- name: Generate key pair if it does not exist
+  community.crypto.openssh_keypair:
+    force: no # Don't regenerate existing keys.
+    path: ~/.ssh/id_rsa
+
+- name: Read public key into tmp to copy over.
+  fetch:
+    src: ~/.ssh/id_rsa.pub
+    dest: /tmp/{{ ansible_hostname }}-id_rsa.pub
+    flat: yes
+
+- name: Add public key to ISIS mirror's authorized keys
+  ansible.posix.authorized_key:
+    user: ubuntu
+    key: "{{ lookup('file','/tmp/{{ ansible_hostname }}-id_rsa.pub')}}"
+  remote_user: ubuntu
+  delegate_to: "{{ data_server_hostname }}"
+
+- name: Touch the known_hosts file if it's missing
+  file:
+    path: ~/.ssh/known_hosts
+    state: touch
+    mode: 0644
+
+- name: Check if known_hosts contains existing server fingerprint
+  command: ssh-keygen -F {{ data_server_hostname }}
+  register: key_exists
+  failed_when: key_exists.stderr != ''
+  changed_when: False
+
+- name: Scan for existing remote ssh fingerprint
+  command: ssh-keyscan -T5 {{ data_server_hostname }}
+  register: keyscan
+  failed_when: keyscan.rc != 0 or keyscan.stdout == ''
+  changed_when: False
+  when: key_exists.rc == 1
+
+- name: Copy ssh-key to local known_hosts
+  lineinfile:
+    name: ~/.ssh/known_hosts
+    create: yes
+    line: "{{ item }}"
+  when: key_exists.rc == 1
+  with_items: "{{ keyscan.stdout_lines|default([]) }}"
diff --git a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml
new file mode 100644
index 0000000..f8db48d
--- /dev/null
+++ b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml
@@ -0,0 +1,29 @@
+- name: Create a directory to hold the mirror of the external data.
+  ansible.builtin.file:
+    path: /external-data/MD5/
+    state: directory
+    mode: '0755'
+
+- name: Check if machine has SSH access to the ISIS data store.
+  ansible.builtin.command: ssh -o BatchMode=True ubuntu@{{ data_server_hostname }} 'echo success'
+  register: connected
+  ignore_errors: True
+
+- name: Exchange SSH keys with linode so we can access the data.
+  import_tasks: exchange-keys.yml
+  when: connected.stdout != "success"
+
+- name: Mirror the external data from the main server in a volume (this may take a while).
+  ansible.builtin.command: "rsync -az --perms -o -g {{ data_server_hostname }}:/external-data/MD5/ /external-data/MD5/"
+
+- name: Copy the data update script onto the mirror machine.
+  ansible.builtin.copy:
+    src: ./update-external-data.sh
+    dest: /external-data/update-external-data.sh
+    mode: '0755'
+
+- name: Create a crontab job that runs periodically to keep the data up to date.
+  ansible.builtin.cron:
+    name: Update external data
+    minute: "*/5"
+    job: /external-data/update-external-data.sh {{ data_server_hostname }} >> /external-data/update-log.txt 2>&1
diff --git a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh
new file mode 100644
index 0000000..4a52d43
--- /dev/null
+++ b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh
@@ -0,0 +1,14 @@
+#! /bin/bash
+
+SERVER_IP=${1}
+
+RSYNC_PROCESS_IDS=$(pidof rsync)
+
+printf "%(%H:%M:%S)T "
+
+if [ -z "${RSYNC_PROCESS_IDS}" ]; then
+        echo "running rsync..."
+        rsync -az --perms -o -g  $SERVER_IP:/external-data/MD5/ /external-data/MD5/
+else
+        echo "rsync is already running. Skipping this time..."
+fi

From c4a44c7ca62712cf50990a43a6b94703035cde5d Mon Sep 17 00:00:00 2001
From: Caila Finn <caila.finn@stfc.ac.uk>
Date: Thu, 18 Sep 2025 10:02:23 +0100
Subject: [PATCH 2/3] Fix SSH connection issues

---
 Linux/jenkins-node/ansible/jenkins-agent-production.yml     | 2 +-
 .../ansible/roles/mirror-data/tasks/exchange-keys.yml       | 3 ++-
 Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml | 6 +++---
 .../ansible/roles/mirror-data/tasks/update-external-data.sh | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/Linux/jenkins-node/ansible/jenkins-agent-production.yml b/Linux/jenkins-node/ansible/jenkins-agent-production.yml
index a93e1ef..d37a1dc 100644
--- a/Linux/jenkins-node/ansible/jenkins-agent-production.yml
+++ b/Linux/jenkins-node/ansible/jenkins-agent-production.yml
@@ -3,7 +3,7 @@
   vars:
     deploy_type: production
     jenkins_url: https://builds.mantidproject.org
-    data_server_hostname: 130.246.80.136
+    data_server_hostname: 172.16.114.127
     pip_install_packages:
       - name: docker
 
diff --git a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml
index 27ce3d6..7337d75 100644
--- a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml
+++ b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml
@@ -11,10 +11,11 @@
 
 - name: Add public key to ISIS mirror's authorized keys
   ansible.posix.authorized_key:
-    user: ubuntu
+    user: mkq48465
     key: "{{ lookup('file','/tmp/{{ ansible_hostname }}-id_rsa.pub')}}"
   remote_user: ubuntu
   delegate_to: "{{ data_server_hostname }}"
+  delegate_facts: true
 
 - name: Touch the known_hosts file if it's missing
   file:
diff --git a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml
index f8db48d..1597f56 100644
--- a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml
+++ b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml
@@ -1,11 +1,11 @@
 - name: Create a directory to hold the mirror of the external data.
   ansible.builtin.file:
-    path: /external-data/MD5/
+    path: /mantid_data/MD5/
     state: directory
     mode: '0755'
 
 - name: Check if machine has SSH access to the ISIS data store.
-  ansible.builtin.command: ssh -o BatchMode=True ubuntu@{{ data_server_hostname }} 'echo success'
+  ansible.builtin.command: ssh -o BatchMode=True mkq48465@{{ data_server_hostname }} 'echo success'
   register: connected
   ignore_errors: True
 
@@ -14,7 +14,7 @@
   when: connected.stdout != "success"
 
 - name: Mirror the external data from the main server in a volume (this may take a while).
-  ansible.builtin.command: "rsync -az --perms -o -g {{ data_server_hostname }}:/external-data/MD5/ /external-data/MD5/"
+  ansible.builtin.command: "rsync -az --perms -o -g mkq48465@{{ data_server_hostname }}:/external-data/MD5/ /mantid_data/MD5 -v"
 
 - name: Copy the data update script onto the mirror machine.
   ansible.builtin.copy:
diff --git a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh
index 4a52d43..5e308b6 100644
--- a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh
+++ b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh
@@ -8,7 +8,7 @@ printf "%(%H:%M:%S)T "
 
 if [ -z "${RSYNC_PROCESS_IDS}" ]; then
         echo "running rsync..."
-        rsync -az --perms -o -g  $SERVER_IP:/external-data/MD5/ /external-data/MD5/
+        rsync -az --perms -o -g  mkq48465@$SERVER_IP:/external-data/MD5/ /mantid_data/MD5/
 else
         echo "rsync is already running. Skipping this time..."
 fi

From ed112be25b37b50f2b5add2e65d5ba4ed88bb7be Mon Sep 17 00:00:00 2001
From: Caila Finn <caila.finn@stfc.ac.uk>
Date: Mon, 13 Oct 2025 18:14:21 +0100
Subject: [PATCH 3/3] Use mounts rather than volumes for ext data

---
 .../ansible/roles/mirror-data/tasks/main.yml         |  2 +-
 .../roles/mirror-data/tasks/update-external-data.sh  |  2 +-
 .../ansible/jenkins-agent-production.yml             |  6 +++---
 .../jenkins-node/ansible/roles/agent/tasks/main.yml  | 12 ++++++------
 .../roles/mirror-data/tasks/exchange-keys.yml        |  2 +-
 .../ansible/roles/mirror-data/tasks/main.yml         | 10 +++++-----
 .../roles/mirror-data/tasks/update-external-data.sh  |  4 +++-
 7 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/Linux/external-data-mirror/ansible/roles/mirror-data/tasks/main.yml b/Linux/external-data-mirror/ansible/roles/mirror-data/tasks/main.yml
index 7ade825..5462b66 100644
--- a/Linux/external-data-mirror/ansible/roles/mirror-data/tasks/main.yml
+++ b/Linux/external-data-mirror/ansible/roles/mirror-data/tasks/main.yml
@@ -14,7 +14,7 @@
   when: connected.stdout != "success"
 
 - name: Mirror the external data from the main server in a volume (this may take a while).
-  ansible.builtin.command: "rsync -az --perms -o -g {{ main_server_hostname }}:/srv/{{ main_data_srv_dir }}/ftp/external-data/MD5/ /external-data/MD5/"
+  ansible.builtin.command: "rsync -azvW --perms -o -g {{ main_server_hostname }}:/srv/{{ main_data_srv_dir }}/ftp/external-data/MD5/ /external-data/MD5/"
 
 - name: Copy the data update script onto the mirror machine.
   ansible.builtin.copy:
diff --git a/Linux/external-data-mirror/ansible/roles/mirror-data/tasks/update-external-data.sh b/Linux/external-data-mirror/ansible/roles/mirror-data/tasks/update-external-data.sh
index 329666b..9dcb2e7 100644
--- a/Linux/external-data-mirror/ansible/roles/mirror-data/tasks/update-external-data.sh
+++ b/Linux/external-data-mirror/ansible/roles/mirror-data/tasks/update-external-data.sh
@@ -9,7 +9,7 @@ printf "%(%H:%M:%S)T "
 
 if [ -z "${RSYNC_PROCESS_IDS}" ]; then
         echo "running rsync..."
-        rsync -az --perms -o -g  $SERVER_IP:/srv/$FTP_SRV_DIR/ftp/external-data/MD5/ /external-data/MD5/
+        rsync -azvW --perms -o -g  $SERVER_IP:/srv/$FTP_SRV_DIR/ftp/external-data/MD5/ /external-data/MD5/
 else
         echo "rsync is already running. Skipping this time..."
 fi
diff --git a/Linux/jenkins-node/ansible/jenkins-agent-production.yml b/Linux/jenkins-node/ansible/jenkins-agent-production.yml
index d37a1dc..414bf10 100644
--- a/Linux/jenkins-node/ansible/jenkins-agent-production.yml
+++ b/Linux/jenkins-node/ansible/jenkins-agent-production.yml
@@ -11,13 +11,13 @@
     - role: setup
       tags: "initial-setup"
     - role: interactive_users
-      tags: "initial-setup"      
+      tags: "initial-setup"
     - role: geerlingguy.pip
       become: yes
-      tags: "initial-setup"      
+      tags: "initial-setup"
     - role: geerlingguy.docker
       become: yes
-      tags: "initial-setup"     
+      tags: "initial-setup"
     - role: mirror-data  # ONLY WORKS FOR ISIS NODES
       become: yes
       tags: ["mirror", never]
diff --git a/Linux/jenkins-node/ansible/roles/agent/tasks/main.yml b/Linux/jenkins-node/ansible/roles/agent/tasks/main.yml
index e736e3f..6ac4503 100644
--- a/Linux/jenkins-node/ansible/roles/agent/tasks/main.yml
+++ b/Linux/jenkins-node/ansible/roles/agent/tasks/main.yml
@@ -9,9 +9,9 @@
     pull: yes
     shm_size: 512M
     volumes:
-      - "{{ agent_name }}:/jenkins_workdir"
-      - "{{ agent_name }}_ccache:/ccache"
-      - "{{ agent_name }}_external_data:/mantid_data"
+      - "/{{ agent_name }}/:/jenkins_workdir"
+      - "/{{ agent_name }}_ccache/:/ccache"
+      - "/{{ agent_name }}_external_data/:/mantid_data"
     env:
       JENKINS_AGENT_NAME: "{{ agent_name }}"
       JENKINS_SECRET: "{{ agent_secret }}"
@@ -29,9 +29,9 @@
     pull: yes
     shm_size: 512M
     volumes:
-      - "{{ agent_name }}:/jenkins_workdir"
-      - "{{ agent_name }}_ccache:/ccache"
-      - "{{ agent_name }}_external_data:/mantid_data"
+      - "/{{ agent_name }}/:/jenkins_workdir"
+      - "/{{ agent_name }}_ccache/:/ccache"
+      - "/{{ agent_name }}_external_data/:/mantid_data"
     env:
       JENKINS_AGENT_NAME: "{{ agent_name }}"
       JENKINS_SECRET: "{{ agent_secret }}"
diff --git a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml
index 7337d75..094048c 100644
--- a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml
+++ b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml
@@ -11,7 +11,7 @@
 
 - name: Add public key to ISIS mirror's authorized keys
   ansible.posix.authorized_key:
-    user: mkq48465
+    user: "{{ ansible_user_id }}"
     key: "{{ lookup('file','/tmp/{{ ansible_hostname }}-id_rsa.pub')}}"
   remote_user: ubuntu
   delegate_to: "{{ data_server_hostname }}"
diff --git a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml
index 1597f56..563e37b 100644
--- a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml
+++ b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml
@@ -1,11 +1,11 @@
 - name: Create a directory to hold the mirror of the external data.
   ansible.builtin.file:
-    path: /mantid_data/MD5/
+    path: /{{ agent_name }}_external_data/MD5/
     state: directory
     mode: '0755'
 
 - name: Check if machine has SSH access to the ISIS data store.
-  ansible.builtin.command: ssh -o BatchMode=True mkq48465@{{ data_server_hostname }} 'echo success'
+  ansible.builtin.command: ssh -o BatchMode=True {{ ansible_user_id }}@{{ data_server_hostname }} 'echo success'
   register: connected
   ignore_errors: True
 
@@ -14,16 +14,16 @@
   when: connected.stdout != "success"
 
 - name: Mirror the external data from the main server in a volume (this may take a while).
-  ansible.builtin.command: "rsync -az --perms -o -g mkq48465@{{ data_server_hostname }}:/external-data/MD5/ /mantid_data/MD5 -v"
+  ansible.builtin.command: "rsync -azvW --perms -o -g {{ ansible_user_id }}@{{ data_server_hostname }}:/external-data/MD5/ /{{ agent_name }}_external_data/MD5 -v"
 
 - name: Copy the data update script onto the mirror machine.
   ansible.builtin.copy:
     src: ./update-external-data.sh
-    dest: /external-data/update-external-data.sh
+    dest: /{{ agent_name }}_external_data/update-external-data.sh
     mode: '0755'
 
 - name: Create a crontab job that runs periodically to keep the data up to date.
   ansible.builtin.cron:
     name: Update external data
     minute: "*/5"
-    job: /external-data/update-external-data.sh {{ data_server_hostname }} >> /external-data/update-log.txt 2>&1
+    job: /{{ agent_name }}_external_data/update-external-data.sh {{ data_server_hostname }} {{ agent_name }} {{ ansible_user_id }} >> /{{ agent_name }}_external_data/update-log.txt 2>&1
diff --git a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh
index 5e308b6..1dd5f45 100644
--- a/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh
+++ b/Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh
@@ -1,6 +1,8 @@
 #! /bin/bash
 
 SERVER_IP=${1}
+HOST_NAME=${2}
+USER_NAME=${3}
 
 RSYNC_PROCESS_IDS=$(pidof rsync)
 
@@ -8,7 +10,7 @@ printf "%(%H:%M:%S)T "
 
 if [ -z "${RSYNC_PROCESS_IDS}" ]; then
         echo "running rsync..."
-        rsync -az --perms -o -g  mkq48465@$SERVER_IP:/external-data/MD5/ /mantid_data/MD5/
+        rsync -azvW --perms -o -g  $USER_NAME@$SERVER_IP:/external-data/MD5/ /${HOST_NAME}_external_data/MD5/
 else
         echo "rsync is already running. Skipping this time..."
 fi