Add Ansible Script to Allow Nodes to Keep Their Own External Data Up-to-date

Linux/external-data-mirror/ansible/roles/mirror-data/tasks/main.yml

@@ -14,7 +14,7 @@
   when: connected.stdout != "success"
 
 - name: Mirror the external data from the main server in a volume (this may take a while).
-  ansible.builtin.command: "rsync -az --perms -o -g {{ main_server_hostname }}:/srv/{{ main_data_srv_dir }}/ftp/external-data/MD5/ /external-data/MD5/"
+  ansible.builtin.command: "rsync -azvW --perms -o -g {{ main_server_hostname }}:/srv/{{ main_data_srv_dir }}/ftp/external-data/MD5/ /external-data/MD5/"
 
 - name: Copy the data update script onto the mirror machine.
   ansible.builtin.copy:

Linux/external-data-mirror/ansible/roles/mirror-data/tasks/update-external-data.sh

@@ -9,7 +9,7 @@ printf "%(%H:%M:%S)T "
 
 if [ -z "${RSYNC_PROCESS_IDS}" ]; then
         echo "running rsync..."
-        rsync -az --perms -o -g  $SERVER_IP:/srv/$FTP_SRV_DIR/ftp/external-data/MD5/ /external-data/MD5/
+        rsync -azvW --perms -o -g  $SERVER_IP:/srv/$FTP_SRV_DIR/ftp/external-data/MD5/ /external-data/MD5/
 else
         echo "rsync is already running. Skipping this time..."
 fi

Linux/jenkins-node/ansible/jenkins-agent-production.yml

@@ -3,20 +3,24 @@
   vars:
     deploy_type: production
     jenkins_url: https://builds.mantidproject.org
+    data_server_hostname: 172.16.114.127
     pip_install_packages:
       - name: docker
 
   roles:
     - role: setup
       tags: "initial-setup"
     - role: interactive_users
-      tags: "initial-setup"      
+      tags: "initial-setup"
     - role: geerlingguy.pip
       become: yes
-      tags: "initial-setup"      
+      tags: "initial-setup"
     - role: geerlingguy.docker
       become: yes
-      tags: "initial-setup"      
+      tags: "initial-setup"
+    - role: mirror-data  # ONLY WORKS FOR ISIS NODES
+      become: yes
+      tags: ["mirror", never]
     - role: agent
       become: yes
       tags: "agent"

Linux/jenkins-node/ansible/roles/agent/tasks/main.yml

@@ -9,9 +9,9 @@
     pull: yes
     shm_size: 512M
     volumes:
-      - "{{ agent_name }}:/jenkins_workdir"
-      - "{{ agent_name }}_ccache:/ccache"
-      - "{{ agent_name }}_external_data:/mantid_data"
+      - "/{{ agent_name }}/:/jenkins_workdir"
+      - "/{{ agent_name }}_ccache/:/ccache"
+      - "/{{ agent_name }}_external_data/:/mantid_data"
     env:
       JENKINS_AGENT_NAME: "{{ agent_name }}"
       JENKINS_SECRET: "{{ agent_secret }}"
@@ -29,9 +29,9 @@
     pull: yes
     shm_size: 512M
     volumes:
-      - "{{ agent_name }}:/jenkins_workdir"
-      - "{{ agent_name }}_ccache:/ccache"
-      - "{{ agent_name }}_external_data:/mantid_data"
+      - "/{{ agent_name }}/:/jenkins_workdir"
+      - "/{{ agent_name }}_ccache/:/ccache"
+      - "/{{ agent_name }}_external_data/:/mantid_data"
     env:
       JENKINS_AGENT_NAME: "{{ agent_name }}"
       JENKINS_SECRET: "{{ agent_secret }}"

Linux/jenkins-node/ansible/roles/mirror-data/tasks/exchange-keys.yml

@@ -0,0 +1,45 @@
+- name: Generate key pair if it does not exist
+  community.crypto.openssh_keypair:
+    force: no # Don't regenerate existing keys.
+    path: ~/.ssh/id_rsa
+
+- name: Read public key into tmp to copy over.
+  fetch:
+    src: ~/.ssh/id_rsa.pub
+    dest: /tmp/{{ ansible_hostname }}-id_rsa.pub
+    flat: yes
+
+- name: Add public key to ISIS mirror's authorized keys
+  ansible.posix.authorized_key:
+    user: "{{ ansible_user_id }}"
+    key: "{{ lookup('file','/tmp/{{ ansible_hostname }}-id_rsa.pub')}}"
+  remote_user: ubuntu
+  delegate_to: "{{ data_server_hostname }}"
+  delegate_facts: true
+
+- name: Touch the known_hosts file if it's missing
+  file:
+    path: ~/.ssh/known_hosts
+    state: touch
+    mode: 0644
+
+- name: Check if known_hosts contains existing server fingerprint
+  command: ssh-keygen -F {{ data_server_hostname }}
+  register: key_exists
+  failed_when: key_exists.stderr != ''
+  changed_when: False
+
+- name: Scan for existing remote ssh fingerprint
+  command: ssh-keyscan -T5 {{ data_server_hostname }}
+  register: keyscan
+  failed_when: keyscan.rc != 0 or keyscan.stdout == ''
+  changed_when: False
+  when: key_exists.rc == 1
+
+- name: Copy ssh-key to local known_hosts
+  lineinfile:
+    name: ~/.ssh/known_hosts
+    create: yes
+    line: "{{ item }}"
+  when: key_exists.rc == 1
+  with_items: "{{ keyscan.stdout_lines|default([]) }}"

Linux/jenkins-node/ansible/roles/mirror-data/tasks/main.yml

@@ -0,0 +1,29 @@
+- name: Create a directory to hold the mirror of the external data.
+  ansible.builtin.file:
+    path: /{{ agent_name }}_external_data/MD5/
+    state: directory
+    mode: '0755'
+
+- name: Check if machine has SSH access to the ISIS data store.
+  ansible.builtin.command: ssh -o BatchMode=True {{ ansible_user_id }}@{{ data_server_hostname }} 'echo success'
+  register: connected
+  ignore_errors: True
+
+- name: Exchange SSH keys with linode so we can access the data.
+  import_tasks: exchange-keys.yml
+  when: connected.stdout != "success"
+
+- name: Mirror the external data from the main server in a volume (this may take a while).
+  ansible.builtin.command: "rsync -azvW --perms -o -g {{ ansible_user_id }}@{{ data_server_hostname }}:/external-data/MD5/ /{{ agent_name }}_external_data/MD5 -v"
+
+- name: Copy the data update script onto the mirror machine.
+  ansible.builtin.copy:
+    src: ./update-external-data.sh
+    dest: /{{ agent_name }}_external_data/update-external-data.sh
+    mode: '0755'
+
+- name: Create a crontab job that runs periodically to keep the data up to date.
+  ansible.builtin.cron:
+    name: Update external data
+    minute: "*/5"
+    job: /{{ agent_name }}_external_data/update-external-data.sh {{ data_server_hostname }} {{ agent_name }} {{ ansible_user_id }} >> /{{ agent_name }}_external_data/update-log.txt 2>&1

Linux/jenkins-node/ansible/roles/mirror-data/tasks/update-external-data.sh

@@ -0,0 +1,16 @@
+#! /bin/bash
+
+SERVER_IP=${1}
+HOST_NAME=${2}
+USER_NAME=${3}
+
+RSYNC_PROCESS_IDS=$(pidof rsync)
+
+printf "%(%H:%M:%S)T "
+
+if [ -z "${RSYNC_PROCESS_IDS}" ]; then
+        echo "running rsync..."
+        rsync -azvW --perms -o -g  $USER_NAME@$SERVER_IP:/external-data/MD5/ /${HOST_NAME}_external_data/MD5/
+else
+        echo "rsync is already running. Skipping this time..."
+fi