DNM: Support multiple Nova Compute Ironic instances

ansible/roles/nova-cell/defaults/main.yml

@@ -72,6 +72,8 @@ nova_cell_services:
     volumes: "{{ nova_compute_ironic_default_volumes + nova_compute_ironic_extra_volumes }}"
     dimensions: "{{ nova_compute_ironic_dimensions }}"
     healthcheck: "{{ nova_compute_ironic_healthcheck }}"
+    iterate: "{{ true if nova_multi_compute_ironic_config | length > 0 else false }}"
+    iterate_var: "{{ nova_multi_compute_ironic_config | length }}"
 
 ####################
 # Config Validate
@@ -442,7 +444,7 @@ nova_compute_default_volumes:
   - "{% if enable_shared_var_lib_nova_mnt | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}"
   - "{{ kolla_dev_repos_directory ~ '/nova:/dev-mode/nova' if nova_dev_mode | bool else '' }}"
 nova_compute_ironic_default_volumes:
-  - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro"
+  - "{{ node_config_directory }}/nova-compute-ironic{{ '-' ~ item if item|default(0)|int > 0 else '' }}/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "kolla_logs:/var/log/kolla/"
@@ -612,3 +614,14 @@ nova_pci_passthrough_whitelist: "{{ enable_neutron_sriov | bool | ternary(neutro
 nova_libvirt_cleanup_running_vms_fatal: true
 # Whether to remove Docker volumes.
 nova_libvirt_cleanup_remove_volumes: false
+
+##############################
+# Nova Compute Ironic scaling
+##############################
+
+# The following option can be used to deploy multiple instances of Nova
+# Compute Ironic per host. This is useful for deployments with a large
+# number of baremetal nodes (>200). Please see the advanced documentation
+# section for further details.
+
+nova_multi_compute_ironic_config: []

ansible/roles/nova-cell/handlers/main.yml

@@ -161,16 +161,21 @@
   vars:
     service_name: "nova-compute-ironic"
     service: "{{ nova_cell_services[service_name] }}"
+    item: "{{ changed_container.item }}"
+    container_name_postfix: "{{ '' if changed_container == 'legacy' else '_' ~ item }}"
   become: true
   kolla_container:
     action: "recreate_or_restart_container"
     common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
+    name: "{{ service.container_name + container_name_postfix }}"
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes | reject('equalto', '') | list }}"
     dimensions: "{{ service.dimensions }}"
     healthcheck: "{{ service.healthcheck | default(omit) }}"
+  loop_control:
+    loop_var: changed_container
+  loop: "{{ nova_compute_ironic_changed_containers | default(['legacy']) }}"
 
 # nova-compute-fake is special. It will start multi numbers of container
 # so put all variables here rather than defaults/main.yml file

ansible/roles/nova-cell/tasks/cleanup.yml

@@ -0,0 +1,26 @@
+---
+- name: Stop and remove containers for unmanaged services
+  become: true
+  vars:
+    service_name: "nova-compute-ironic"
+    service: "{{ nova_cell_services[service_name] }}"
+  kolla_container:
+    action: "stop_and_remove_container"
+    name: "{{ service.container_name + '_' ~ item }}"
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - service.iterate | bool
+    - inventory_hostname not in groups['nova-compute-ironic-' ~ item]
+
+- name: Removing config for unmanaged services
+  vars:
+    service_name: "nova-compute-ironic"
+    service: "{{ nova_cell_services[service_name] }}"
+  ansible.builtin.file:
+    path: "{{ node_config_directory }}/{{ service_name + '-' ~ item }}"
+    state: "absent"
+  become: true
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - service.iterate | bool
+    - inventory_hostname not in groups['nova-compute-ironic-' ~ item]

ansible/roles/nova-cell/tasks/config-compute-ironic.yml

@@ -0,0 +1,66 @@
+---
+- name: "Ensuring config directories exist for Nova Compute Ironic container {{ item }}"
+  vars:
+    nova_compute_ironic_id: "{{ item }}"
+  ansible.builtin.file:
+    path: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+
+- name: "Copying over config.json files for Nova Compute Ironic container {{ item }}"
+  vars:
+    nova_compute_ironic_id: "{{ item }}"
+  ansible.builtin.template:
+    src: "nova-compute-ironic.json.j2"
+    dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/config.json"
+    mode: "0660"
+  become: true
+
+- name: "Get Nova Compute Ironic multi-instance configuration for instance {{ item }}"
+  ansible.builtin.set_fact:
+    nova_multi_compute_ironic_instance_config: "{{ nova_multi_compute_ironic_config[(item | int) - 1] }}"
+
+- name: "Generate config files for Nova Compute Ironic {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+    nova_compute_shard_key: "{{ nova_multi_compute_ironic_instance_config.get('shard_key', 'default') }}"
+    nova_compute_conductor_group: "{{ nova_multi_compute_ironic_instance_config.get('conductor_group', 'default') }}"
+    nova_compute_ironic_custom_host: "{{ nova_multi_compute_ironic_instance_config.get('custom_host', '') }}"
+    nova_compute_ironic_id: "{{ item }}"
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/nova.conf.j2"
+      - "{{ node_custom_config }}/global.conf"
+      - "{{ node_custom_config }}/nova.conf"
+      - "{{ node_custom_config }}/nova/{{ service_name }}.conf"
+      - "{{ node_custom_config }}/nova/{{ service_name }}-{{ item }}.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/nova.conf"
+      - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ service_name }}-{{ item }}.conf"
+    dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/nova.conf"
+    mode: "0660"
+  become: true
+
+- name: "Copying over existing policy file for item {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+  ansible.builtin.template:
+    src: "{{ nova_policy_file_path }}"
+    dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/{{ nova_policy_file }}"
+    mode: "0660"
+  become: true
+  when:
+    - nova_policy_file is defined
+
+- name: "Copying over vendordata file for Nova Compute Ironic {{ item }}"
+  vars:
+    service_name: "nova-compute-ironic"
+  ansible.builtin.copy:
+    src: "{{ vendordata_file_path }}"
+    dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/vendordata.json"
+    mode: "0660"
+  become: true
+  when:
+    - vendordata_file_path is defined

ansible/roles/nova-cell/tasks/config.yml

@@ -7,6 +7,8 @@
     owner: "{{ config_owner_user }}"
     group: "{{ config_owner_group }}"
     mode: "0770"
+  when:
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - include_tasks: copy-certs.yml
@@ -56,6 +58,8 @@
     src: "{{ item.key }}.json.j2"
     dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
     mode: "0660"
+  when:
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over nova.conf
@@ -74,8 +78,18 @@
     mode: "0660"
   when:
     - item.key in nova_cell_services_require_nova_conf
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
+- name: Copying over config for Nova compute Ironic multi-instance
+  vars:
+    service: "{{ nova_cell_services['nova-compute-ironic'] }}"
+  ansible.builtin.include_tasks: config-compute-ironic.yml
+  loop: "{{ range(1, nova_multi_compute_ironic_config | length + 1) | list }}"
+  when:
+    - nova_multi_compute_ironic_config | length > 0
+    - inventory_hostname in groups[service.group + '-' ~ item]
+
 - name: Copying over Nova compute provider config
   become: true
   vars:
@@ -187,6 +201,7 @@
   when:
     - nova_policy_file is defined
     - item.key in nova_cell_services_require_policy_json
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}"
 
 - name: Copying over vendordata file to containers
@@ -200,6 +215,7 @@
   when:
     - vendordata_file_path is defined
     - service | service_enabled_and_mapped_to_host
+    - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0
   with_items:
     - nova-compute
     - nova-compute-ironic

ansible/roles/nova-cell/tasks/deploy.yml

@@ -4,6 +4,10 @@
 
 - import_tasks: version-check.yml
 
+# NOTE(dougszu): Currently only covers nova-compute-ironic
+- name: Cleanup stale configuration
+  ansible.builtin.import_tasks: cleanup.yml
+
 - import_tasks: config-host.yml
 
 - import_tasks: config.yml

ansible/roles/nova-cell/tasks/wait_discover_computes.yml

@@ -79,9 +79,11 @@
     # configure for [DEFAULT] host in nova.conf.
     ironic_compute_service_hosts: >-
       {{ ironic_computes_in_batch |
-         map('extract', hostvars) | json_query('[].nova_compute_ironic_custom_host || [].ansible_facts.hostname') |
-         map('regex_replace', '^(.*)$', '\1-ironic') |
-         list }}
+         map('extract', hostvars) |
+         json_query(
+           '{classic: [].nova_compute_ironic_custom_host || [].ansible_facts.hostname,
+             multi: [].nova_multi_compute_ironic_config}') |
+         get_expected_ironic_compute_services }}
     expected_compute_service_hosts: "{{ virt_compute_service_hosts + ironic_compute_service_hosts }}"
 
 - name: Include discover_computes.yml

ansible/roles/nova-cell/templates/nova.conf.j2

@@ -8,9 +8,27 @@ state_path = /var/lib/nova
 
 allow_resize_to_same_host = true
 
+{% set nova_compute_shard_key = nova_compute_shard_key | default("") %}
+{% set nova_compute_conductor_group = nova_compute_conductor_group | default("") %}
+{% set nova_compute_ironic_custom_host = nova_compute_ironic_custom_host | default("") %}
 {% if service_name == "nova-compute-ironic" %}
-host={{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic
+{%     if nova_compute_ironic_custom_host | length > 0 %}
+{#         NOTE(dougszu): This is provided for backwards compatibility #}
+{%         set host_value = nova_compute_ironic_custom_host ~ "-ironic" %}
+{%     elif nova_compute_conductor_group | length > 0 or nova_compute_shard_key | length > 0 %}
+{%         set host_value = (nova_compute_conductor_group ~ "-" ~ nova_compute_shard_key) ~ "-ironic" %}
+{%     else %}
+{#         NOTE(dougszu): This is to support legacy behaviour #}
+{%         set host_value = ansible_facts.hostname ~ "-ironic" %}
+{%     endif %}
+host = {{ host_value }}
+
+{%     if nova_compute_ironic_id is defined %}
+log_file = /var/log/kolla/nova/nova-compute-ironic-{{ nova_compute_ironic_id }}.log
+{%     else %}
 log_file = /var/log/kolla/nova/nova-compute-ironic.log
+{%     endif %}
+
 compute_driver = ironic.IronicDriver
 ram_allocation_ratio = 1.0
 reserved_host_memory_mb = 0
@@ -96,6 +114,19 @@ project_name = service
 user_domain_name = {{ default_user_domain_name }}
 project_domain_name = {{ default_project_domain_name }}
 endpoint_override = {{ ironic_internal_endpoint }}/v1
+
+{%     if nova_compute_shard_key is not in [none, 'default'] %}
+shard = {{ nova_compute_shard_key }}
+{%     endif %}
+{%     if nova_compute_conductor_group is not in [none, 'default'] %}
+conductor_group = {{ nova_compute_conductor_group }}
+{%     endif %}
+{%     if nova_compute_conductor_group is not in [none, 'default'] and nova_compute_shard_key is not in [none, 'default'] %}
+# NOTE(dougszu): In this case only, Nova Compute Ironic won't start unless the peer_list
+# is populated. peer_list is a deprecated config option and should never have more than
+# one host in it.
+peer_list = {{ nova_compute_conductor_group }}-ironic
+{%     endif %}
 {% endif %}
 
 [oslo_concurrency]
@@ -191,7 +222,7 @@ driver = noop
 [oslo_messaging_rabbit]
 use_queue_manager = true
 {% if service_name == "nova-compute-ironic" %}
-hostname = {{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic
+hostname = {{ host_value }}
 {% endif %}
 heartbeat_in_pthread = false
 {% if om_enable_rabbitmq_tls | bool %}

ansible/roles/service-cert-copy/tasks/iterated.yml

@@ -0,0 +1,10 @@
+---
+- name: "Copying over extra CA certificates for {{ project_name }}"
+  become: true
+  ansible.builtin.copy:
+    src: "{{ kolla_certificates_dir }}/ca/"
+    dest: "{{ node_config_directory }}/{{ outer_item.key }}-{{ item }}/ca-certificates"
+    mode: "0644"
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - inventory_hostname in groups[service.group + '-' ~ item]

ansible/roles/service-cert-copy/tasks/main.yml

@@ -7,8 +7,19 @@
     mode: "0644"
   when:
     - kolla_copy_ca_into_containers | bool
+    - not (item.value.iterate | default(False)) | bool
   with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}"
 
+# NOTE(dougszu): For current iterable services, we only need CA certs
+- name: "Check containers that require iteration for {{ kolla_role_name | default(project_name) }}"
+  vars:
+    service: "{{ outer_item.value }}"
+  ansible.builtin.include_tasks: iterated.yml
+  loop: "{{ project_services | select_services_enabled_and_mapped_to_host | dict2items }}"
+  loop_control:
+    loop_var: outer_item
+  when: (service.iterate | default(False)) | bool
+
 - name: "{{ project_name }} | Copying over backend internal TLS certificate"
   vars:
     certs:

ansible/roles/service-check-containers/tasks/iterated.yml

@@ -23,6 +23,11 @@
     command: "{{ service.command | default(omit) }}"
     cgroupns_mode: "{{ service.cgroupns_mode | default(omit) }}"
   loop: "{{ range(1, (iterate_count | int) + 1) | list }}"
+  when:
+    # NOTE(dougszu): Due to issues with nova-compute-ironic HA, we can't have
+    # more than one instance for a given 'item'. Eg. nova_compute_ironic_1
+    # should never run on more than one host.
+    - service_name != 'nova-compute-ironic' or (service_name == 'nova-compute-ironic' and inventory_hostname in groups['nova-compute-ironic-' + item | string])
   register: container_check
 
 # NOTE(yoctozepto): Must be a separate task because one cannot see the whole

ansible/roles/service-precheck/tasks/iterated.yml

@@ -0,0 +1,31 @@
+---
+- name: "Validate inventory groups for {{ project_name }}"
+  vars:
+    service_name: "{{ item.key }}"
+    service: "{{ item.value }}"
+  ansible.builtin.fail:
+    msg: >-
+      Ansible inventory does not contain the expected group {{ service.group }}
+      for service {{ service_name }} in {{ project_name }}.
+  loop: "{{ query('dict', service_precheck_services) }}"
+  when:
+    - not (service.iterate | default(False)) | bool
+    - "'group' in service"
+    - service.group not in groups
+  loop_control:
+    label: "{{ service_name }}"
+
+- name: "Validate inventory groups (iterated) for {{ project_name }}"
+  vars:
+    service_name: "{{ outer_item.key }}"
+    service: "{{ outer_item.value }}"
+    service_group: "{{ service.group + '-' + item | string }}"
+  ansible.builtin.fail:
+    msg: >-
+      Ansible inventory does not contain the expected group {{ service_group }}
+      for service {{ service_name }} in {{ project_name }}.
+  loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}"
+  when:
+    - (service.iterate | default(False)) | bool
+    - "'group' in service"
+    - service_group not in groups

ansible/roles/service-precheck/tasks/main.yml

@@ -9,7 +9,17 @@
       for service {{ service_name }} in {{ project_name }}.
   loop: "{{ query('dict', service_precheck_services) }}"
   when:
+    - not (service.iterate | default(False)) | bool
     - "'group' in service"
     - service.group not in groups
   loop_control:
     label: "{{ service_name }}"
+
+- name: Include tasks
+  vars:
+    service: "{{ outer_item.value }}"
+  ansible.builtin.include_tasks: iterated.yml
+  loop: "{{ query('dict', service_precheck_services) }}"
+  loop_control:
+    loop_var: outer_item
+  when: (service.iterate | default(False)) | bool