diff --git a/ansible/roles/nova-cell/defaults/main.yml b/ansible/roles/nova-cell/defaults/main.yml index 4b023df20a..4616d08eaf 100644 --- a/ansible/roles/nova-cell/defaults/main.yml +++ b/ansible/roles/nova-cell/defaults/main.yml @@ -72,6 +72,8 @@ nova_cell_services: volumes: "{{ nova_compute_ironic_default_volumes + nova_compute_ironic_extra_volumes }}" dimensions: "{{ nova_compute_ironic_dimensions }}" healthcheck: "{{ nova_compute_ironic_healthcheck }}" + iterate: "{{ true if nova_multi_compute_ironic_config | length > 0 else false }}" + iterate_var: "{{ nova_multi_compute_ironic_config | length }}" #################### # Config Validate @@ -442,7 +444,7 @@ nova_compute_default_volumes: - "{% if enable_shared_var_lib_nova_mnt | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}" - "{{ kolla_dev_repos_directory ~ '/nova:/dev-mode/nova' if nova_dev_mode | bool else '' }}" nova_compute_ironic_default_volumes: - - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro" + - "{{ node_config_directory }}/nova-compute-ironic{{ '-' ~ item if item|default(0)|int > 0 else '' }}/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}" - "kolla_logs:/var/log/kolla/" @@ -612,3 +614,14 @@ nova_pci_passthrough_whitelist: "{{ enable_neutron_sriov | bool | ternary(neutro nova_libvirt_cleanup_running_vms_fatal: true # Whether to remove Docker volumes. nova_libvirt_cleanup_remove_volumes: false + +############################## +# Nova Compute Ironic scaling +############################## + +# The following option can be used to deploy multiple instances of Nova +# Compute Ironic per host. This is useful for deployments with a large +# number of baremetal nodes (>200). Please see the advanced documentation +# section for further details. + +nova_multi_compute_ironic_config: [] diff --git a/ansible/roles/nova-cell/handlers/main.yml b/ansible/roles/nova-cell/handlers/main.yml index 17658ef2ed..ebd5c1b40d 100644 --- a/ansible/roles/nova-cell/handlers/main.yml +++ b/ansible/roles/nova-cell/handlers/main.yml @@ -161,16 +161,21 @@ vars: service_name: "nova-compute-ironic" service: "{{ nova_cell_services[service_name] }}" + item: "{{ changed_container.item }}" + container_name_postfix: "{{ '' if changed_container == 'legacy' else '_' ~ item }}" become: true kolla_container: action: "recreate_or_restart_container" common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" + name: "{{ service.container_name + container_name_postfix }}" image: "{{ service.image }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes | reject('equalto', '') | list }}" dimensions: "{{ service.dimensions }}" healthcheck: "{{ service.healthcheck | default(omit) }}" + loop_control: + loop_var: changed_container + loop: "{{ nova_compute_ironic_changed_containers | default(['legacy']) }}" # nova-compute-fake is special. It will start multi numbers of container # so put all variables here rather than defaults/main.yml file diff --git a/ansible/roles/nova-cell/tasks/cleanup.yml b/ansible/roles/nova-cell/tasks/cleanup.yml new file mode 100644 index 0000000000..d2dc354360 --- /dev/null +++ b/ansible/roles/nova-cell/tasks/cleanup.yml @@ -0,0 +1,26 @@ +--- +- name: Stop and remove containers for unmanaged services + become: true + vars: + service_name: "nova-compute-ironic" + service: "{{ nova_cell_services[service_name] }}" + kolla_container: + action: "stop_and_remove_container" + name: "{{ service.container_name + '_' ~ item }}" + loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}" + when: + - service.iterate | bool + - inventory_hostname not in groups['nova-compute-ironic-' ~ item] + +- name: Removing config for unmanaged services + vars: + service_name: "nova-compute-ironic" + service: "{{ nova_cell_services[service_name] }}" + ansible.builtin.file: + path: "{{ node_config_directory }}/{{ service_name + '-' ~ item }}" + state: "absent" + become: true + loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}" + when: + - service.iterate | bool + - inventory_hostname not in groups['nova-compute-ironic-' ~ item] diff --git a/ansible/roles/nova-cell/tasks/config-compute-ironic.yml b/ansible/roles/nova-cell/tasks/config-compute-ironic.yml new file mode 100644 index 0000000000..811523f2cc --- /dev/null +++ b/ansible/roles/nova-cell/tasks/config-compute-ironic.yml @@ -0,0 +1,66 @@ +--- +- name: "Ensuring config directories exist for Nova Compute Ironic container {{ item }}" + vars: + nova_compute_ironic_id: "{{ item }}" + ansible.builtin.file: + path: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}" + state: "directory" + owner: "{{ config_owner_user }}" + group: "{{ config_owner_group }}" + mode: "0770" + become: true + +- name: "Copying over config.json files for Nova Compute Ironic container {{ item }}" + vars: + nova_compute_ironic_id: "{{ item }}" + ansible.builtin.template: + src: "nova-compute-ironic.json.j2" + dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/config.json" + mode: "0660" + become: true + +- name: "Get Nova Compute Ironic multi-instance configuration for instance {{ item }}" + ansible.builtin.set_fact: + nova_multi_compute_ironic_instance_config: "{{ nova_multi_compute_ironic_config[(item | int) - 1] }}" + +- name: "Generate config files for Nova Compute Ironic {{ item }}" + vars: + service_name: "nova-compute-ironic" + nova_compute_shard_key: "{{ nova_multi_compute_ironic_instance_config.get('shard_key', 'default') }}" + nova_compute_conductor_group: "{{ nova_multi_compute_ironic_instance_config.get('conductor_group', 'default') }}" + nova_compute_ironic_custom_host: "{{ nova_multi_compute_ironic_instance_config.get('custom_host', '') }}" + nova_compute_ironic_id: "{{ item }}" + merge_configs: + sources: + - "{{ role_path }}/templates/nova.conf.j2" + - "{{ node_custom_config }}/global.conf" + - "{{ node_custom_config }}/nova.conf" + - "{{ node_custom_config }}/nova/{{ service_name }}.conf" + - "{{ node_custom_config }}/nova/{{ service_name }}-{{ item }}.conf" + - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/nova.conf" + - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ service_name }}-{{ item }}.conf" + dest: "{{ node_config_directory }}/nova-compute-ironic-{{ item }}/nova.conf" + mode: "0660" + become: true + +- name: "Copying over existing policy file for item {{ item }}" + vars: + service_name: "nova-compute-ironic" + ansible.builtin.template: + src: "{{ nova_policy_file_path }}" + dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/{{ nova_policy_file }}" + mode: "0660" + become: true + when: + - nova_policy_file is defined + +- name: "Copying over vendordata file for Nova Compute Ironic {{ item }}" + vars: + service_name: "nova-compute-ironic" + ansible.builtin.copy: + src: "{{ vendordata_file_path }}" + dest: "{{ node_config_directory }}/{{ service_name }}-{{ item }}/vendordata.json" + mode: "0660" + become: true + when: + - vendordata_file_path is defined diff --git a/ansible/roles/nova-cell/tasks/config.yml b/ansible/roles/nova-cell/tasks/config.yml index dd49e56de0..aad50df60f 100644 --- a/ansible/roles/nova-cell/tasks/config.yml +++ b/ansible/roles/nova-cell/tasks/config.yml @@ -7,6 +7,8 @@ owner: "{{ config_owner_user }}" group: "{{ config_owner_group }}" mode: "0770" + when: + - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0 with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}" - include_tasks: copy-certs.yml @@ -56,6 +58,8 @@ src: "{{ item.key }}.json.j2" dest: "{{ node_config_directory }}/{{ item.key }}/config.json" mode: "0660" + when: + - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0 with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}" - name: Copying over nova.conf @@ -74,8 +78,18 @@ mode: "0660" when: - item.key in nova_cell_services_require_nova_conf + - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0 with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}" +- name: Copying over config for Nova compute Ironic multi-instance + vars: + service: "{{ nova_cell_services['nova-compute-ironic'] }}" + ansible.builtin.include_tasks: config-compute-ironic.yml + loop: "{{ range(1, nova_multi_compute_ironic_config | length + 1) | list }}" + when: + - nova_multi_compute_ironic_config | length > 0 + - inventory_hostname in groups[service.group + '-' ~ item] + - name: Copying over Nova compute provider config become: true vars: @@ -187,6 +201,7 @@ when: - nova_policy_file is defined - item.key in nova_cell_services_require_policy_json + - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0 with_dict: "{{ nova_cell_services | select_services_enabled_and_mapped_to_host }}" - name: Copying over vendordata file to containers @@ -200,6 +215,7 @@ when: - vendordata_file_path is defined - service | service_enabled_and_mapped_to_host + - item.key != "nova-compute-ironic" or nova_multi_compute_ironic_config | length == 0 with_items: - nova-compute - nova-compute-ironic diff --git a/ansible/roles/nova-cell/tasks/deploy.yml b/ansible/roles/nova-cell/tasks/deploy.yml index ab9c91c587..0eb65888d9 100644 --- a/ansible/roles/nova-cell/tasks/deploy.yml +++ b/ansible/roles/nova-cell/tasks/deploy.yml @@ -4,6 +4,10 @@ - import_tasks: version-check.yml +# NOTE(dougszu): Currently only covers nova-compute-ironic +- name: Cleanup stale configuration + ansible.builtin.import_tasks: cleanup.yml + - import_tasks: config-host.yml - import_tasks: config.yml diff --git a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml index 1603af5dea..88329f6e7c 100644 --- a/ansible/roles/nova-cell/tasks/wait_discover_computes.yml +++ b/ansible/roles/nova-cell/tasks/wait_discover_computes.yml @@ -79,9 +79,11 @@ # configure for [DEFAULT] host in nova.conf. ironic_compute_service_hosts: >- {{ ironic_computes_in_batch | - map('extract', hostvars) | json_query('[].nova_compute_ironic_custom_host || [].ansible_facts.hostname') | - map('regex_replace', '^(.*)$', '\1-ironic') | - list }} + map('extract', hostvars) | + json_query( + '{classic: [].nova_compute_ironic_custom_host || [].ansible_facts.hostname, + multi: [].nova_multi_compute_ironic_config}') | + get_expected_ironic_compute_services }} expected_compute_service_hosts: "{{ virt_compute_service_hosts + ironic_compute_service_hosts }}" - name: Include discover_computes.yml diff --git a/ansible/roles/nova-cell/templates/nova.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.j2 index 4aa7781ce1..75ad0782a6 100644 --- a/ansible/roles/nova-cell/templates/nova.conf.j2 +++ b/ansible/roles/nova-cell/templates/nova.conf.j2 @@ -8,9 +8,27 @@ state_path = /var/lib/nova allow_resize_to_same_host = true +{% set nova_compute_shard_key = nova_compute_shard_key | default("") %} +{% set nova_compute_conductor_group = nova_compute_conductor_group | default("") %} +{% set nova_compute_ironic_custom_host = nova_compute_ironic_custom_host | default("") %} {% if service_name == "nova-compute-ironic" %} -host={{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic +{% if nova_compute_ironic_custom_host | length > 0 %} +{# NOTE(dougszu): This is provided for backwards compatibility #} +{% set host_value = nova_compute_ironic_custom_host ~ "-ironic" %} +{% elif nova_compute_conductor_group | length > 0 or nova_compute_shard_key | length > 0 %} +{% set host_value = (nova_compute_conductor_group ~ "-" ~ nova_compute_shard_key) ~ "-ironic" %} +{% else %} +{# NOTE(dougszu): This is to support legacy behaviour #} +{% set host_value = ansible_facts.hostname ~ "-ironic" %} +{% endif %} +host = {{ host_value }} + +{% if nova_compute_ironic_id is defined %} +log_file = /var/log/kolla/nova/nova-compute-ironic-{{ nova_compute_ironic_id }}.log +{% else %} log_file = /var/log/kolla/nova/nova-compute-ironic.log +{% endif %} + compute_driver = ironic.IronicDriver ram_allocation_ratio = 1.0 reserved_host_memory_mb = 0 @@ -96,6 +114,19 @@ project_name = service user_domain_name = {{ default_user_domain_name }} project_domain_name = {{ default_project_domain_name }} endpoint_override = {{ ironic_internal_endpoint }}/v1 + +{% if nova_compute_shard_key is not in [none, 'default'] %} +shard = {{ nova_compute_shard_key }} +{% endif %} +{% if nova_compute_conductor_group is not in [none, 'default'] %} +conductor_group = {{ nova_compute_conductor_group }} +{% endif %} +{% if nova_compute_conductor_group is not in [none, 'default'] and nova_compute_shard_key is not in [none, 'default'] %} +# NOTE(dougszu): In this case only, Nova Compute Ironic won't start unless the peer_list +# is populated. peer_list is a deprecated config option and should never have more than +# one host in it. +peer_list = {{ nova_compute_conductor_group }}-ironic +{% endif %} {% endif %} [oslo_concurrency] @@ -191,7 +222,7 @@ driver = noop [oslo_messaging_rabbit] use_queue_manager = true {% if service_name == "nova-compute-ironic" %} -hostname = {{ nova_compute_ironic_custom_host | default(ansible_facts.hostname) }}-ironic +hostname = {{ host_value }} {% endif %} heartbeat_in_pthread = false {% if om_enable_rabbitmq_tls | bool %} diff --git a/ansible/roles/service-cert-copy/tasks/iterated.yml b/ansible/roles/service-cert-copy/tasks/iterated.yml new file mode 100644 index 0000000000..4afb1d3bc4 --- /dev/null +++ b/ansible/roles/service-cert-copy/tasks/iterated.yml @@ -0,0 +1,10 @@ +--- +- name: "Copying over extra CA certificates for {{ project_name }}" + become: true + ansible.builtin.copy: + src: "{{ kolla_certificates_dir }}/ca/" + dest: "{{ node_config_directory }}/{{ outer_item.key }}-{{ item }}/ca-certificates" + mode: "0644" + loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}" + when: + - inventory_hostname in groups[service.group + '-' ~ item] diff --git a/ansible/roles/service-cert-copy/tasks/main.yml b/ansible/roles/service-cert-copy/tasks/main.yml index e9dd74eab1..2a06e87365 100644 --- a/ansible/roles/service-cert-copy/tasks/main.yml +++ b/ansible/roles/service-cert-copy/tasks/main.yml @@ -7,8 +7,19 @@ mode: "0644" when: - kolla_copy_ca_into_containers | bool + - not (item.value.iterate | default(False)) | bool with_dict: "{{ project_services | select_services_enabled_and_mapped_to_host }}" +# NOTE(dougszu): For current iterable services, we only need CA certs +- name: "Check containers that require iteration for {{ kolla_role_name | default(project_name) }}" + vars: + service: "{{ outer_item.value }}" + ansible.builtin.include_tasks: iterated.yml + loop: "{{ project_services | select_services_enabled_and_mapped_to_host | dict2items }}" + loop_control: + loop_var: outer_item + when: (service.iterate | default(False)) | bool + - name: "{{ project_name }} | Copying over backend internal TLS certificate" vars: certs: diff --git a/ansible/roles/service-check-containers/tasks/iterated.yml b/ansible/roles/service-check-containers/tasks/iterated.yml index 3e2837a87c..009be9fc4f 100644 --- a/ansible/roles/service-check-containers/tasks/iterated.yml +++ b/ansible/roles/service-check-containers/tasks/iterated.yml @@ -23,6 +23,11 @@ command: "{{ service.command | default(omit) }}" cgroupns_mode: "{{ service.cgroupns_mode | default(omit) }}" loop: "{{ range(1, (iterate_count | int) + 1) | list }}" + when: + # NOTE(dougszu): Due to issues with nova-compute-ironic HA, we can't have + # more than one instance for a given 'item'. Eg. nova_compute_ironic_1 + # should never run on more than one host. + - service_name != 'nova-compute-ironic' or (service_name == 'nova-compute-ironic' and inventory_hostname in groups['nova-compute-ironic-' + item | string]) register: container_check # NOTE(yoctozepto): Must be a separate task because one cannot see the whole diff --git a/ansible/roles/service-precheck/tasks/iterated.yml b/ansible/roles/service-precheck/tasks/iterated.yml new file mode 100644 index 0000000000..104aff93c6 --- /dev/null +++ b/ansible/roles/service-precheck/tasks/iterated.yml @@ -0,0 +1,31 @@ +--- +- name: "Validate inventory groups for {{ project_name }}" + vars: + service_name: "{{ item.key }}" + service: "{{ item.value }}" + ansible.builtin.fail: + msg: >- + Ansible inventory does not contain the expected group {{ service.group }} + for service {{ service_name }} in {{ project_name }}. + loop: "{{ query('dict', service_precheck_services) }}" + when: + - not (service.iterate | default(False)) | bool + - "'group' in service" + - service.group not in groups + loop_control: + label: "{{ service_name }}" + +- name: "Validate inventory groups (iterated) for {{ project_name }}" + vars: + service_name: "{{ outer_item.key }}" + service: "{{ outer_item.value }}" + service_group: "{{ service.group + '-' + item | string }}" + ansible.builtin.fail: + msg: >- + Ansible inventory does not contain the expected group {{ service_group }} + for service {{ service_name }} in {{ project_name }}. + loop: "{{ range(1, (service.iterate_var | int) + 1) | list }}" + when: + - (service.iterate | default(False)) | bool + - "'group' in service" + - service_group not in groups diff --git a/ansible/roles/service-precheck/tasks/main.yml b/ansible/roles/service-precheck/tasks/main.yml index 24d0259d06..be579eb64a 100644 --- a/ansible/roles/service-precheck/tasks/main.yml +++ b/ansible/roles/service-precheck/tasks/main.yml @@ -9,7 +9,17 @@ for service {{ service_name }} in {{ project_name }}. loop: "{{ query('dict', service_precheck_services) }}" when: + - not (service.iterate | default(False)) | bool - "'group' in service" - service.group not in groups loop_control: label: "{{ service_name }}" + +- name: Include tasks + vars: + service: "{{ outer_item.value }}" + ansible.builtin.include_tasks: iterated.yml + loop: "{{ query('dict', service_precheck_services) }}" + loop_control: + loop_var: outer_item + when: (service.iterate | default(False)) | bool diff --git a/doc/source/admin/advanced-configuration.rst b/doc/source/admin/advanced-configuration.rst index cb4ceabdcf..f3fe661c68 100644 --- a/doc/source/admin/advanced-configuration.rst +++ b/doc/source/admin/advanced-configuration.rst @@ -176,6 +176,79 @@ operator needs to create ``/etc/kolla/config/global.conf`` with content: [database] max_pool_size = 100 +Large baremetal deployments +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Out of the box, a typical Kolla Ansible deployment can support managing a few +hundred baremetal nodes. Beyond this number, it becomes necessary to configure +scaling mechanisms built into Nova and Ironic. + +There are two mechanisms to consider: + - shards + - conductor groups + +Please see the Ironic documentation for further `information +`_. + +As an example, consider a deployment of 700 baremetal nodes spread over two +distinct locations. Location 1 consists of 500 baremetal nodes, and location 2 +consists of 200 baremetal nodes. + +For location 1, all 500 nodes are placed into the same Ironic conductor +group. Ironic conductors configured to use this group are deployed on each +of the three nodes in the control plane. A single Nova Compute Ironic service +would struggle to manage this many nodes, so two shards are created. Due to +HA limitations with Nova Compute Ironic, a single instance is created for +each shard, and the instances may be deployed to any node in the control +plane. + +For location 2, no shards are required due to the smaller number of baremetal +nodes. A single conductor group is configured. The result is that three Ironic +conductors are deployed (one on each node in the control plane), and a single +instance of Nova Compute Ironic configured to use this conductor group. +Furthermore, for this location ``nova_compute_ironic_custom_host`` is used +to override the automatically generated host field in the configuration. This +is useful for migrating to the multi-instance configuration described here. + +Finally, Ironic and Nova services are deployed with no configured shards or +conductor groups as a catch all. + +.. code-block:: yaml + + nova_multi_compute_ironic_config: + - "shard_key": "shard_1" + "conductor_group": "location_1" + - "shard_key": "shard_2" + "conductor_group": "location_1" + - "conductor_group": "location_2" + "custom_host": "some_custom_host_field" + - {} + +The placement of the above services is managed via the inventory. This +allows flexible placement, should a controller fail. + +.. code-block:: yaml + + [nova-compute-ironic-1] + controller-01 + [nova-compute-ironic-2] + controller-02 + [nova-compute-ironic-3] + controller-03 + [nova-compute-ironic-4] + controller-03 + + [nova-compute-ironic:children] + nova-compute-ironic-1 + nova-compute-ironic-2 + nova-compute-ironic-3 + nova-compute-ironic-4 + +The ``nova-compute-ironic`` service instances may be configured +individually via the existing override mechanism. For example, the +creation of ``nova/nova-compute-ironic-1.conf`` will allow variables +for that specific service to be overridden. + OpenStack policy customisation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/kolla_ansible/nova_filters.py b/kolla_ansible/nova_filters.py index 4bb5cbdf66..c440254f1e 100644 --- a/kolla_ansible/nova_filters.py +++ b/kolla_ansible/nova_filters.py @@ -85,8 +85,31 @@ def _namespace(name): return services +def get_expected_ironic_compute_services(ironic_compute_conf): + """Return a List of expected Ironic compute services + + :param ironic_compute_conf: Nova compute Ironic instance config + :returns: A list of Nova Compute Ironic service hostnames. + """ + classic_config = ironic_compute_conf['classic'] + multi_compute_config = ironic_compute_conf['multi'] + + expected_services = [] + if len(multi_compute_config) == 0: + if len(classic_config) > 0: + expected_services.append(classic_config[0] + '-ironic') + return expected_services + + for item in multi_compute_config[0]: + cg = item.get("conductor_group") or "default" + sk = item.get("shard_key") or "default" + expected_services.append(f"{cg}-{sk}-ironic") + return expected_services + + def get_filters(): return { "extract_cell": extract_cell, "namespace_haproxy_for_cell": namespace_haproxy_for_cell, + "get_expected_ironic_compute_services": get_expected_ironic_compute_services, # noqa } diff --git a/kolla_ansible/tests/unit/test_nova_filters.py b/kolla_ansible/tests/unit/test_nova_filters.py index e8d5c507aa..f11d2a4efd 100644 --- a/kolla_ansible/tests/unit/test_nova_filters.py +++ b/kolla_ansible/tests/unit/test_nova_filters.py @@ -73,6 +73,48 @@ def test_extract_duplicate_cell(self): self.assertRaisesRegex(jinja2.TemplateRuntimeError, 'duplicates', self._test_extract_cell, test_data, 'cell0001') + def test_get_expected_ironic_compute_services_multi_compute(self): + example_ironic_compute_conf = { + 'classic': ['custom-host-nova-compute'], + 'multi': [[ + {}, + {"shard_key": "shard_1", "conductor_group": "location_1"}, + {"shard_key": "shard_2", "conductor_group": "location_1"}, + {"conductor_group": "location_2"}, + {"shard_key": "shard_1"}, + ]] + } + actual = filters.get_expected_ironic_compute_services( + example_ironic_compute_conf) + expected = [ + 'default-default-ironic', + 'location_1-shard_1-ironic', + 'location_1-shard_2-ironic', + 'location_2-default-ironic', + 'default-shard_1-ironic', + ] + self.assertListEqual(actual, expected) + + def test_get_expected_ironic_compute_services_no_compute(self): + example_ironic_compute_conf = { + 'classic': [], + 'multi': [] + } + actual = filters.get_expected_ironic_compute_services( + example_ironic_compute_conf) + expected = [] + self.assertListEqual(actual, expected) + + def test_get_expected_ironic_compute_services_classic_compute(self): + example_ironic_compute_conf = { + 'classic': ['custom-foo'], + 'multi': [] + } + actual = filters.get_expected_ironic_compute_services( + example_ironic_compute_conf) + expected = ['custom-foo-ironic'] + self.assertListEqual(actual, expected) + def test_namespace_haproxy_for_cell_with_empty_name(self): example_services = { 'nova-novncproxy': { diff --git a/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml b/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml new file mode 100644 index 0000000000..3b680770bc --- /dev/null +++ b/releasenotes/notes/add-nova-multi-compute-ironic-support-1bf5c64f8a530cc9.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for deploying multiple instances of the Nova Compute Ironic + service on the same host. This is useful in large baremetal deployments. diff --git a/tests/run.yml b/tests/run.yml index 2dc2c1b9ad..ea0afdc234 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -218,10 +218,10 @@ # ironic.conf - src: "tests/templates/ironic-overrides.j2" dest: /etc/kolla/config/ironic.conf - when: "{{ scenario == 'ironic' }}" + when: "{{ scenario == 'ironic' or scenario == 'multi-compute-ironic' }}" - src: "tests/templates/tenks-deploy-config.yml.j2" dest: "{{ ansible_env.HOME }}/tenks.yml" - when: "{{ scenario == 'ironic' }}" + when: "{{ scenario == 'ironic' or scenario == 'multi-compute-ironic' }}" when: item.when | default(true) - block: @@ -240,7 +240,7 @@ dest: ironic-agent.initramfs - src: "tinyipa-{{ zuul.branch | replace('/', '-') }}.vmlinuz" dest: ironic-agent.kernel - when: scenario == "ironic" + when: scenario == "ironic" or scenario == "multi-compute-ironic" - block: - name: Slurp requirements.yml @@ -429,7 +429,7 @@ EXT_NET_GATEWAY: "{{ neutron_external_network_prefix }}1" EXT_NET_DEMO_ROUTER_ADDR: "{{ neutron_external_network_prefix }}10" SCENARIO: "{{ scenario }}" - when: openstack_core_tested or scenario in ['ironic', 'magnum', 'scenario_nfv', 'zun', 'octavia'] + when: openstack_core_tested or scenario in ['ironic', 'multi-compute-ironic, ''magnum', 'scenario_nfv', 'zun', 'octavia'] - name: Run test-ovn.sh script script: @@ -491,7 +491,8 @@ chdir: "{{ kolla_ansible_src_dir }}" environment: TLS_ENABLED: "{{ tls_enabled }}" - when: scenario == "ironic" + MULTI_COMPUTE_IRONIC: "{{ scenario == 'multi-compute-ironic' }}" + when: scenario == "ironic" or scenario == "multi-compute-ironic" - name: Run test-magnum.sh script script: diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index b0f97e6d74..03b7c27f9c 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -144,6 +144,21 @@ enable_proxysql: "yes" enable_prometheus_proxysql_exporter: "yes" {% endif %} +{% if scenario == "multi-compute-ironic" %} +nova_multi_compute_ironic_config: + - {} + - "shard_key": "shard_1" + "conductor_group": "location_1" + - "shard_key": "shard_2" + "conductor_group": "location_1" + - "conductor_group": "location_2" + +enable_ironic: "yes" +enable_ironic_pxe_filter: "yes" +ironic_dnsmasq_dhcp_ranges: + - range: "10.42.0.2,10.42.0.254,255.255.255.0" +{% endif %} + {% if scenario == "mariadb" %} enable_fluentd: "yes" enable_mariadb: "yes" diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 index 9dc36c5037..5cdb8dbe73 100644 --- a/tests/templates/inventory.j2 +++ b/tests/templates/inventory.j2 @@ -293,8 +293,25 @@ nova [nova-spicehtml5proxy:children] nova +{% if scenario == 'multi-compute-ironic' %} +[nova-compute-ironic-1] +primary +[nova-compute-ironic-2] +secondary1 +[nova-compute-ironic-3] +secondary2 +[nova-compute-ironic-4] +secondary2 + +[nova-compute-ironic:children] +nova-compute-ironic-1 +nova-compute-ironic-2 +nova-compute-ironic-3 +nova-compute-ironic-4 +{% else %} [nova-compute-ironic:children] nova +{% endif %} [nova-serialproxy:children] nova diff --git a/tests/templates/ironic-overrides.j2 b/tests/templates/ironic-overrides.j2 index 19aa737aa6..a038a9c304 100644 --- a/tests/templates/ironic-overrides.j2 +++ b/tests/templates/ironic-overrides.j2 @@ -1,3 +1,17 @@ +[DEFAULT] +enabled_inspect_interfaces = no-inspect, agent +default_inspect_interface = agent + +{% if scenario == "multi-compute-ironic" %} +{% raw %} +{% set cg = 'location_1' if inventory_hostname == 'secondary1' else 'location_2' if inventory_hostname == 'secondary2' %} +{% if cg %} +[conductor] +conductor_group = {{ cg }} +{% endif %} +{% endraw %} +{% endif %} + [neutron] cleaning_network = public1 provisioning_network = public1 diff --git a/tests/test-ironic.sh b/tests/test-ironic.sh index b182dcc52d..9a13303021 100755 --- a/tests/test-ironic.sh +++ b/tests/test-ironic.sh @@ -26,6 +26,13 @@ function test_ironic_logged { openstack baremetal node power off tk0 openstack baremetal node show tk0 openstack baremetal node manage tk0 + if [[ "$MULTI_COMPUTE_IRONIC" = "True" ]]; then + # NOTE(dougszu): The node is registered by Tenks with no conductor + # group set. Setting the conductor group below will move the node to + # another conductor. This *should* be fine. + openstack baremetal node set --shard shard_1 tk0 + openstack baremetal node set --conductor-group location_1 tk0 + fi openstack baremetal node show tk0 openstack baremetal node provide tk0 openstack baremetal node show tk0 diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index e0af65e876..36e82c527e 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -45,6 +45,7 @@ - kolla-ansible-rocky9-ironic - kolla-ansible-debian-ironic - kolla-ansible-ubuntu-ironic + - kolla-ansible-scenario-multi-compute-ironic - kolla-ansible-rocky9-upgrade - kolla-ansible-debian-upgrade: voting: false diff --git a/zuul.d/scenarios/multi-compute-ironic.yaml b/zuul.d/scenarios/multi-compute-ironic.yaml new file mode 100644 index 0000000000..814116a1fd --- /dev/null +++ b/zuul.d/scenarios/multi-compute-ironic.yaml @@ -0,0 +1,47 @@ +--- +- job: + name: kolla-ansible-multi-compute-ironic-base + parent: kolla-ansible-base + voting: false + files: !inherit + - ^ansible/group_vars/all/(nova|ironic).yml + - ^ansible/roles/(nova|nova-cell|ironic)/ + - ^tests/deploy-tenks\.sh$ + - ^tests/templates/ironic-overrides\.j2$ + - ^tests/templates/tenks-deploy-config\.yml\.j2$ + - ^tests/test-dashboard\.sh$ + - ^tests/test-ironic\.sh$ + required-projects: + - openstack/tenks + vars: + scenario: multi-compute-ironic + scenario_images_extra: + - ^dnsmasq + - ^ironic + - ^iscsid + tls_enabled: false + +- job: + name: kolla-ansible-debian-trixie-multi-compute-ironic + parent: kolla-ansible-multi-compute-ironic-base + nodeset: kolla-ansible-debian-trixie-multi-16GB + +- job: + name: kolla-ansible-rocky-10-multi-compute-ironic + parent: kolla-ansible-multi-compute-ironic-base + nodeset: kolla-ansible-rocky-10-multi-16GB + +- job: + name: kolla-ansible-ubuntu-noble-multi-compute-ironic + parent: kolla-ansible-multi-compute-ironic-base + nodeset: kolla-ansible-ubuntu-noble-multi-16GB + +- project-template: + name: kolla-ansible-scenario-multi-compute-ironic + description: | + Runs Kolla-Ansible Nova Multi Compute Ironic scenario jobs. + check: + jobs: + - kolla-ansible-debian-trixie-multi-compute-ironic + - kolla-ansible-rocky-10-multi-compute-ironic + - kolla-ansible-ubuntu-noble-multi-compute-ironic